@delali/narsil 0.3.1-next.155 → 0.3.1-next.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/dist/{analysis-change-3TFIHT5E.mjs → analysis-change-CKEMM2MC.mjs} +25 -18
- package/dist/{react-native/chunks/analysis-change-entry-JLL42PH6.mjs → analysis-change-entry-SEDMJ6MW.mjs} +1 -1
- package/dist/browser/{analysis-change-TC36BU47.mjs → analysis-change-ZKGJFENM.mjs} +42 -32
- package/dist/{analysis-change-entry-KA6JAO4X.mjs → browser/analysis-change-entry-5BFUVMHJ.mjs} +1 -1
- package/dist/browser/{chunk-KKAMCABR.mjs → chunk-25MTVDYD.mjs} +1 -1
- package/dist/browser/{chunk-K5AXJBLL.mjs → chunk-3NLMB5CC.mjs} +3 -3
- package/dist/browser/{chunk-23XRXIBW.mjs → chunk-5YWB3E2W.mjs} +8 -1
- package/dist/browser/chunk-AI5UI46D.mjs +12 -0
- package/dist/browser/{chunk-A5KUEGDY.mjs → chunk-BLT4RHCJ.mjs} +3 -3
- package/dist/browser/{chunk-RRO3EV4C.mjs → chunk-C26SPCGQ.mjs} +8 -7
- package/dist/browser/{chunk-ABCXENU5.mjs → chunk-C4FJ3WDY.mjs} +1 -1
- package/dist/browser/{chunk-3UIEB4QF.mjs → chunk-CXNTHHA7.mjs} +2 -2
- package/dist/browser/{chunk-JIJXMI42.mjs → chunk-DUH6QUVU.mjs} +547 -2
- package/dist/browser/chunk-EKMGCRSR.mjs +61 -0
- package/dist/browser/{chunk-JVBP5EYO.mjs → chunk-GFAZPSWK.mjs} +7 -7
- package/dist/browser/{chunk-CJSYBQ4J.mjs → chunk-HUCSHDTB.mjs} +1 -1
- package/dist/browser/{chunk-RND5CLX3.mjs → chunk-K3IPRX6D.mjs} +52 -21
- package/dist/browser/chunk-MY2XKH22.mjs +40 -0
- package/dist/browser/{chunk-SMEQ36NT.mjs → chunk-N23GL72Q.mjs} +629 -1416
- package/dist/browser/{chunk-LFIWRCEA.mjs → chunk-NXQG7DRA.mjs} +29 -6
- package/dist/browser/{chunk-26A5JFT2.mjs → chunk-TK63WFXK.mjs} +2 -1
- package/dist/browser/{chunk-4EWUGZYZ.mjs → chunk-U4YDIBUT.mjs} +2 -2
- package/dist/browser/{chunk-RQBUNZQW.mjs → chunk-UATIJ4KU.mjs} +2 -2
- package/dist/browser/{chunk-BXM5XOCA.mjs → chunk-UKOWD2K5.mjs} +3 -3
- package/dist/browser/{chunk-FNIFPFB7.mjs → chunk-UTMHUHUV.mjs} +2 -2
- package/dist/browser/{chunk-56W3HQVJ.mjs → chunk-V43N6TZO.mjs} +5 -5
- package/dist/browser/{chunk-K4GCSZLI.mjs → chunk-VLJQW4J6.mjs} +2985 -2985
- package/dist/browser/{chunk-IKDKX5VQ.mjs → chunk-VY5KW3HN.mjs} +3 -3
- package/dist/browser/{chunk-5TCXK5SB.mjs → chunk-WN464PTL.mjs} +1 -1
- package/dist/browser/{chunk-SUDNZKPL.mjs → chunk-X7TXYUKO.mjs} +1 -1
- package/dist/browser/{chunk-XY4VVBBO.mjs → chunk-Y6IBAUNO.mjs} +2 -86
- package/dist/browser/{cold-module-NEUGHUPA.mjs → cold-module-YNUUDNOM.mjs} +18 -15
- package/dist/browser/{durability-wiring-CVGPRO6D.mjs → durability-wiring-GVZEE46O.mjs} +71 -27
- package/dist/browser/{engine-hooks-YXMPHZKZ.mjs → engine-hooks-HBVS6YM4.mjs} +8 -6
- package/dist/browser/geo-index-OTV7VPIR.mjs +1387 -0
- package/dist/browser/{import-OM6QAW45.mjs → import-5ESVMFDJ.mjs} +9 -7
- package/dist/browser/{module-E33X4AOK.mjs → module-NDSQLPMQ.mjs} +14 -11
- package/dist/browser/{module-3HNFCBJ5.mjs → module-TCT66FWA.mjs} +6 -6
- package/dist/browser/{phrase-module-ZXXU7S3B.mjs → phrase-module-HAT3TLZN.mjs} +1 -1
- package/dist/{release-BH3DKW3T.mjs → browser/release-HYXPVJVS.mjs} +0 -1
- package/dist/browser/{restore-hooks-IF2ZROUP.mjs → restore-hooks-IE3GGYHG.mjs} +11 -9
- package/dist/browser/{restore-queue-ZBS7CJA6.mjs → restore-queue-LQBBTODR.mjs} +3 -3
- package/dist/browser/{retention-ASWUHAYI.mjs → retention-TY3L3BFN.mjs} +11 -9
- package/dist/browser/{snapshot-PN7ZW6Q4.mjs → snapshot-5FHPPCNZ.mjs} +27 -24
- package/dist/browser/{start-ZIRB7MQ2.mjs → start-2UZTOBG5.mjs} +12 -10
- package/dist/browser/{write-HMDYCTV2.mjs → write-ULXAN5B4.mjs} +14 -11
- package/dist/{capabilities-Da4ywVVl.d.ts → capabilities-D3AoBJoF.d.ts} +2 -2
- package/dist/{chunk-KBKEXQZA.mjs → chunk-24FNVRKB.mjs} +1 -1
- package/dist/{chunk-VHBNF4MA.mjs → chunk-2ROUFMQX.mjs} +2 -2
- package/dist/{chunk-AILVAOLX.mjs → chunk-4B7JIGQD.mjs} +2 -2
- package/dist/{chunk-DCNKVEDB.mjs → chunk-4LMSZAK6.mjs} +16 -6
- package/dist/{chunk-RSVULW4Q.mjs → chunk-4ZTHGTVX.mjs} +4 -4
- package/dist/{chunk-VQEUR6T6.mjs → chunk-5VV4R6P7.mjs} +3040 -3040
- package/dist/chunk-6HTGOXIK.mjs +875 -0
- package/dist/chunk-7GFE3PEY.mjs +40 -0
- package/dist/{chunk-JUKYYXSP.mjs → chunk-7V5IUP2T.mjs} +1 -31
- package/dist/{chunk-4BYWRYPM.mjs → chunk-A4L7YTKX.mjs} +52 -89
- package/dist/{chunk-J4IWBIVY.mjs → chunk-BNL5OPRV.mjs} +6 -6
- package/dist/{chunk-BY6ZDGYS.mjs → chunk-C7XQYG6U.mjs} +2 -2
- package/dist/{chunk-LC3FDWTX.mjs → chunk-CITFCZKF.mjs} +3 -2
- package/dist/{chunk-KXCVQWOR.mjs → chunk-CQZXZ75V.mjs} +3 -97
- package/dist/{chunk-SY3EWKQY.mjs → chunk-CWS55TEX.mjs} +2 -2
- package/dist/{chunk-3HHQLEEQ.mjs → chunk-EYD7K3H6.mjs} +7 -7
- package/dist/chunk-FB6463IN.mjs +1499 -0
- package/dist/{chunk-KFLK5NA2.mjs → chunk-FCJ553HS.mjs} +1 -1
- package/dist/{chunk-KMUNHGBQ.mjs → chunk-FEZVDOWX.mjs} +169 -253
- package/dist/{chunk-SG3JBNHY.mjs → chunk-FJKKCV4D.mjs} +1 -1
- package/dist/{chunk-BDRIVYOK.mjs → chunk-FUD5XSZ5.mjs} +1 -1
- package/dist/chunk-G6HRSGOU.mjs +74 -0
- package/dist/{chunk-S2EQU2GE.mjs → chunk-GPWMUPCA.mjs} +10 -10
- package/dist/{chunk-ZHCUZZE5.mjs → chunk-HHBUCIUY.mjs} +1 -1
- package/dist/{chunk-YEDIDCR7.mjs → chunk-I5B7RXU7.mjs} +91 -22
- package/dist/{chunk-2MANFZJ3.mjs → chunk-IIRIG7H5.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-KWRW5U65.mjs → chunk-J4IJTTEX.mjs} +1 -1
- package/dist/{chunk-Q622CTMC.mjs → chunk-JSV6OQ4C.mjs} +1 -1
- package/dist/{chunk-NF4DT3IH.mjs → chunk-JTYPJF3T.mjs} +8 -1
- package/dist/chunk-KCQYFO4K.mjs +62 -0
- package/dist/{chunk-Q7G3BV3Y.mjs → chunk-KKQC526K.mjs} +19 -11
- package/dist/{chunk-PCRA7HTI.mjs → chunk-LDNCLRZU.mjs} +1 -1
- package/dist/{chunk-U4BS74YN.mjs → chunk-MW6BR4QP.mjs} +2 -2
- package/dist/{chunk-DIBHI6W5.mjs → chunk-MWD5VPN7.mjs} +2 -2
- package/dist/{chunk-UZRZKNNZ.mjs → chunk-ODYKO5TO.mjs} +1 -1
- package/dist/{chunk-PHQNACDT.mjs → chunk-OG7UGJ6F.mjs} +2 -2
- package/dist/{chunk-QH3YIQP4.mjs → chunk-OQLD2BSW.mjs} +2 -2
- package/dist/chunk-ORN5BLL7.mjs +11 -0
- package/dist/{chunk-L7ZD4L3K.mjs → chunk-OYABFKTW.mjs} +7 -6
- package/dist/{chunk-KBDBPVLN.mjs → chunk-P5LANSH6.mjs} +1 -1
- package/dist/{chunk-ZAUJFLEN.mjs → chunk-PIH5SJ4N.mjs} +122 -1697
- package/dist/{chunk-DUOLGBTE.mjs → chunk-PSZI4O24.mjs} +51 -4
- package/dist/{chunk-4KRC2APY.mjs → chunk-PV2CG7EJ.mjs} +5 -567
- package/dist/{chunk-6L3XJXGU.mjs → chunk-PWGYQXML.mjs} +1 -1
- package/dist/{chunk-5FVU47TB.mjs → chunk-QLDABJZM.mjs} +3 -3
- package/dist/{chunk-KPIV4RUC.mjs → chunk-RCX4WNLC.mjs} +2 -2
- package/dist/{chunk-3HB3PEC6.mjs → chunk-RIHIVFIQ.mjs} +6 -6
- package/dist/chunk-RLOES5SX.mjs +31 -0
- package/dist/{chunk-H6VIGE62.mjs → chunk-RWURUDSP.mjs} +10 -9
- package/dist/{chunk-ZTW5WTGG.mjs → chunk-SMOOSIAW.mjs} +4 -4
- package/dist/{chunk-MT2NRTA6.mjs → chunk-SOI6LTHM.mjs} +3 -3
- package/dist/chunk-SQJWAIF4.mjs +12 -0
- package/dist/{react-native/chunks/chunk-PPEPTUVV.mjs → chunk-TBC3XUL2.mjs} +29 -6
- package/dist/chunk-TDIK6NTV.mjs +10 -0
- package/dist/{react-native/chunks/chunk-JV6VQY2S.mjs → chunk-U55XJEIG.mjs} +2 -1
- package/dist/{chunk-H4KHPLHJ.mjs → chunk-UD5HIS4U.mjs} +1 -1
- package/dist/{chunk-HW5TNXBO.mjs → chunk-UJDJB3NE.mjs} +40 -31
- package/dist/{chunk-QUJHMPTP.mjs → chunk-VSLSGASN.mjs} +2 -2
- package/dist/{chunk-Z2BE2MAM.mjs → chunk-VZ2ONHEE.mjs} +32 -15
- package/dist/chunk-W5EB5XKI.mjs +741 -0
- package/dist/{chunk-3R4UZWQG.mjs → chunk-W7OSSKH5.mjs} +50 -20
- package/dist/{chunk-SMVH6JBE.mjs → chunk-XXJZMTZT.mjs} +1 -1
- package/dist/{chunk-VGF7CFRP.mjs → chunk-ZPZ3YGXV.mjs} +5 -4
- package/dist/{client-DRv7UVeY.d.ts → client-kqSMevVi.d.ts} +15 -3
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +4 -0
- package/dist/{cold-module-7XXCEGTZ.mjs → cold-module-VOJXMNZB.mjs} +31 -24
- package/dist/{config-DMwffLKn.d.ts → config-B4v_Htpu.d.ts} +1 -1
- package/dist/{config-CoSig5Ox.d.ts → config-CTP01fns.d.ts} +1 -1
- package/dist/direct-executor-V5DRIECI.mjs +34 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +94 -80
- package/dist/{durability-wiring-NCKUC5EO.mjs → durability-wiring-3TD62MBI.mjs} +57 -42
- package/dist/{engine-CFxPYqEk.d.ts → engine-CkPn2PWE.d.ts} +1 -1
- package/dist/{engine-hooks-DHCMOADY.mjs → engine-hooks-HA3SBOBR.mjs} +24 -17
- package/dist/geo-index-MRH5JMIR.mjs +1387 -0
- package/dist/{import-NBFXFCSG.mjs → import-GPKLTSKA.mjs} +24 -17
- package/dist/index.browser.mjs +7 -5
- package/dist/index.d.ts +131 -6
- package/dist/index.mjs +23 -16
- package/dist/index.react-native.mjs +12 -10
- package/dist/invalidation/filesystem.mjs +2 -2
- package/dist/{long-query-thread-J4DVRM4A.mjs → long-query-thread-QDFQSJD6.mjs} +26 -21
- package/dist/{module-DWKPBSMN.mjs → module-JYHFTVPS.mjs} +28 -20
- package/dist/{module-UN76TF7I.mjs → module-N55W4Q5C.mjs} +9 -9
- package/dist/{paged-partition-archive-DPNWVSK2.mjs → paged-partition-archive-RLRGOXDX.mjs} +33 -27
- package/dist/partial-words.mjs +8 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +32 -22
- package/dist/{phrase-module-ZI4WSQVH.mjs → phrase-module-XW5MKWP5.mjs} +3 -2
- package/dist/query-suggestions/merge-worker.mjs +25 -18
- package/dist/query-suggestions.browser.mjs +13 -11
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +34 -27
- package/dist/react-native/chunks/{analysis-change-6BPDPH6K.mjs → analysis-change-QUVDNABS.mjs} +46 -36
- package/dist/{browser/analysis-change-entry-4MH3CZXH.mjs → react-native/chunks/analysis-change-entry-OC4XASBF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-7PXRWYUU.mjs → chunk-2PRHILX3.mjs} +53 -19
- package/dist/{chunk-R6A773WK.mjs → react-native/chunks/chunk-3WI35N75.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GIZTJXYW.mjs → chunk-4AV5PPH6.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-4Y53T4YE.mjs → chunk-4COWCEY7.mjs} +189 -178
- package/dist/react-native/chunks/{chunk-OL3WDV53.mjs → chunk-4NWR3NGS.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-C3PTWYKT.mjs → chunk-5MSX5FRT.mjs} +5 -5
- package/dist/react-native/chunks/{chunk-6WA3C3DV.mjs → chunk-6OFDUYVX.mjs} +8 -1
- package/dist/react-native/chunks/{chunk-4LCEN2IU.mjs → chunk-6ZELFCK5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-CUH5F57W.mjs → chunk-BITYFZSZ.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TYIRMPD3.mjs → chunk-DMIFXZ3Q.mjs} +1 -1
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +12 -0
- package/dist/{chunk-YTJCGKM4.mjs → react-native/chunks/chunk-EJ33LCHV.mjs} +29 -6
- package/dist/react-native/chunks/{chunk-EIOYM5TO.mjs → chunk-ENLMYKSX.mjs} +3024 -3054
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +40 -0
- package/dist/{chunk-4UMZ7KQL.mjs → react-native/chunks/chunk-FTZHMRD2.mjs} +2 -1
- package/dist/react-native/chunks/{chunk-ALOIBXWN.mjs → chunk-IAENKE2D.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-QJXTNPLQ.mjs → chunk-KSBYEMCB.mjs} +1 -1
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +164 -0
- package/dist/react-native/chunks/{chunk-HUOYZJ3Q.mjs → chunk-OCSLSH27.mjs} +1561 -933
- package/dist/react-native/chunks/chunk-OXM66LO6.mjs +62 -0
- package/dist/react-native/chunks/{chunk-KTTAU63H.mjs → chunk-SK2PWYDH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-PEHS2UO3.mjs → chunk-TORU2OHQ.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-VXGE4ZXL.mjs → chunk-WVWJNUMN.mjs} +51 -21
- package/dist/react-native/chunks/{chunk-KJIG57UD.mjs → chunk-WW2X2AEI.mjs} +7 -7
- package/dist/react-native/chunks/{chunk-JLIIWJUV.mjs → chunk-YZW4CHM6.mjs} +50 -11
- package/dist/react-native/chunks/{chunk-GJFXPHXI.mjs → chunk-ZNZ6CPAZ.mjs} +2 -87
- package/dist/react-native/chunks/{chunk-7GRCBBME.mjs → chunk-ZVQMBF2H.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-N2WUOWLL.mjs → chunk-ZWAIN4GG.mjs} +7 -7
- package/dist/react-native/chunks/{cold-module-5PHF6DZR.mjs → cold-module-N6KFNNZA.mjs} +20 -17
- package/dist/react-native/chunks/{durability-wiring-KO565FZC.mjs → durability-wiring-RN64S2TL.mjs} +76 -32
- package/dist/react-native/chunks/{engine-hooks-MVRRHH6B.mjs → engine-hooks-CWA5CKCA.mjs} +13 -11
- package/dist/react-native/chunks/geo-index-PX3BIHAH.mjs +1387 -0
- package/dist/react-native/chunks/{import-YVZA4V4V.mjs → import-5UAWBRJP.mjs} +14 -12
- package/dist/react-native/chunks/{module-ZL5ZG6YB.mjs → module-6RP4ZCBW.mjs} +17 -14
- package/dist/react-native/chunks/{module-H5MVPSP2.mjs → module-TTEVPRCW.mjs} +8 -8
- package/dist/react-native/chunks/{phrase-module-RTVKEWJT.mjs → phrase-module-QK22HSQ2.mjs} +1 -1
- package/dist/react-native/chunks/{release-D7ZTIANM.mjs → release-2SIHZWDO.mjs} +0 -1
- package/dist/react-native/chunks/{restore-hooks-SILFPU6B.mjs → restore-hooks-ENFMDTWC.mjs} +17 -15
- package/dist/react-native/chunks/{restore-queue-HSYP2AEC.mjs → restore-queue-4R6AASE5.mjs} +3 -3
- package/dist/react-native/chunks/{retention-7I7UAWYK.mjs → retention-5GTFT3GR.mjs} +18 -16
- package/dist/react-native/chunks/{snapshot-EZYB6SLF.mjs → snapshot-5LGPQXFS.mjs} +32 -29
- package/dist/react-native/chunks/{start-SHVZ72I5.mjs → start-ZQJVZ5DN.mjs} +18 -16
- package/dist/react-native/chunks/{write-JUESXHDL.mjs → write-6BRYIIZ7.mjs} +17 -14
- package/dist/react-native/partial-words.mjs +7 -6
- package/dist/react-native/pattern.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +19 -17
- package/dist/react-native/sync.mjs +47 -13
- package/dist/react-native/synonyms.mjs +92 -24
- package/dist/react.d.ts +7 -7
- package/dist/{browser/release-76MXRPGV.mjs → release-SMR4WJD4.mjs} +0 -1
- package/dist/{restore-hooks-HZ3ILAMO.mjs → restore-hooks-HXLIEB4A.mjs} +28 -21
- package/dist/{restore-queue-JVGENGGE.mjs → restore-queue-LZH7LMLR.mjs} +3 -3
- package/dist/{retention-HROPV2QU.mjs → retention-3K7ARGAG.mjs} +29 -22
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +55 -41
- package/dist/{snapshot-4HS2G2UM.mjs → snapshot-O7ICP7NJ.mjs} +46 -39
- package/dist/{start-UTWHGH42.mjs → start-S4PFP6RS.mjs} +31 -24
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +3 -3
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +46 -12
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +46 -12
- package/dist/synonyms.mjs +7 -6
- package/dist/{tasks-qrJojgST.d.ts → tasks-CoNjxbop.d.ts} +2 -2
- package/dist/{thread-6MOXY2KD.mjs → thread-YLJ3ZPQ3.mjs} +25 -20
- package/dist/{types-BZCGjfLn.d.ts → types-DPMjfOr_.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +17 -6
- package/dist/{write-GTTWWRRG.mjs → write-2OBJGLFS.mjs} +27 -19
- package/openapi.json +71 -0
- package/package.json +15 -8
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-HNVVEQP7.mjs +0 -44
- package/dist/chunk-WQRN64Q6.mjs +0 -196
- package/dist/direct-executor-G5TK7NLA.mjs +0 -29
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +0 -191
- package/dist/browser/{chunk-4WCVMOU3.mjs → chunk-XAWNZ4G2.mjs} +10 -10
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { emitSyncChange, withOpenIndex, oneAtATime } from './chunk-ML7THVLJ.mjs';
|
|
2
2
|
import { createRequestId, isValidWorkerResponse } from './chunk-VI7BQR4U.mjs';
|
|
3
3
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
4
|
-
import { parsedTypeOf, flattenSchema, isSingleStringFieldType, partialFieldsOf, resolveIndexAnalysis, schemaFieldsOf, isWordIndexedBase, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireWithinFileLimit, HEADER_SIZE, payloadByteLength, checksumOfChunks, packEnvelopeWithEngineVersion, INDEX_CONFIGURATION_ENVELOPE_VERSION, validateSchema, requireCreatableSchema, getLanguage,
|
|
4
|
+
import { parsedTypeOf, isLatitude, isLongitude, flattenSchema, isSingleStringFieldType, partialFieldsOf, resolveIndexAnalysis, getNestedValue, schemaFieldsOf, tokenizeOptions, isWordIndexedBase, raiseEarlierReleaseGraphSetting, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, requireWithinFileLimit, HEADER_SIZE, payloadByteLength, checksumOfChunks, packEnvelopeWithEngineVersion, messagePackUpperBoundBytes, INDEX_CONFIGURATION_ENVELOPE_VERSION, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, getLanguage, validateDocument, validateDocumentStrict, textFieldsChanged, PROTOTYPE_POLLUTION_KEYS, encodeDocument, extractVectorFieldsFromSchema, getFlatSchema, isPlainObject, isWordIndexedFieldType, getOrCreateFieldNameIndex, readHeapStatistics, isStringValuedBase, MAX_NESTING_DEPTH, withSortableOption, isGeopointOrVectorType, isListBase, VECTOR_PATTERN, validateRequiredFields, requireDocumentWithinSizeLimit } from './chunk-DUH6QUVU.mjs';
|
|
5
5
|
import { fnv1a, FNV_OFFSET_BASIS, fnv1aContinue, fnv1aContinueTaggedNumbers } from './chunk-RA6LSBCM.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { highWord, SMALLEST_NORMAL_HIGH_WORD, MAGNITUDE_MASK, lowWord, INFINITE_HIGH_WORD, withHighWord } from './chunk-MY2XKH22.mjs';
|
|
7
|
+
import { MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS } from './chunk-AI5UI46D.mjs';
|
|
8
|
+
import { TARGET_BLOCK_ENTRIES, bitsetHas, createBitSet, LOOKUP_KEY_SCRATCH_BYTES, GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, bitsetIsEmpty, bitsetSet, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, FIELD_RUN_ENTRIES_START, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-TK63WFXK.mjs';
|
|
9
|
+
import { createDocIdMap, turnsAreBudgeted, turnIsSpent, yieldTurn, indexChangeOutsideExternalWaits, changeIndexOutsideExternalWaits, detectRuntime, resolveWorkerEntry, missingWorkerEntry, countWrite, MIN_OVERSAMPLE, noteThreadSlotGone, threadSlotOfWorker } from './chunk-XAWNZ4G2.mjs';
|
|
8
10
|
import { splitWorkerBudget, resolveWorkerCount, resolveRequestThreadCount, DEFAULT_REQUEST_TIMEOUT_MS, MAX_RENUMBERED_SEARCH_ATTEMPTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-TY7L6OXO.mjs';
|
|
9
11
|
import { MAX_INDEX_CONFIGURATION_BYTES, BATCH_CHUNK_SIZE, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-4RD3AFSB.mjs';
|
|
10
|
-
import { singleFoldTable, appendFoldedCodePoint,
|
|
12
|
+
import { singleFoldTable, appendFoldedCodePoint, compareCodePoints, toReducedSortValue, compareComparableValues, defaultSortMode, withUnpairedSurrogatesReplaced, holdsUnpairedSurrogate, readSortField, truncateSortString, toComparableSortValue, exceedsCodePoints, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, compareSortValues, codePointCount } from './chunk-NXQG7DRA.mjs';
|
|
11
13
|
import { BM25_DISPLAYED_NAME_LENGTH, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-WXLBZJMT.mjs';
|
|
12
14
|
import { ErrorCodes, NarsilError, createNarsilError } from './chunk-SRMZXJSL.mjs';
|
|
13
15
|
import { Decoder, decode, encode } from '@msgpack/msgpack';
|
|
@@ -215,6 +217,7 @@ var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
|
|
|
215
217
|
var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
|
|
216
218
|
var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
|
|
217
219
|
var DEFAULT_MIN_TOKEN_LENGTH = 1;
|
|
220
|
+
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
218
221
|
|
|
219
222
|
// src/core/tokenizer/normalize.ts
|
|
220
223
|
var DOTTED_CAPITAL_I = "\u0130";
|
|
@@ -356,21 +359,21 @@ function evictOldestEntries(count) {
|
|
|
356
359
|
}
|
|
357
360
|
}
|
|
358
361
|
}
|
|
362
|
+
function transformInTypeScript(raw, language, stem, removeDiacritics) {
|
|
363
|
+
let normalized = raw;
|
|
364
|
+
if (language.normalizer) normalized = language.normalizer(normalized);
|
|
365
|
+
if (removeDiacritics) normalized = stripDiacritics(normalized);
|
|
366
|
+
if (stem && language.stemmer) normalized = language.stemmer(normalized);
|
|
367
|
+
return normalized;
|
|
368
|
+
}
|
|
359
369
|
function transformToken(raw, language, stem, removeDiacritics) {
|
|
360
370
|
const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
|
|
361
371
|
let bucket = getCacheBucket(language, flags);
|
|
362
372
|
const cached = bucket.get(raw);
|
|
363
373
|
if (cached !== void 0) return cached;
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
}
|
|
368
|
-
if (removeDiacritics) {
|
|
369
|
-
normalized = stripDiacritics(normalized);
|
|
370
|
-
}
|
|
371
|
-
if (stem && language.stemmer) {
|
|
372
|
-
normalized = language.stemmer(normalized);
|
|
373
|
-
}
|
|
374
|
+
const normalized = withUnpairedSurrogatesReplaced(
|
|
375
|
+
transformInTypeScript(raw, language, stem, removeDiacritics)
|
|
376
|
+
);
|
|
374
377
|
if (normalizationCacheSize >= maxCacheSize) {
|
|
375
378
|
evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
|
|
376
379
|
bucket = getCacheBucket(language, flags);
|
|
@@ -532,7 +535,6 @@ function producesSurfaceForms(language, options) {
|
|
|
532
535
|
if (!options?.collectSurfaces || options.customTokenizer) return false;
|
|
533
536
|
return (options.stem ?? true) && language.stemmer !== null;
|
|
534
537
|
}
|
|
535
|
-
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
536
538
|
function resolveStopWords(language, override) {
|
|
537
539
|
if (!override) return language.stopWords;
|
|
538
540
|
if (override instanceof Set) return override;
|
|
@@ -562,6 +564,18 @@ function stripPossessive(token) {
|
|
|
562
564
|
}
|
|
563
565
|
return token;
|
|
564
566
|
}
|
|
567
|
+
function customTokensOf(tokenizer, text) {
|
|
568
|
+
const entries = tokenizer.tokenize(text);
|
|
569
|
+
let repaired = null;
|
|
570
|
+
for (let at = 0; at < entries.length; at++) {
|
|
571
|
+
const entry = entries[at];
|
|
572
|
+
const token = withUnpairedSurrogatesReplaced(entry.token);
|
|
573
|
+
if (token === entry.token) continue;
|
|
574
|
+
repaired ??= entries.slice();
|
|
575
|
+
repaired[at] = { token, position: entry.position };
|
|
576
|
+
}
|
|
577
|
+
return repaired ?? entries;
|
|
578
|
+
}
|
|
565
579
|
function visitTokens(text, language, options, visit) {
|
|
566
580
|
const {
|
|
567
581
|
stem = true,
|
|
@@ -572,7 +586,9 @@ function visitTokens(text, language, options, visit) {
|
|
|
572
586
|
stopWordOverride
|
|
573
587
|
} = options ?? {};
|
|
574
588
|
if (customTokenizer) {
|
|
575
|
-
for (const entry of customTokenizer
|
|
589
|
+
for (const entry of customTokensOf(customTokenizer, text)) {
|
|
590
|
+
visit(entry.token, entry.position, void 0, entry.token);
|
|
591
|
+
}
|
|
576
592
|
return;
|
|
577
593
|
}
|
|
578
594
|
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
@@ -591,10 +607,15 @@ function visitTokens(text, language, options, visit) {
|
|
|
591
607
|
visit(processed, position, void 0, part);
|
|
592
608
|
continue;
|
|
593
609
|
}
|
|
594
|
-
const
|
|
595
|
-
visit(processed, position,
|
|
610
|
+
const shown = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
611
|
+
visit(processed, position, surfaceOf(shown, processed), part);
|
|
596
612
|
}
|
|
597
613
|
}
|
|
614
|
+
function surfaceOf(shown, term) {
|
|
615
|
+
if (shown === term) return void 0;
|
|
616
|
+
const surface = withUnpairedSurrogatesReplaced(shown);
|
|
617
|
+
return surface === term ? void 0 : surface;
|
|
618
|
+
}
|
|
598
619
|
function visitSurfaceSpellings(text, language, visit) {
|
|
599
620
|
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
600
621
|
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
@@ -603,12 +624,14 @@ function visitSurfaceSpellings(text, language, visit) {
|
|
|
603
624
|
for (const part of rawParts) {
|
|
604
625
|
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
605
626
|
if (candidate.length < minLength) continue;
|
|
606
|
-
visit(
|
|
627
|
+
visit(
|
|
628
|
+
removeDiacritics ? transformToken(candidate, language, false, true) : withUnpairedSurrogatesReplaced(candidate)
|
|
629
|
+
);
|
|
607
630
|
}
|
|
608
631
|
}
|
|
609
632
|
function tokenize(text, language, options) {
|
|
610
633
|
if (options?.customTokenizer) {
|
|
611
|
-
const customResult = options.customTokenizer
|
|
634
|
+
const customResult = customTokensOf(options.customTokenizer, text);
|
|
612
635
|
return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
|
|
613
636
|
}
|
|
614
637
|
const tokens = [];
|
|
@@ -912,503 +935,6 @@ function registerEngineRerankAdapter(core, name, adapter) {
|
|
|
912
935
|
core.rerankAdapters.set(name, adapter);
|
|
913
936
|
}
|
|
914
937
|
|
|
915
|
-
// src/geo/constants.ts
|
|
916
|
-
var DEG_TO_RAD = Math.PI / 180;
|
|
917
|
-
var MAX_LATITUDE_DEGREES = 90;
|
|
918
|
-
var MAX_LONGITUDE_DEGREES = 180;
|
|
919
|
-
var MIN_POLYGON_POINTS = 3;
|
|
920
|
-
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
921
|
-
var FULL_TURN_DEGREES = 360;
|
|
922
|
-
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
923
|
-
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
924
|
-
var MAX_COVERING_CELLS = 16;
|
|
925
|
-
var RADIUS_BOX_MARGIN_RATIO = 1.02;
|
|
926
|
-
var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
|
|
927
|
-
var VINCENTY_MAX_ITERATIONS = 200;
|
|
928
|
-
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
929
|
-
|
|
930
|
-
// src/geo/coordinates.ts
|
|
931
|
-
function isLatitude(value) {
|
|
932
|
-
return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
|
|
933
|
-
}
|
|
934
|
-
function isLongitude(value) {
|
|
935
|
-
return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
// src/schema/validator/field-values.ts
|
|
939
|
-
function validateGeopoint(path, value) {
|
|
940
|
-
if (!isPlainObject(value)) {
|
|
941
|
-
throw new NarsilError(
|
|
942
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
943
|
-
`Field "${path}" expected a geopoint object with lat and lon`,
|
|
944
|
-
{
|
|
945
|
-
field: path,
|
|
946
|
-
received: Array.isArray(value) ? "array" : typeof value
|
|
947
|
-
}
|
|
948
|
-
);
|
|
949
|
-
}
|
|
950
|
-
if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
|
|
951
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
|
|
952
|
-
field: path,
|
|
953
|
-
lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
|
|
954
|
-
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
955
|
-
});
|
|
956
|
-
}
|
|
957
|
-
if (!isLatitude(value.lat)) {
|
|
958
|
-
throw new NarsilError(
|
|
959
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
960
|
-
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
961
|
-
{
|
|
962
|
-
field: path,
|
|
963
|
-
lat: value.lat
|
|
964
|
-
}
|
|
965
|
-
);
|
|
966
|
-
}
|
|
967
|
-
if (!isLongitude(value.lon)) {
|
|
968
|
-
throw new NarsilError(
|
|
969
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
970
|
-
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
971
|
-
{
|
|
972
|
-
field: path,
|
|
973
|
-
lon: value.lon
|
|
974
|
-
}
|
|
975
|
-
);
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
function validateVector(path, value, expectedDimension) {
|
|
979
|
-
const isArray = Array.isArray(value);
|
|
980
|
-
const isFloat32 = value instanceof Float32Array;
|
|
981
|
-
if (!isArray && !isFloat32) {
|
|
982
|
-
throw new NarsilError(
|
|
983
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
984
|
-
`Field "${path}" expected a vector (Array or Float32Array)`,
|
|
985
|
-
{ field: path, received: typeof value }
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
const arr = value;
|
|
989
|
-
if (arr.length !== expectedDimension) {
|
|
990
|
-
throw new NarsilError(
|
|
991
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
992
|
-
`Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
|
|
993
|
-
{ field: path, expected: expectedDimension, received: arr.length }
|
|
994
|
-
);
|
|
995
|
-
}
|
|
996
|
-
for (let i = 0; i < arr.length; i++) {
|
|
997
|
-
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
998
|
-
throw new NarsilError(
|
|
999
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1000
|
-
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
1001
|
-
{ field: path, index: i, value: arr[i] }
|
|
1002
|
-
);
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
function validateTypedArray(path, value, elementType) {
|
|
1007
|
-
if (!Array.isArray(value)) {
|
|
1008
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
|
|
1009
|
-
field: path,
|
|
1010
|
-
received: typeof value
|
|
1011
|
-
});
|
|
1012
|
-
}
|
|
1013
|
-
for (let i = 0; i < value.length; i++) {
|
|
1014
|
-
const el = value[i];
|
|
1015
|
-
if (typeof el !== elementType) {
|
|
1016
|
-
throw new NarsilError(
|
|
1017
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1018
|
-
`Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
|
|
1019
|
-
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
1020
|
-
);
|
|
1021
|
-
}
|
|
1022
|
-
if (elementType === "number" && !Number.isFinite(el)) {
|
|
1023
|
-
throw new NarsilError(
|
|
1024
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1025
|
-
`Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
|
|
1026
|
-
{ field: path, index: i }
|
|
1027
|
-
);
|
|
1028
|
-
}
|
|
1029
|
-
}
|
|
1030
|
-
}
|
|
1031
|
-
function validatePatternValue(path, value, patternValueLimit, index) {
|
|
1032
|
-
if (!exceedsCodePoints(value, patternValueLimit)) return;
|
|
1033
|
-
const subject = index === void 0 ? `The value of field "${path}"` : `Element ${index} of field "${path}"`;
|
|
1034
|
-
throw new NarsilError(
|
|
1035
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1036
|
-
`${subject} is longer than ${patternValueLimit} code points, which is the pattern value limit of this index`,
|
|
1037
|
-
{ field: path, limit: patternValueLimit, ...index === void 0 ? {} : { index } }
|
|
1038
|
-
);
|
|
1039
|
-
}
|
|
1040
|
-
function validateFieldValue(path, value, type, patternValueLimit) {
|
|
1041
|
-
switch (type.base) {
|
|
1042
|
-
case "string":
|
|
1043
|
-
case "verbatim":
|
|
1044
|
-
if (typeof value !== "string") {
|
|
1045
|
-
throw new NarsilError(
|
|
1046
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1047
|
-
`Field "${path}" expected string, got ${typeof value}`,
|
|
1048
|
-
{
|
|
1049
|
-
field: path,
|
|
1050
|
-
expected: "string",
|
|
1051
|
-
received: typeof value
|
|
1052
|
-
}
|
|
1053
|
-
);
|
|
1054
|
-
}
|
|
1055
|
-
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
1056
|
-
return;
|
|
1057
|
-
case "number":
|
|
1058
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
1059
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
|
|
1060
|
-
field: path,
|
|
1061
|
-
expected: "number",
|
|
1062
|
-
received: typeof value === "number" ? String(value) : typeof value
|
|
1063
|
-
});
|
|
1064
|
-
}
|
|
1065
|
-
return;
|
|
1066
|
-
case "boolean":
|
|
1067
|
-
if (typeof value !== "boolean") {
|
|
1068
|
-
throw new NarsilError(
|
|
1069
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1070
|
-
`Field "${path}" expected boolean, got ${typeof value}`,
|
|
1071
|
-
{
|
|
1072
|
-
field: path,
|
|
1073
|
-
expected: "boolean",
|
|
1074
|
-
received: typeof value
|
|
1075
|
-
}
|
|
1076
|
-
);
|
|
1077
|
-
}
|
|
1078
|
-
return;
|
|
1079
|
-
case "enum":
|
|
1080
|
-
if (typeof value !== "string") {
|
|
1081
|
-
throw new NarsilError(
|
|
1082
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1083
|
-
`Field "${path}" expected enum (string), got ${typeof value}`,
|
|
1084
|
-
{
|
|
1085
|
-
field: path,
|
|
1086
|
-
expected: "enum",
|
|
1087
|
-
received: typeof value
|
|
1088
|
-
}
|
|
1089
|
-
);
|
|
1090
|
-
}
|
|
1091
|
-
return;
|
|
1092
|
-
case "geopoint":
|
|
1093
|
-
validateGeopoint(path, value);
|
|
1094
|
-
return;
|
|
1095
|
-
case "string[]":
|
|
1096
|
-
case "verbatim[]":
|
|
1097
|
-
validateTypedArray(path, value, "string");
|
|
1098
|
-
if (type.pattern) {
|
|
1099
|
-
const elements = value;
|
|
1100
|
-
for (let i = 0; i < elements.length; i++) validatePatternValue(path, elements[i], patternValueLimit, i);
|
|
1101
|
-
}
|
|
1102
|
-
return;
|
|
1103
|
-
case "number[]":
|
|
1104
|
-
validateTypedArray(path, value, "number");
|
|
1105
|
-
return;
|
|
1106
|
-
case "boolean[]":
|
|
1107
|
-
validateTypedArray(path, value, "boolean");
|
|
1108
|
-
return;
|
|
1109
|
-
case "enum[]":
|
|
1110
|
-
validateTypedArray(path, value, "string");
|
|
1111
|
-
return;
|
|
1112
|
-
case "vector":
|
|
1113
|
-
validateVector(path, value, type.dimension);
|
|
1114
|
-
return;
|
|
1115
|
-
}
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
// src/schema/validator/storable.ts
|
|
1119
|
-
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
1120
|
-
var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
|
|
1121
|
-
function reject(path, reason) {
|
|
1122
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
1123
|
-
field: path
|
|
1124
|
-
});
|
|
1125
|
-
}
|
|
1126
|
-
function isPlainObject2(value) {
|
|
1127
|
-
const proto = Object.getPrototypeOf(value);
|
|
1128
|
-
return proto === Object.prototype || proto === null;
|
|
1129
|
-
}
|
|
1130
|
-
function walkStorable(value, path, depth, ancestors, skip) {
|
|
1131
|
-
if (value === null || value === void 0) return;
|
|
1132
|
-
const kind = typeof value;
|
|
1133
|
-
if (kind === "string") {
|
|
1134
|
-
if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
|
|
1135
|
-
return;
|
|
1136
|
-
}
|
|
1137
|
-
if (kind === "boolean" || kind === "number") return;
|
|
1138
|
-
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
1139
|
-
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
1140
|
-
}
|
|
1141
|
-
const container = value;
|
|
1142
|
-
if (container instanceof Date) {
|
|
1143
|
-
if (Number.isNaN(container.getTime())) reject(path, "holds an invalid date, which the engine cannot store");
|
|
1144
|
-
return;
|
|
1145
|
-
}
|
|
1146
|
-
if (container instanceof Uint8Array) return;
|
|
1147
|
-
const isArray = Array.isArray(container);
|
|
1148
|
-
if (!isArray && !isPlainObject2(container)) {
|
|
1149
|
-
reject(
|
|
1150
|
-
path,
|
|
1151
|
-
"holds a value that does not survive storage; use plain objects, arrays, strings, numbers, booleans, dates, or Uint8Array bytes"
|
|
1152
|
-
);
|
|
1153
|
-
}
|
|
1154
|
-
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) {
|
|
1155
|
-
reject(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
1156
|
-
}
|
|
1157
|
-
if (ancestors.has(container)) {
|
|
1158
|
-
reject(path, "refers back to one of its own ancestors");
|
|
1159
|
-
}
|
|
1160
|
-
ancestors.add(container);
|
|
1161
|
-
if (isArray) {
|
|
1162
|
-
const items = container;
|
|
1163
|
-
for (let i = 0; i < items.length; i++) {
|
|
1164
|
-
walkStorable(items[i], `${path}[${i}]`, depth + 1, ancestors, skip);
|
|
1165
|
-
}
|
|
1166
|
-
} else {
|
|
1167
|
-
const record = container;
|
|
1168
|
-
for (const key of Object.keys(record)) {
|
|
1169
|
-
const keyPath = path === "" ? key : `${path}.${key}`;
|
|
1170
|
-
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
1171
|
-
reject(keyPath, "uses a reserved object key");
|
|
1172
|
-
}
|
|
1173
|
-
if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
|
|
1174
|
-
if (skip.has(keyPath)) continue;
|
|
1175
|
-
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
1176
|
-
}
|
|
1177
|
-
}
|
|
1178
|
-
ancestors.delete(container);
|
|
1179
|
-
}
|
|
1180
|
-
function assertStorableDocument(document, skipFieldPaths) {
|
|
1181
|
-
walkStorable(document, "", 0, /* @__PURE__ */ new Set(), skipFieldPaths ?? /* @__PURE__ */ new Set());
|
|
1182
|
-
}
|
|
1183
|
-
|
|
1184
|
-
// src/schema/validator/document.ts
|
|
1185
|
-
var vectorPathCache = /* @__PURE__ */ new WeakMap();
|
|
1186
|
-
function vectorFieldPaths(schema) {
|
|
1187
|
-
let paths = vectorPathCache.get(schema);
|
|
1188
|
-
if (paths !== void 0) return paths;
|
|
1189
|
-
paths = /* @__PURE__ */ new Set();
|
|
1190
|
-
const collect = (level, prefix) => {
|
|
1191
|
-
for (const [field, type] of Object.entries(level)) {
|
|
1192
|
-
const path = prefix === "" ? field : `${prefix}.${field}`;
|
|
1193
|
-
if (typeof type === "string" && VECTOR_PATTERN.test(type)) {
|
|
1194
|
-
paths?.add(path);
|
|
1195
|
-
} else if (isPlainObject(type)) {
|
|
1196
|
-
collect(type, path);
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1199
|
-
};
|
|
1200
|
-
collect(schema, "");
|
|
1201
|
-
vectorPathCache.set(schema, paths);
|
|
1202
|
-
return paths;
|
|
1203
|
-
}
|
|
1204
|
-
var parsedLevelCache = /* @__PURE__ */ new WeakMap();
|
|
1205
|
-
function parsedLevel(schema, prefix) {
|
|
1206
|
-
let entries = parsedLevelCache.get(schema);
|
|
1207
|
-
if (entries !== void 0) return entries;
|
|
1208
|
-
entries = Object.entries(schema).map(([field, type]) => {
|
|
1209
|
-
if (isPlainObject(type)) return { field, nested: type, type: null };
|
|
1210
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
1211
|
-
return { field, nested: null, type: parseFieldType(type, path) };
|
|
1212
|
-
});
|
|
1213
|
-
parsedLevelCache.set(schema, entries);
|
|
1214
|
-
return entries;
|
|
1215
|
-
}
|
|
1216
|
-
function validateDocumentFields(doc, schema, prefix, patternValueLimit) {
|
|
1217
|
-
for (const { field, nested, type } of parsedLevel(schema, prefix)) {
|
|
1218
|
-
const value = doc[field];
|
|
1219
|
-
if (value === void 0 || value === null) continue;
|
|
1220
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
1221
|
-
if (nested !== null) {
|
|
1222
|
-
if (!isPlainObject(value)) {
|
|
1223
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
1224
|
-
field: path,
|
|
1225
|
-
received: Array.isArray(value) ? "array" : typeof value
|
|
1226
|
-
});
|
|
1227
|
-
}
|
|
1228
|
-
validateDocumentFields(value, nested, path, patternValueLimit);
|
|
1229
|
-
continue;
|
|
1230
|
-
}
|
|
1231
|
-
if (type !== null) validateFieldValue(path, value, type, patternValueLimit);
|
|
1232
|
-
}
|
|
1233
|
-
}
|
|
1234
|
-
function validateDocument(document, schema, patternValueLimit = DEFAULT_PATTERN_VALUE_LIMIT) {
|
|
1235
|
-
if (!isPlainObject(document)) {
|
|
1236
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
1237
|
-
received: typeof document
|
|
1238
|
-
});
|
|
1239
|
-
}
|
|
1240
|
-
requireIdFieldKey(document);
|
|
1241
|
-
assertStorableDocument(document, vectorFieldPaths(schema));
|
|
1242
|
-
validateDocumentFields(document, schema, "", patternValueLimit);
|
|
1243
|
-
}
|
|
1244
|
-
function requireIdFieldKey(document) {
|
|
1245
|
-
if (!Object.hasOwn(document, "id")) return;
|
|
1246
|
-
const id = document.id;
|
|
1247
|
-
if (typeof id === "string" && id.length > 0) return;
|
|
1248
|
-
throw new NarsilError(
|
|
1249
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1250
|
-
`Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
|
|
1251
|
-
{ field: "id", received: id === null ? "null" : typeof id }
|
|
1252
|
-
);
|
|
1253
|
-
}
|
|
1254
|
-
function collectExtraFields(doc, schema, prefix, extras) {
|
|
1255
|
-
for (const key of Object.keys(doc)) {
|
|
1256
|
-
if (key === "id") continue;
|
|
1257
|
-
const path = prefix ? `${prefix}.${key}` : key;
|
|
1258
|
-
if (!Object.hasOwn(schema, key)) {
|
|
1259
|
-
extras.push(path);
|
|
1260
|
-
continue;
|
|
1261
|
-
}
|
|
1262
|
-
const schemaType = schema[key];
|
|
1263
|
-
if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
|
|
1264
|
-
collectExtraFields(doc[key], schemaType, path, extras);
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
function validateDocumentStrict(document, schema) {
|
|
1269
|
-
const extras = [];
|
|
1270
|
-
collectExtraFields(document, schema, "", extras);
|
|
1271
|
-
if (extras.length > 0) {
|
|
1272
|
-
throw new NarsilError(
|
|
1273
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1274
|
-
`Document contains fields not defined in schema: ${extras.join(", ")}`,
|
|
1275
|
-
{ extraFields: extras }
|
|
1276
|
-
);
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
function resolveNestedValue(document, path) {
|
|
1280
|
-
if (!path.includes(".")) return document[path];
|
|
1281
|
-
const segments = path.split(".");
|
|
1282
|
-
let current = document;
|
|
1283
|
-
for (const segment of segments) {
|
|
1284
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
1285
|
-
current = current[segment];
|
|
1286
|
-
}
|
|
1287
|
-
return current;
|
|
1288
|
-
}
|
|
1289
|
-
function validateRequiredFields(document, required) {
|
|
1290
|
-
if (required.length === 0) return;
|
|
1291
|
-
for (const field of required) {
|
|
1292
|
-
const value = resolveNestedValue(document, field);
|
|
1293
|
-
if (value === void 0 || value === null) {
|
|
1294
|
-
throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
|
|
1295
|
-
field
|
|
1296
|
-
});
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
// src/schema/validator/document-size.ts
|
|
1302
|
-
var LARGEST_HEADER_BYTES = 5;
|
|
1303
|
-
var LARGEST_NUMBER_BYTES = 9;
|
|
1304
|
-
var LARGEST_TIMESTAMP_BYTES = 15;
|
|
1305
|
-
var LARGEST_UTF8_BYTES_PER_UNIT = 3;
|
|
1306
|
-
function upperBoundBytes(value, budget, depth) {
|
|
1307
|
-
if (value === null || value === void 0 || typeof value === "boolean") return 1;
|
|
1308
|
-
if (typeof value === "number") return LARGEST_NUMBER_BYTES;
|
|
1309
|
-
if (typeof value === "string") return LARGEST_HEADER_BYTES + value.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
1310
|
-
if (value instanceof Date) return LARGEST_TIMESTAMP_BYTES;
|
|
1311
|
-
if (ArrayBuffer.isView(value)) return LARGEST_HEADER_BYTES + value.byteLength;
|
|
1312
|
-
if (typeof value !== "object" || depth > MAX_DOCUMENT_NESTING_DEPTH) return LARGEST_NUMBER_BYTES;
|
|
1313
|
-
let total = LARGEST_HEADER_BYTES;
|
|
1314
|
-
if (Array.isArray(value)) {
|
|
1315
|
-
for (const item of value) {
|
|
1316
|
-
total += upperBoundBytes(item, budget - total, depth + 1);
|
|
1317
|
-
if (total > budget) return total;
|
|
1318
|
-
}
|
|
1319
|
-
return total;
|
|
1320
|
-
}
|
|
1321
|
-
const record = value;
|
|
1322
|
-
for (const key of Object.keys(record)) {
|
|
1323
|
-
total += LARGEST_HEADER_BYTES + key.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
1324
|
-
total += upperBoundBytes(record[key], budget - total, depth + 1);
|
|
1325
|
-
if (total > budget) return total;
|
|
1326
|
-
}
|
|
1327
|
-
return total;
|
|
1328
|
-
}
|
|
1329
|
-
function messagePackUpperBoundBytes(value) {
|
|
1330
|
-
return upperBoundBytes(value, Number.POSITIVE_INFINITY, 0);
|
|
1331
|
-
}
|
|
1332
|
-
function encodedBytesOf(document) {
|
|
1333
|
-
try {
|
|
1334
|
-
return encodeDocument(document).byteLength;
|
|
1335
|
-
} catch {
|
|
1336
|
-
return null;
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
function requireDocumentWithinSizeLimit(document) {
|
|
1340
|
-
if (upperBoundBytes(document, MAX_DOCUMENT_BYTES, 0) <= MAX_DOCUMENT_BYTES) return;
|
|
1341
|
-
const encodedBytes = encodedBytesOf(document);
|
|
1342
|
-
if (encodedBytes === null || encodedBytes <= MAX_DOCUMENT_BYTES) return;
|
|
1343
|
-
throw new NarsilError(
|
|
1344
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1345
|
-
`The document takes ${encodedBytes} bytes as MessagePack, vector values included, but a document may take at most ${MAX_DOCUMENT_BYTES}`,
|
|
1346
|
-
{ bytes: encodedBytes, maximum: MAX_DOCUMENT_BYTES }
|
|
1347
|
-
);
|
|
1348
|
-
}
|
|
1349
|
-
|
|
1350
|
-
// src/schema/validator/vector-promotion.ts
|
|
1351
|
-
var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
|
|
1352
|
-
var STORAGE_MODES = /* @__PURE__ */ new Set(["memory", "disk"]);
|
|
1353
|
-
function fail(message, details) {
|
|
1354
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
1355
|
-
}
|
|
1356
|
-
function isPositiveInteger(value) {
|
|
1357
|
-
return Number.isInteger(value) && value >= 1;
|
|
1358
|
-
}
|
|
1359
|
-
function isQuantizationMode(value) {
|
|
1360
|
-
return typeof value === "string" && QUANTIZATION_MODES.has(value);
|
|
1361
|
-
}
|
|
1362
|
-
function isStorageMode(value) {
|
|
1363
|
-
return typeof value === "string" && STORAGE_MODES.has(value);
|
|
1364
|
-
}
|
|
1365
|
-
function validateVectorStorage(config, filesystemDurability) {
|
|
1366
|
-
if (config?.storage === "disk" && !filesystemDurability) {
|
|
1367
|
-
fail("vectorPromotion.storage 'disk' needs filesystem durability, because a checkpoint file holds the vectors", {
|
|
1368
|
-
storage: config.storage
|
|
1369
|
-
});
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
function validateTextStorage(textStorage, filesystemDurability) {
|
|
1373
|
-
if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
|
|
1374
|
-
fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
|
|
1375
|
-
}
|
|
1376
|
-
function raiseEarlierReleaseGraphSetting(config) {
|
|
1377
|
-
if (typeof config !== "object" || config === null) return config;
|
|
1378
|
-
const hnsw = config.hnswConfig;
|
|
1379
|
-
if (typeof hnsw !== "object" || hnsw === null) return config;
|
|
1380
|
-
if (hnsw.m !== EARLIER_RELEASE_MIN_M) return config;
|
|
1381
|
-
return { ...config, hnswConfig: { ...hnsw, m: MIN_M } };
|
|
1382
|
-
}
|
|
1383
|
-
function validateVectorPromotion(config) {
|
|
1384
|
-
if (config === void 0) return;
|
|
1385
|
-
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
1386
|
-
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
1387
|
-
}
|
|
1388
|
-
const quantization = config.quantization;
|
|
1389
|
-
if (quantization !== void 0 && !isQuantizationMode(quantization)) {
|
|
1390
|
-
fail("vectorPromotion.quantization must be 'osq8', 'osq4', 'osq2', 'osq1', or 'none'", { quantization });
|
|
1391
|
-
}
|
|
1392
|
-
const storage = config.storage;
|
|
1393
|
-
if (storage !== void 0 && !isStorageMode(storage)) {
|
|
1394
|
-
fail("vectorPromotion.storage must be 'memory' or 'disk'", { storage });
|
|
1395
|
-
}
|
|
1396
|
-
const hnsw = config.hnswConfig;
|
|
1397
|
-
if (hnsw === void 0) return;
|
|
1398
|
-
if (hnsw.m !== void 0 && (!Number.isInteger(hnsw.m) || hnsw.m < MIN_M || hnsw.m > MAX_M)) {
|
|
1399
|
-
fail(`vectorPromotion.hnswConfig.m must be a whole number from ${MIN_M} to ${MAX_M}`, { m: hnsw.m });
|
|
1400
|
-
}
|
|
1401
|
-
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
1402
|
-
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
1403
|
-
efConstruction: hnsw.efConstruction
|
|
1404
|
-
});
|
|
1405
|
-
}
|
|
1406
|
-
const metric = hnsw.metric;
|
|
1407
|
-
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
1408
|
-
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
1409
|
-
}
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
938
|
// src/schema/embedding-validator.ts
|
|
1413
939
|
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
1414
940
|
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
@@ -1526,66 +1052,9 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
1526
1052
|
}
|
|
1527
1053
|
}
|
|
1528
1054
|
|
|
1529
|
-
// src/core/
|
|
1530
|
-
function
|
|
1531
|
-
|
|
1532
|
-
if (existing !== void 0) return existing;
|
|
1533
|
-
const idx = table.names.length;
|
|
1534
|
-
table.names.push(fieldName);
|
|
1535
|
-
table.indexMap.set(fieldName, idx);
|
|
1536
|
-
return idx;
|
|
1537
|
-
}
|
|
1538
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
1539
|
-
function getNestedValue(obj, path) {
|
|
1540
|
-
const parts = path.split(".");
|
|
1541
|
-
let current = obj;
|
|
1542
|
-
for (let i = 0; i < parts.length; i++) {
|
|
1543
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
1544
|
-
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
1545
|
-
current = current[parts[i]];
|
|
1546
|
-
}
|
|
1547
|
-
return current;
|
|
1548
|
-
}
|
|
1549
|
-
function stringArraysEqual(a, b) {
|
|
1550
|
-
if (a === b) return true;
|
|
1551
|
-
if (!a || !b) return false;
|
|
1552
|
-
if (a.length !== b.length) return false;
|
|
1553
|
-
for (let i = 0; i < a.length; i++) {
|
|
1554
|
-
if (a[i] !== b[i]) return false;
|
|
1555
|
-
}
|
|
1556
|
-
return true;
|
|
1557
|
-
}
|
|
1558
|
-
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
1559
|
-
for (const { path, base } of fields) {
|
|
1560
|
-
if (!isWordIndexedBase(base)) continue;
|
|
1561
|
-
const oldVal = getNestedValue(oldDoc, path);
|
|
1562
|
-
const newVal = getNestedValue(newDoc, path);
|
|
1563
|
-
if (base === "string") {
|
|
1564
|
-
if (oldVal !== newVal) return true;
|
|
1565
|
-
} else {
|
|
1566
|
-
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
1567
|
-
}
|
|
1568
|
-
}
|
|
1569
|
-
return false;
|
|
1570
|
-
}
|
|
1571
|
-
function parseVectorDimension(fieldType) {
|
|
1572
|
-
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
1573
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
1574
|
-
}
|
|
1575
|
-
function tokenizeOptions(options) {
|
|
1576
|
-
return {
|
|
1577
|
-
stem: true,
|
|
1578
|
-
removeStopWords: true,
|
|
1579
|
-
collectSurfaces: options?.collectSurfaces === true,
|
|
1580
|
-
stopWordOverride: options?.stopWordOverride,
|
|
1581
|
-
customTokenizer: options?.customTokenizer
|
|
1582
|
-
};
|
|
1583
|
-
}
|
|
1584
|
-
function getFlatSchema(state, schema) {
|
|
1585
|
-
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
1586
|
-
state.flatSchemaCache = flattenSchema(schema);
|
|
1587
|
-
state.lastSchemaRef = schema;
|
|
1588
|
-
return state.flatSchemaCache;
|
|
1055
|
+
// src/core/document-copy.browser.ts
|
|
1056
|
+
function copyStoredValue(value) {
|
|
1057
|
+
return structuredClone(value);
|
|
1589
1058
|
}
|
|
1590
1059
|
|
|
1591
1060
|
// src/core/projection.ts
|
|
@@ -1661,8 +1130,8 @@ function writePath(target, path, value) {
|
|
|
1661
1130
|
}
|
|
1662
1131
|
function cloneExcept(value, path, exclude) {
|
|
1663
1132
|
const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
|
|
1664
|
-
if (!reachesDeeper) return
|
|
1665
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return
|
|
1133
|
+
if (!reachesDeeper) return copyStoredValue(value);
|
|
1134
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return copyStoredValue(value);
|
|
1666
1135
|
const source = value;
|
|
1667
1136
|
const kept = {};
|
|
1668
1137
|
for (const key of Object.keys(source)) {
|
|
@@ -1701,7 +1170,7 @@ function projectionKeepsField(projection, fieldPath) {
|
|
|
1701
1170
|
return !projection.exclude.some((excluded) => covers(excluded, path));
|
|
1702
1171
|
}
|
|
1703
1172
|
function cloneProjected(document, projection) {
|
|
1704
|
-
if (projection === void 0 || projection.kind === "full") return
|
|
1173
|
+
if (projection === void 0 || projection.kind === "full") return copyStoredValue(document);
|
|
1705
1174
|
if (projection.kind === "none") return {};
|
|
1706
1175
|
if (projection.include === null) return cloneExcept(document, [], projection.exclude);
|
|
1707
1176
|
return projectIncludedPaths(projection.include, projection.exclude, (path) => readProjectedPath(document, path));
|
|
@@ -2525,7 +1994,7 @@ var LivePostings = class {
|
|
|
2525
1994
|
terms = new TermColumns(TERM_TABLE_START_CAPACITY);
|
|
2526
1995
|
pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
2527
1996
|
positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
2528
|
-
retired = new
|
|
1997
|
+
retired = new Uint32Array(RETIRED_DOCUMENTS_START_CAPACITY >>> 5);
|
|
2529
1998
|
batchMode = false;
|
|
2530
1999
|
dirty = /* @__PURE__ */ new Set();
|
|
2531
2000
|
cachedViews = /* @__PURE__ */ new Map();
|
|
@@ -2612,7 +2081,7 @@ var LivePostings = class {
|
|
|
2612
2081
|
this.repackIfSparse();
|
|
2613
2082
|
}
|
|
2614
2083
|
isRetired(internalId) {
|
|
2615
|
-
return
|
|
2084
|
+
return bitsetHas(this.retired, internalId);
|
|
2616
2085
|
}
|
|
2617
2086
|
beginBatch() {
|
|
2618
2087
|
this.batchMode = true;
|
|
@@ -2636,7 +2105,7 @@ var LivePostings = class {
|
|
|
2636
2105
|
this.pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
2637
2106
|
this.pool.epoch = epoch;
|
|
2638
2107
|
this.positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
2639
|
-
this.retired = new
|
|
2108
|
+
this.retired = new Uint32Array(RETIRED_DOCUMENTS_START_CAPACITY >>> 5);
|
|
2640
2109
|
this.dirty.clear();
|
|
2641
2110
|
this.cachedViews.clear();
|
|
2642
2111
|
this.compactedSlack = 0;
|
|
@@ -2645,12 +2114,13 @@ var LivePostings = class {
|
|
|
2645
2114
|
return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
|
|
2646
2115
|
}
|
|
2647
2116
|
retire(internalId) {
|
|
2648
|
-
|
|
2649
|
-
|
|
2117
|
+
const word = internalId >>> 5;
|
|
2118
|
+
if (word >= this.retired.length) {
|
|
2119
|
+
const grown = new Uint32Array(Math.max(word + 1, this.retired.length * 2));
|
|
2650
2120
|
grown.set(this.retired);
|
|
2651
2121
|
this.retired = grown;
|
|
2652
2122
|
}
|
|
2653
|
-
this.retired
|
|
2123
|
+
bitsetSet(this.retired, internalId);
|
|
2654
2124
|
}
|
|
2655
2125
|
purgeRetired() {
|
|
2656
2126
|
const terms = this.terms;
|
|
@@ -2662,7 +2132,7 @@ var LivePostings = class {
|
|
|
2662
2132
|
compact(ordinal) {
|
|
2663
2133
|
if (this.terms.deadDocuments[ordinal] === 0) return;
|
|
2664
2134
|
const retired = this.retired;
|
|
2665
|
-
this.filterEntries(ordinal, (id) =>
|
|
2135
|
+
this.filterEntries(ordinal, (id) => bitsetHas(retired, id));
|
|
2666
2136
|
this.terms.deadDocuments[ordinal] = 0;
|
|
2667
2137
|
}
|
|
2668
2138
|
filterEntries(ordinal, drop) {
|
|
@@ -2882,6 +2352,7 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
2882
2352
|
return entries;
|
|
2883
2353
|
}
|
|
2884
2354
|
return {
|
|
2355
|
+
livePostings: postings,
|
|
2885
2356
|
insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
|
|
2886
2357
|
postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
|
|
2887
2358
|
},
|
|
@@ -3365,442 +2836,76 @@ function computeFacets(state, matched, config, schema) {
|
|
|
3365
2836
|
const entries = ordered.slice(0, keep);
|
|
3366
2837
|
let errorBound = 0;
|
|
3367
2838
|
for (let index = keep; index < ordered.length; index++) {
|
|
3368
|
-
if (ordered[index][1] > errorBound) errorBound = ordered[index][1];
|
|
3369
|
-
}
|
|
3370
|
-
const values = {};
|
|
3371
|
-
for (const [key, count] of entries) {
|
|
3372
|
-
values[key] = count;
|
|
3373
|
-
}
|
|
3374
|
-
result[fieldPath] = { values, count: entries.length, errorBound };
|
|
3375
|
-
}
|
|
3376
|
-
return result;
|
|
3377
|
-
}
|
|
3378
|
-
|
|
3379
|
-
// src/core/pattern-index/host.ts
|
|
3380
|
-
var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
|
|
3381
|
-
function patternIndexOf(indexes, fieldPath) {
|
|
3382
|
-
let index = indexes.get(fieldPath);
|
|
3383
|
-
if (index === void 0) {
|
|
3384
|
-
index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
|
|
3385
|
-
indexes.set(fieldPath, index);
|
|
3386
|
-
}
|
|
3387
|
-
return index;
|
|
3388
|
-
}
|
|
3389
|
-
|
|
3390
|
-
// src/geo/cells.ts
|
|
3391
|
-
var AXIS_CELLS = 2 ** LOCATION_CODE_BITS_PER_AXIS;
|
|
3392
|
-
function axisCell(value, lowest, span) {
|
|
3393
|
-
const cell = Math.floor((value - lowest) / span * AXIS_CELLS);
|
|
3394
|
-
if (!(cell > 0)) return 0;
|
|
3395
|
-
return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
|
|
3396
|
-
}
|
|
3397
|
-
function latitudeCell(lat) {
|
|
3398
|
-
return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
|
|
3399
|
-
}
|
|
3400
|
-
function longitudeCell(lon) {
|
|
3401
|
-
return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
|
|
3402
|
-
}
|
|
3403
|
-
function interleave(row, column, bits) {
|
|
3404
|
-
let code = 0;
|
|
3405
|
-
for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
|
|
3406
|
-
return code;
|
|
3407
|
-
}
|
|
3408
|
-
function locationCode(lat, lon) {
|
|
3409
|
-
return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
|
|
3410
|
-
}
|
|
3411
|
-
function appendBoxRanges(box, out) {
|
|
3412
|
-
const south = latitudeCell(box.south);
|
|
3413
|
-
const north = latitudeCell(box.north);
|
|
3414
|
-
const west = longitudeCell(box.west);
|
|
3415
|
-
const east = longitudeCell(box.east);
|
|
3416
|
-
let level = LOCATION_CODE_BITS_PER_AXIS;
|
|
3417
|
-
while (level > 0) {
|
|
3418
|
-
const shift2 = LOCATION_CODE_BITS_PER_AXIS - level;
|
|
3419
|
-
const rows = (north >>> shift2) - (south >>> shift2) + 1;
|
|
3420
|
-
const columns = (east >>> shift2) - (west >>> shift2) + 1;
|
|
3421
|
-
if (rows * columns <= MAX_COVERING_CELLS) break;
|
|
3422
|
-
level--;
|
|
3423
|
-
}
|
|
3424
|
-
const shift = LOCATION_CODE_BITS_PER_AXIS - level;
|
|
3425
|
-
const codesPerCell = 4 ** shift;
|
|
3426
|
-
for (let row = south >>> shift; row <= north >>> shift; row++) {
|
|
3427
|
-
for (let column = west >>> shift; column <= east >>> shift; column++) {
|
|
3428
|
-
const from = interleave(row, column, level) * codesPerCell;
|
|
3429
|
-
out.push({ from, to: from + codesPerCell });
|
|
3430
|
-
}
|
|
3431
|
-
}
|
|
3432
|
-
}
|
|
3433
|
-
function coveringRanges(boxes) {
|
|
3434
|
-
const ranges = [];
|
|
3435
|
-
for (const box of boxes) appendBoxRanges(box, ranges);
|
|
3436
|
-
ranges.sort((left, right) => left.from - right.from);
|
|
3437
|
-
const merged = [];
|
|
3438
|
-
for (const range of ranges) {
|
|
3439
|
-
const last = merged[merged.length - 1];
|
|
3440
|
-
if (last !== void 0 && range.from <= last.to) {
|
|
3441
|
-
if (range.to > last.to) last.to = range.to;
|
|
3442
|
-
} else {
|
|
3443
|
-
merged.push({ from: range.from, to: range.to });
|
|
3444
|
-
}
|
|
3445
|
-
}
|
|
3446
|
-
return merged;
|
|
3447
|
-
}
|
|
3448
|
-
function longitudeBoxes(south, north, west, east) {
|
|
3449
|
-
if (east - west >= FULL_TURN_DEGREES) {
|
|
3450
|
-
return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
|
|
3451
|
-
}
|
|
3452
|
-
if (west < -MAX_LONGITUDE_DEGREES) {
|
|
3453
|
-
return [
|
|
3454
|
-
{ south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
|
|
3455
|
-
{ south, north, west: -MAX_LONGITUDE_DEGREES, east }
|
|
3456
|
-
];
|
|
3457
|
-
}
|
|
3458
|
-
if (east > MAX_LONGITUDE_DEGREES) {
|
|
3459
|
-
return [
|
|
3460
|
-
{ south, north, west, east: MAX_LONGITUDE_DEGREES },
|
|
3461
|
-
{ south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
|
|
3462
|
-
];
|
|
3463
|
-
}
|
|
3464
|
-
return [{ south, north, west, east }];
|
|
3465
|
-
}
|
|
3466
|
-
function radiusBoxes(lat, lon, distanceMeters) {
|
|
3467
|
-
const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
|
|
3468
|
-
const angleDegrees = angle / DEG_TO_RAD;
|
|
3469
|
-
const south = lat - angleDegrees;
|
|
3470
|
-
const north = lat + angleDegrees;
|
|
3471
|
-
if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
|
|
3472
|
-
return [
|
|
3473
|
-
{
|
|
3474
|
-
south: Math.max(south, -MAX_LATITUDE_DEGREES),
|
|
3475
|
-
north: Math.min(north, MAX_LATITUDE_DEGREES),
|
|
3476
|
-
west: -MAX_LONGITUDE_DEGREES,
|
|
3477
|
-
east: MAX_LONGITUDE_DEGREES
|
|
3478
|
-
}
|
|
3479
|
-
];
|
|
3480
|
-
}
|
|
3481
|
-
const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
|
|
3482
|
-
return longitudeBoxes(south, north, lon - spread, lon + spread);
|
|
3483
|
-
}
|
|
3484
|
-
|
|
3485
|
-
// src/geo/haversine.ts
|
|
3486
|
-
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
3487
|
-
const phi1 = lat1 * DEG_TO_RAD;
|
|
3488
|
-
const phi2 = lat2 * DEG_TO_RAD;
|
|
3489
|
-
const deltaPhi = (lat2 - lat1) * DEG_TO_RAD;
|
|
3490
|
-
const deltaLambda = (lon2 - lon1) * DEG_TO_RAD;
|
|
3491
|
-
const halfDeltaPhi = Math.sin(deltaPhi / 2);
|
|
3492
|
-
const halfDeltaLambda = Math.sin(deltaLambda / 2);
|
|
3493
|
-
const a = halfDeltaPhi * halfDeltaPhi + Math.cos(phi1) * Math.cos(phi2) * halfDeltaLambda * halfDeltaLambda;
|
|
3494
|
-
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
|
3495
|
-
return EARTH_RADIUS_METERS * c;
|
|
3496
|
-
}
|
|
3497
|
-
|
|
3498
|
-
// src/geo/edge-tree.ts
|
|
3499
|
-
function buildLongitudeEdgeTree(lons) {
|
|
3500
|
-
const n = lons.length;
|
|
3501
|
-
const edges = [];
|
|
3502
|
-
for (let i = 0; i < n; i++) {
|
|
3503
|
-
if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
|
|
3504
|
-
}
|
|
3505
|
-
const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
|
|
3506
|
-
edges.sort((left, right) => lowOf(left) - lowOf(right));
|
|
3507
|
-
const count = edges.length;
|
|
3508
|
-
const tree = {
|
|
3509
|
-
from: new Int32Array(count),
|
|
3510
|
-
to: new Int32Array(count),
|
|
3511
|
-
low: new Float64Array(count),
|
|
3512
|
-
high: new Float64Array(count),
|
|
3513
|
-
subtreeHigh: new Float64Array(count)
|
|
3514
|
-
};
|
|
3515
|
-
edges.forEach((i, at) => {
|
|
3516
|
-
const j = i === 0 ? n - 1 : i - 1;
|
|
3517
|
-
tree.from[at] = i;
|
|
3518
|
-
tree.to[at] = j;
|
|
3519
|
-
tree.low[at] = Math.min(lons[i], lons[j]);
|
|
3520
|
-
tree.high[at] = Math.max(lons[i], lons[j]);
|
|
3521
|
-
});
|
|
3522
|
-
const fillSubtreeHigh = (start, end) => {
|
|
3523
|
-
if (start >= end) return Number.NEGATIVE_INFINITY;
|
|
3524
|
-
const mid = start + end >>> 1;
|
|
3525
|
-
const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
|
|
3526
|
-
tree.subtreeHigh[mid] = highest;
|
|
3527
|
-
return highest;
|
|
3528
|
-
};
|
|
3529
|
-
fillSubtreeHigh(0, count);
|
|
3530
|
-
return tree;
|
|
3531
|
-
}
|
|
3532
|
-
function forEachEdgeSpanning(tree, lon, visit) {
|
|
3533
|
-
const walk = (start, end) => {
|
|
3534
|
-
if (start >= end) return;
|
|
3535
|
-
const mid = start + end >>> 1;
|
|
3536
|
-
if (tree.subtreeHigh[mid] <= lon) return;
|
|
3537
|
-
walk(start, mid);
|
|
3538
|
-
if (tree.low[mid] > lon) return;
|
|
3539
|
-
if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
|
|
3540
|
-
walk(mid + 1, end);
|
|
3541
|
-
};
|
|
3542
|
-
walk(0, tree.from.length);
|
|
3543
|
-
}
|
|
3544
|
-
|
|
3545
|
-
// src/geo/polygon.ts
|
|
3546
|
-
function requirePolygonRing(points) {
|
|
3547
|
-
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
3548
|
-
throw new NarsilError(
|
|
3549
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
3550
|
-
`A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
|
|
3551
|
-
{ points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
|
|
3552
|
-
);
|
|
3553
|
-
}
|
|
3554
|
-
points.forEach((point, position) => {
|
|
3555
|
-
const { lat, lon } = point ?? {};
|
|
3556
|
-
if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
|
|
3557
|
-
throw new NarsilError(
|
|
3558
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
3559
|
-
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
3560
|
-
{ position }
|
|
3561
|
-
);
|
|
3562
|
-
}
|
|
3563
|
-
if (isLatitude(lat) && isLongitude(lon)) return;
|
|
3564
|
-
throw new NarsilError(
|
|
3565
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
3566
|
-
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
3567
|
-
{ position, lat, lon }
|
|
3568
|
-
);
|
|
3569
|
-
});
|
|
3570
|
-
}
|
|
3571
|
-
function signedArea(polygon) {
|
|
3572
|
-
let area = 0;
|
|
3573
|
-
for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
|
|
3574
|
-
area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
|
|
3575
|
-
}
|
|
3576
|
-
return area;
|
|
3577
|
-
}
|
|
3578
|
-
function crossesAntimeridian(polygon) {
|
|
3579
|
-
let west = Number.POSITIVE_INFINITY;
|
|
3580
|
-
let east = Number.NEGATIVE_INFINITY;
|
|
3581
|
-
for (const point of polygon) {
|
|
3582
|
-
if (point.lon < west) west = point.lon;
|
|
3583
|
-
if (point.lon > east) east = point.lon;
|
|
3584
|
-
}
|
|
3585
|
-
return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
|
|
3586
|
-
}
|
|
3587
|
-
function eastward(lon) {
|
|
3588
|
-
return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
|
|
3589
|
-
}
|
|
3590
|
-
function polygonContainment(polygon) {
|
|
3591
|
-
const n = polygon.length;
|
|
3592
|
-
if (n < MIN_POLYGON_POINTS) return () => false;
|
|
3593
|
-
const unwrap = crossesAntimeridian(polygon);
|
|
3594
|
-
const lats = new Float64Array(n);
|
|
3595
|
-
const lons = new Float64Array(n);
|
|
3596
|
-
for (let i = 0; i < n; i++) {
|
|
3597
|
-
lats[i] = polygon[i].lat;
|
|
3598
|
-
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
3599
|
-
}
|
|
3600
|
-
const edges = buildLongitudeEdgeTree(lons);
|
|
3601
|
-
return (lat, rawLon) => {
|
|
3602
|
-
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
3603
|
-
let inside = false;
|
|
3604
|
-
forEachEdgeSpanning(edges, lon, (i, j) => {
|
|
3605
|
-
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
|
|
3606
|
-
if (lat < intersectLat) inside = !inside;
|
|
3607
|
-
});
|
|
3608
|
-
return inside;
|
|
3609
|
-
};
|
|
3610
|
-
}
|
|
3611
|
-
function polygonBoxes(polygon) {
|
|
3612
|
-
if (polygon.length < MIN_POLYGON_POINTS) return [];
|
|
3613
|
-
const unwrap = crossesAntimeridian(polygon);
|
|
3614
|
-
let south = Number.POSITIVE_INFINITY;
|
|
3615
|
-
let north = Number.NEGATIVE_INFINITY;
|
|
3616
|
-
let west = Number.POSITIVE_INFINITY;
|
|
3617
|
-
let east = Number.NEGATIVE_INFINITY;
|
|
3618
|
-
for (const point of polygon) {
|
|
3619
|
-
const lon = unwrap ? eastward(point.lon) : point.lon;
|
|
3620
|
-
south = Math.min(south, point.lat);
|
|
3621
|
-
north = Math.max(north, point.lat);
|
|
3622
|
-
west = Math.min(west, lon);
|
|
3623
|
-
east = Math.max(east, lon);
|
|
3624
|
-
}
|
|
3625
|
-
if (west >= MAX_LONGITUDE_DEGREES)
|
|
3626
|
-
return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
|
|
3627
|
-
return longitudeBoxes(south, north, west, east);
|
|
3628
|
-
}
|
|
3629
|
-
|
|
3630
|
-
// src/geo/vincenty.ts
|
|
3631
|
-
var WGS84_A = 6378137;
|
|
3632
|
-
var WGS84_F = 1 / 298.257223563;
|
|
3633
|
-
var WGS84_B = WGS84_A * (1 - WGS84_F);
|
|
3634
|
-
function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
3635
|
-
const phi1 = lat1 * DEG_TO_RAD;
|
|
3636
|
-
const phi2 = lat2 * DEG_TO_RAD;
|
|
3637
|
-
const L = (lon2 - lon1) * DEG_TO_RAD;
|
|
3638
|
-
const U1 = Math.atan((1 - WGS84_F) * Math.tan(phi1));
|
|
3639
|
-
const U2 = Math.atan((1 - WGS84_F) * Math.tan(phi2));
|
|
3640
|
-
const sinU1 = Math.sin(U1);
|
|
3641
|
-
const cosU1 = Math.cos(U1);
|
|
3642
|
-
const sinU2 = Math.sin(U2);
|
|
3643
|
-
const cosU2 = Math.cos(U2);
|
|
3644
|
-
let lambda = L;
|
|
3645
|
-
let prevLambda;
|
|
3646
|
-
let sinSigma = 0;
|
|
3647
|
-
let cosSigma = 0;
|
|
3648
|
-
let sigma = 0;
|
|
3649
|
-
let sinAlpha = 0;
|
|
3650
|
-
let cos2Alpha = 0;
|
|
3651
|
-
let cos2SigmaM = 0;
|
|
3652
|
-
let iterations = VINCENTY_MAX_ITERATIONS;
|
|
3653
|
-
do {
|
|
3654
|
-
const sinLambda = Math.sin(lambda);
|
|
3655
|
-
const cosLambda = Math.cos(lambda);
|
|
3656
|
-
const term1 = cosU2 * sinLambda;
|
|
3657
|
-
const term2 = cosU1 * sinU2 - sinU1 * cosU2 * cosLambda;
|
|
3658
|
-
sinSigma = Math.sqrt(term1 * term1 + term2 * term2);
|
|
3659
|
-
if (sinSigma === 0) return 0;
|
|
3660
|
-
cosSigma = sinU1 * sinU2 + cosU1 * cosU2 * cosLambda;
|
|
3661
|
-
sigma = Math.atan2(sinSigma, cosSigma);
|
|
3662
|
-
sinAlpha = cosU1 * cosU2 * sinLambda / sinSigma;
|
|
3663
|
-
cos2Alpha = 1 - sinAlpha * sinAlpha;
|
|
3664
|
-
cos2SigmaM = cos2Alpha === 0 ? 0 : cosSigma - 2 * sinU1 * sinU2 / cos2Alpha;
|
|
3665
|
-
const C = WGS84_F / 16 * cos2Alpha * (4 + WGS84_F * (4 - 3 * cos2Alpha));
|
|
3666
|
-
prevLambda = lambda;
|
|
3667
|
-
lambda = L + (1 - C) * WGS84_F * sinAlpha * (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM)));
|
|
3668
|
-
} while (Math.abs(lambda - prevLambda) > VINCENTY_CONVERGENCE_THRESHOLD && --iterations > 0);
|
|
3669
|
-
if (iterations === 0) {
|
|
3670
|
-
return haversineDistance(lat1, lon1, lat2, lon2);
|
|
3671
|
-
}
|
|
3672
|
-
const uSq = cos2Alpha * (WGS84_A * WGS84_A - WGS84_B * WGS84_B) / (WGS84_B * WGS84_B);
|
|
3673
|
-
const A = 1 + uSq / 16384 * (4096 + uSq * (-768 + uSq * (320 - 175 * uSq)));
|
|
3674
|
-
const B = uSq / 1024 * (256 + uSq * (-128 + uSq * (74 - 47 * uSq)));
|
|
3675
|
-
const deltaSigma = B * sinSigma * (cos2SigmaM + B / 4 * (cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM) - B / 6 * cos2SigmaM * (-3 + 4 * sinSigma * sinSigma) * (-3 + 4 * cos2SigmaM * cos2SigmaM)));
|
|
3676
|
-
return WGS84_B * A * (sigma - deltaSigma);
|
|
2839
|
+
if (ordered[index][1] > errorBound) errorBound = ordered[index][1];
|
|
2840
|
+
}
|
|
2841
|
+
const values = {};
|
|
2842
|
+
for (const [key, count] of entries) {
|
|
2843
|
+
values[key] = count;
|
|
2844
|
+
}
|
|
2845
|
+
result[fieldPath] = { values, count: entries.length, errorBound };
|
|
2846
|
+
}
|
|
2847
|
+
return result;
|
|
3677
2848
|
}
|
|
3678
2849
|
|
|
3679
|
-
// src/
|
|
3680
|
-
var
|
|
3681
|
-
function
|
|
3682
|
-
let
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
if (entries[mid].code < code) lo = mid + 1;
|
|
3687
|
-
else hi = mid;
|
|
2850
|
+
// src/core/pattern-index/host.ts
|
|
2851
|
+
var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
|
|
2852
|
+
function patternIndexOf(indexes, fieldPath) {
|
|
2853
|
+
let index = indexes.get(fieldPath);
|
|
2854
|
+
if (index === void 0) {
|
|
2855
|
+
index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
|
|
2856
|
+
indexes.set(fieldPath, index);
|
|
3688
2857
|
}
|
|
3689
|
-
return
|
|
2858
|
+
return index;
|
|
3690
2859
|
}
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
2860
|
+
|
|
2861
|
+
// src/schema/declared-types.ts
|
|
2862
|
+
function mayDeclare(schema, typeName, depth) {
|
|
2863
|
+
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
2864
|
+
for (const type of Object.values(schema)) {
|
|
2865
|
+
if (typeof type === "string" ? type.includes(typeName) : mayDeclare(type, typeName, depth + 1)) return true;
|
|
3695
2866
|
}
|
|
3696
|
-
|
|
2867
|
+
return false;
|
|
3697
2868
|
}
|
|
3698
|
-
function
|
|
3699
|
-
|
|
3700
|
-
const mainLength = entries.length;
|
|
3701
|
-
for (const entry of pending) entries.push(entry);
|
|
3702
|
-
let left = mainLength - 1;
|
|
3703
|
-
let right = pending.length - 1;
|
|
3704
|
-
let write = entries.length - 1;
|
|
3705
|
-
while (right >= 0) {
|
|
3706
|
-
const later = left >= 0 ? entries[left] : void 0;
|
|
3707
|
-
const next = pending[right];
|
|
3708
|
-
if (later !== void 0 && (later.code > next.code || later.code === next.code && later.docId > next.docId)) {
|
|
3709
|
-
entries[write--] = later;
|
|
3710
|
-
left--;
|
|
3711
|
-
} else {
|
|
3712
|
-
entries[write--] = next;
|
|
3713
|
-
right--;
|
|
3714
|
-
}
|
|
3715
|
-
}
|
|
2869
|
+
function schemaMayDeclare(schema, typeName) {
|
|
2870
|
+
return mayDeclare(schema, typeName, 1);
|
|
3716
2871
|
}
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
let
|
|
3722
|
-
function
|
|
3723
|
-
if (
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
2872
|
+
|
|
2873
|
+
// src/workers/code-loader.ts
|
|
2874
|
+
function codeLoader(importCode, typeName) {
|
|
2875
|
+
let loaded2 = null;
|
|
2876
|
+
let loading2 = null;
|
|
2877
|
+
function load() {
|
|
2878
|
+
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
2879
|
+
if (loading2 === null) {
|
|
2880
|
+
loading2 = importCode().then(
|
|
2881
|
+
(code) => {
|
|
2882
|
+
loaded2 = code;
|
|
2883
|
+
return code;
|
|
2884
|
+
},
|
|
2885
|
+
(err) => {
|
|
2886
|
+
loading2 = null;
|
|
2887
|
+
throw err;
|
|
2888
|
+
}
|
|
2889
|
+
);
|
|
3734
2890
|
}
|
|
3735
|
-
|
|
2891
|
+
return loading2;
|
|
3736
2892
|
}
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
if (removedCount >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) ensureSorted();
|
|
3743
|
-
}
|
|
3744
|
-
function matching(boxes, passes, inside) {
|
|
3745
|
-
ensureSorted();
|
|
3746
|
-
const within = /* @__PURE__ */ new Set();
|
|
3747
|
-
for (const range of coveringRanges(boxes)) {
|
|
3748
|
-
for (let at = firstAtOrAfter(entries, range.from); at < entries.length && entries[at].code < range.to; at++) {
|
|
3749
|
-
if (passes(entries[at])) within.add(entries[at].docId);
|
|
3750
|
-
}
|
|
3751
|
-
}
|
|
3752
|
-
if (inside) return within;
|
|
3753
|
-
const outside = /* @__PURE__ */ new Set();
|
|
3754
|
-
for (const entry of entries) {
|
|
3755
|
-
if (!within.has(entry.docId)) outside.add(entry.docId);
|
|
3756
|
-
}
|
|
3757
|
-
return outside;
|
|
3758
|
-
}
|
|
3759
|
-
const index = {
|
|
3760
|
-
insert(internalId, lat, lon) {
|
|
3761
|
-
const existing = byDocument.get(internalId);
|
|
3762
|
-
if (existing !== void 0) removeEntry(existing);
|
|
3763
|
-
const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
|
|
3764
|
-
pending.push(entry);
|
|
3765
|
-
byDocument.set(internalId, entry);
|
|
3766
|
-
},
|
|
3767
|
-
remove(internalId) {
|
|
3768
|
-
const existing = byDocument.get(internalId);
|
|
3769
|
-
if (existing !== void 0) removeEntry(existing);
|
|
3770
|
-
},
|
|
3771
|
-
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
3772
|
-
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
3773
|
-
return matching(
|
|
3774
|
-
radiusBoxes(lat, lon, distanceMeters),
|
|
3775
|
-
(entry) => distanceFn(lat, lon, entry.lat, entry.lon) <= distanceMeters,
|
|
3776
|
-
inside
|
|
3777
|
-
);
|
|
3778
|
-
},
|
|
3779
|
-
polygonQuery(points, inside) {
|
|
3780
|
-
const contains = polygonContainment(points);
|
|
3781
|
-
return matching(polygonBoxes(points), (entry) => contains(entry.lat, entry.lon), inside);
|
|
3782
|
-
},
|
|
3783
|
-
documentsBitset(capacity) {
|
|
3784
|
-
const bits = createBitSet(capacity);
|
|
3785
|
-
for (const docId of byDocument.keys()) bitsetSet(bits, docId);
|
|
3786
|
-
return bits;
|
|
3787
|
-
},
|
|
3788
|
-
clear() {
|
|
3789
|
-
entries = [];
|
|
3790
|
-
pending = [];
|
|
3791
|
-
byDocument.clear();
|
|
3792
|
-
removedCount = 0;
|
|
3793
|
-
},
|
|
3794
|
-
serialize() {
|
|
3795
|
-
ensureSorted();
|
|
3796
|
-
return entries.map((entry) => ({ lat: entry.lat, lon: entry.lon, docId: entry.docId }));
|
|
2893
|
+
return {
|
|
2894
|
+
loaded: () => loaded2,
|
|
2895
|
+
require() {
|
|
2896
|
+
if (loaded2 === null) throw new Error(`The engine reached ${typeName} code before it loaded that code`);
|
|
2897
|
+
return loaded2;
|
|
3797
2898
|
},
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
for (const entry of data) index.insert(entry.docId, entry.lat, entry.lon);
|
|
3801
|
-
}
|
|
2899
|
+
load,
|
|
2900
|
+
loadFor: (schema) => loaded2 !== null || !schemaMayDeclare(schema, typeName) ? null : load()
|
|
3802
2901
|
};
|
|
3803
|
-
|
|
2902
|
+
}
|
|
2903
|
+
|
|
2904
|
+
// src/geo/geo-code.ts
|
|
2905
|
+
var geoCode = codeLoader(() => import('./geo-index-OTV7VPIR.mjs'), "geopoint");
|
|
2906
|
+
var loadGeoCodeFor = geoCode.loadFor;
|
|
2907
|
+
function createLoadedGeoIndex() {
|
|
2908
|
+
return geoCode.require().createGeoIndex(PENDING_REMOVAL_MERGE_FLOOR, PENDING_REMOVAL_MERGE_DIVISOR);
|
|
3804
2909
|
}
|
|
3805
2910
|
|
|
3806
2911
|
// src/core/field-index/boolean.ts
|
|
@@ -3884,7 +2989,7 @@ function reversedUnits(value) {
|
|
|
3884
2989
|
for (let at = value.length - 1; at >= 0; at--) reversed += value[at];
|
|
3885
2990
|
return reversed;
|
|
3886
2991
|
}
|
|
3887
|
-
function
|
|
2992
|
+
function firstAtOrAfter(sorted, key) {
|
|
3888
2993
|
let lo = 0;
|
|
3889
2994
|
let hi = sorted.length;
|
|
3890
2995
|
while (lo < hi) {
|
|
@@ -3942,7 +3047,7 @@ var ValueAffixes = class {
|
|
|
3942
3047
|
}
|
|
3943
3048
|
matching(column, start) {
|
|
3944
3049
|
const found = [];
|
|
3945
|
-
for (let at =
|
|
3050
|
+
for (let at = firstAtOrAfter(column.keys, start); at < column.keys.length; at++) {
|
|
3946
3051
|
if (!column.keys[at].startsWith(start)) break;
|
|
3947
3052
|
found.push(column.values[at]);
|
|
3948
3053
|
}
|
|
@@ -4180,7 +3285,7 @@ function removeFromPending(pending, marked, byDocument) {
|
|
|
4180
3285
|
}
|
|
4181
3286
|
pending.length = write;
|
|
4182
3287
|
}
|
|
4183
|
-
function
|
|
3288
|
+
function mergeSortedPending(entries, pending) {
|
|
4184
3289
|
pending.sort((a, b) => a.value - b.value);
|
|
4185
3290
|
const mainLength = entries.length;
|
|
4186
3291
|
for (const entry of pending) entries.push(entry);
|
|
@@ -4216,7 +3321,7 @@ function createNumericIndex() {
|
|
|
4216
3321
|
pending.sort((a, b) => a.value - b.value);
|
|
4217
3322
|
entries = pending;
|
|
4218
3323
|
} else {
|
|
4219
|
-
|
|
3324
|
+
mergeSortedPending(entries, pending);
|
|
4220
3325
|
}
|
|
4221
3326
|
pending = [];
|
|
4222
3327
|
}
|
|
@@ -4404,7 +3509,7 @@ function ensureFieldIndex(state, fieldPath, base) {
|
|
|
4404
3509
|
} else if ((base === "enum" || base === "enum[]") && !state.enumIndexes.has(fieldPath)) {
|
|
4405
3510
|
state.enumIndexes.set(fieldPath, createEnumIndex());
|
|
4406
3511
|
} else if (base === "geopoint" && !state.geoIndexes.has(fieldPath)) {
|
|
4407
|
-
state.geoIndexes.set(fieldPath,
|
|
3512
|
+
state.geoIndexes.set(fieldPath, createLoadedGeoIndex());
|
|
4408
3513
|
}
|
|
4409
3514
|
}
|
|
4410
3515
|
function getOrCreateNumericIndex(state, fieldPath) {
|
|
@@ -5046,6 +4151,33 @@ function requireListLength(fieldPath, operator, values) {
|
|
|
5046
4151
|
);
|
|
5047
4152
|
}
|
|
5048
4153
|
|
|
4154
|
+
// src/geo/polygon-ring.ts
|
|
4155
|
+
function requirePolygonRing(points) {
|
|
4156
|
+
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
4157
|
+
throw new NarsilError(
|
|
4158
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4159
|
+
`A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
|
|
4160
|
+
{ points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
|
|
4161
|
+
);
|
|
4162
|
+
}
|
|
4163
|
+
points.forEach((point, position) => {
|
|
4164
|
+
const { lat, lon } = point ?? {};
|
|
4165
|
+
if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
|
|
4166
|
+
throw new NarsilError(
|
|
4167
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4168
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
4169
|
+
{ position }
|
|
4170
|
+
);
|
|
4171
|
+
}
|
|
4172
|
+
if (isLatitude(lat) && isLongitude(lon)) return;
|
|
4173
|
+
throw new NarsilError(
|
|
4174
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4175
|
+
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
4176
|
+
{ position, lat, lon }
|
|
4177
|
+
);
|
|
4178
|
+
});
|
|
4179
|
+
}
|
|
4180
|
+
|
|
5049
4181
|
// src/filters/operands/geo.ts
|
|
5050
4182
|
function requireFlag(fieldPath, operator, flag, value) {
|
|
5051
4183
|
if (value === void 0 || typeof value === "boolean") return;
|
|
@@ -6168,17 +5300,21 @@ function presentBitsetOf(state, fieldPath, type, capacity) {
|
|
|
6168
5300
|
if (base === "number") present = state.numericIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6169
5301
|
else if (base === "boolean") present = state.booleanIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6170
5302
|
else if (base === "enum") present = state.enumIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6171
|
-
else if (base === "geopoint") present = state
|
|
5303
|
+
else if (base === "geopoint") present = geoDocumentsBitset(state, fieldPath, capacity);
|
|
6172
5304
|
else if (base !== void 0 && keepsValueSize(base)) {
|
|
6173
5305
|
present = state.sizeIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6174
5306
|
}
|
|
6175
5307
|
return present ?? createBitSet(capacity);
|
|
6176
5308
|
}
|
|
6177
|
-
function
|
|
5309
|
+
function bitsetOfDocuments(internalIds, capacity) {
|
|
6178
5310
|
const bits = createBitSet(capacity);
|
|
6179
|
-
for (const internalId of
|
|
5311
|
+
for (const internalId of internalIds) bitsetSet(bits, internalId);
|
|
6180
5312
|
return bits;
|
|
6181
5313
|
}
|
|
5314
|
+
function geoDocumentsBitset(state, fieldPath, capacity) {
|
|
5315
|
+
const geoIndex = state.geoIndexes.get(fieldPath);
|
|
5316
|
+
return geoIndex === void 0 ? void 0 : bitsetOfDocuments(geoIndex.documentIds(), capacity);
|
|
5317
|
+
}
|
|
6182
5318
|
function buildFilterContext(state, schema, meter) {
|
|
6183
5319
|
const flat = getFlatSchema(state, schema);
|
|
6184
5320
|
const capacity = state.docStore.internalIdCapacity();
|
|
@@ -6230,7 +5366,7 @@ function buildFilterContext(state, schema, meter) {
|
|
|
6230
5366
|
},
|
|
6231
5367
|
capacity,
|
|
6232
5368
|
get allDocIdsBitset() {
|
|
6233
|
-
cachedAllDocIdsBitset ??=
|
|
5369
|
+
cachedAllDocIdsBitset ??= bitsetOfDocuments(state.docStore.allInternalIds(), capacity);
|
|
6234
5370
|
return cachedAllDocIdsBitset;
|
|
6235
5371
|
}
|
|
6236
5372
|
};
|
|
@@ -6304,7 +5440,7 @@ function requireLoadedPhraseCode() {
|
|
|
6304
5440
|
}
|
|
6305
5441
|
function loadPhraseCode() {
|
|
6306
5442
|
if (loading === null) {
|
|
6307
|
-
loading = import('./phrase-module-
|
|
5443
|
+
loading = import('./phrase-module-HAT3TLZN.mjs').then(
|
|
6308
5444
|
(code) => {
|
|
6309
5445
|
loaded = code;
|
|
6310
5446
|
return code;
|
|
@@ -6338,10 +5474,82 @@ function analyseQueryPhrases(term, language, analysis) {
|
|
|
6338
5474
|
);
|
|
6339
5475
|
}
|
|
6340
5476
|
|
|
5477
|
+
// src/math/log.ts
|
|
5478
|
+
var EXPONENT_BIAS = 1023;
|
|
5479
|
+
var EXPONENT_SHIFT = 20;
|
|
5480
|
+
var SUBNORMAL_SCALE_EXPONENT = 54;
|
|
5481
|
+
var ROUNDING_OFFSET = 614244;
|
|
5482
|
+
var HALVING_BIT = 1048576;
|
|
5483
|
+
var ONE_HIGH_WORD = 1072693248;
|
|
5484
|
+
var SIGNIFICAND_MASK = 1048575;
|
|
5485
|
+
var NEAR_ONE_LIMIT = 3;
|
|
5486
|
+
var LOWER_SPLIT = 398458;
|
|
5487
|
+
var UPPER_SPLIT = 440401;
|
|
5488
|
+
var LN2_HI = 0.6931471803691238;
|
|
5489
|
+
var LN2_LO = 19082149292705877e-26;
|
|
5490
|
+
var TWO54 = 18014398509481984;
|
|
5491
|
+
var TWO = 2;
|
|
5492
|
+
var THIRD = 0.3333333333333333;
|
|
5493
|
+
var HALF = 0.5;
|
|
5494
|
+
var LG1 = 0.6666666666666735;
|
|
5495
|
+
var LG2 = 0.3999999999940942;
|
|
5496
|
+
var LG3 = 0.2857142874366239;
|
|
5497
|
+
var LG4 = 0.22222198432149784;
|
|
5498
|
+
var LG5 = 0.1818357216161805;
|
|
5499
|
+
var LG6 = 0.15313837699209373;
|
|
5500
|
+
var LG7 = 0.14798198605116586;
|
|
5501
|
+
function nearOne(fraction, scale) {
|
|
5502
|
+
if (fraction === 0) {
|
|
5503
|
+
if (scale === 0) return 0;
|
|
5504
|
+
return scale * LN2_HI + scale * LN2_LO;
|
|
5505
|
+
}
|
|
5506
|
+
const cubic = fraction * fraction * (HALF - THIRD * fraction);
|
|
5507
|
+
if (scale === 0) return fraction - cubic;
|
|
5508
|
+
return scale * LN2_HI - (cubic - scale * LN2_LO - fraction);
|
|
5509
|
+
}
|
|
5510
|
+
function series(fraction, scale, significand) {
|
|
5511
|
+
const ratio = fraction / (TWO + fraction);
|
|
5512
|
+
const square = ratio * ratio;
|
|
5513
|
+
const below = significand - LOWER_SPLIT;
|
|
5514
|
+
const fourth = square * square;
|
|
5515
|
+
const above = UPPER_SPLIT - significand;
|
|
5516
|
+
const even = fourth * (LG2 + fourth * (LG4 + fourth * LG6));
|
|
5517
|
+
const odd = square * (LG1 + fourth * (LG3 + fourth * (LG5 + fourth * LG7)));
|
|
5518
|
+
const poly = odd + even;
|
|
5519
|
+
if ((below | above) > 0) {
|
|
5520
|
+
const halfSquare = HALF * fraction * fraction;
|
|
5521
|
+
if (scale === 0) return fraction - (halfSquare - ratio * (halfSquare + poly));
|
|
5522
|
+
return scale * LN2_HI - (halfSquare - (ratio * (halfSquare + poly) + scale * LN2_LO) - fraction);
|
|
5523
|
+
}
|
|
5524
|
+
if (scale === 0) return fraction - ratio * (fraction - poly);
|
|
5525
|
+
return scale * LN2_HI - (ratio * (fraction - poly) - scale * LN2_LO - fraction);
|
|
5526
|
+
}
|
|
5527
|
+
function log(value) {
|
|
5528
|
+
let high = highWord(value);
|
|
5529
|
+
let scale = 0;
|
|
5530
|
+
let scaled = value;
|
|
5531
|
+
if (high < SMALLEST_NORMAL_HIGH_WORD) {
|
|
5532
|
+
if ((high & MAGNITUDE_MASK | lowWord(value)) === 0) return Number.NEGATIVE_INFINITY;
|
|
5533
|
+
if (high < 0) return Number.NaN;
|
|
5534
|
+
scale -= SUBNORMAL_SCALE_EXPONENT;
|
|
5535
|
+
scaled *= TWO54;
|
|
5536
|
+
high = highWord(scaled);
|
|
5537
|
+
}
|
|
5538
|
+
if (high >= INFINITE_HIGH_WORD) return scaled + scaled;
|
|
5539
|
+
scale += (high >> EXPONENT_SHIFT) - EXPONENT_BIAS;
|
|
5540
|
+
const significand = high & SIGNIFICAND_MASK;
|
|
5541
|
+
const halving = significand + ROUNDING_OFFSET & HALVING_BIT;
|
|
5542
|
+
scaled = withHighWord(scaled, significand | halving ^ ONE_HIGH_WORD);
|
|
5543
|
+
scale += halving >> EXPONENT_SHIFT;
|
|
5544
|
+
const fraction = scaled - 1;
|
|
5545
|
+
if ((SIGNIFICAND_MASK & 2 + significand) < NEAR_ONE_LIMIT) return nearOne(fraction, scale);
|
|
5546
|
+
return series(fraction, scale, significand);
|
|
5547
|
+
}
|
|
5548
|
+
|
|
6341
5549
|
// src/core/scorer.ts
|
|
6342
5550
|
function computeIDF(docFrequency, totalDocs) {
|
|
6343
5551
|
if (totalDocs === 0) return 0;
|
|
6344
|
-
return
|
|
5552
|
+
return log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
6345
5553
|
}
|
|
6346
5554
|
function pairKeepsBoundSound(k1, b) {
|
|
6347
5555
|
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
@@ -6405,6 +5613,43 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
6405
5613
|
return idf * (numerator / denominator);
|
|
6406
5614
|
}
|
|
6407
5615
|
|
|
5616
|
+
// src/core/partition/field-order.ts
|
|
5617
|
+
var fieldRun = { entries: new Uint32Array(FIELD_RUN_ENTRIES_START), length: 0 };
|
|
5618
|
+
var ranksByFieldNames = /* @__PURE__ */ new WeakMap();
|
|
5619
|
+
var ONLY_FIELD_RANKS = new Uint32Array(1);
|
|
5620
|
+
function fieldNameRanks(fieldNames) {
|
|
5621
|
+
if (fieldNames.length < 2) return ONLY_FIELD_RANKS;
|
|
5622
|
+
const held = ranksByFieldNames.get(fieldNames);
|
|
5623
|
+
if (held !== void 0 && held.length === fieldNames.length) return held;
|
|
5624
|
+
const order = Array.from(fieldNames.keys()).sort(
|
|
5625
|
+
(left, right) => compareCodePoints(fieldNames[left], fieldNames[right])
|
|
5626
|
+
);
|
|
5627
|
+
const ranks = new Uint32Array(order.length);
|
|
5628
|
+
for (let rank = 0; rank < order.length; rank++) ranks[order[rank]] = rank;
|
|
5629
|
+
ranksByFieldNames.set(fieldNames, ranks);
|
|
5630
|
+
return ranks;
|
|
5631
|
+
}
|
|
5632
|
+
function orderRunByFieldName(docIds, fieldIndices, ranks, start, count) {
|
|
5633
|
+
let entries = fieldRun.entries;
|
|
5634
|
+
let end = start;
|
|
5635
|
+
for (; end < count && docIds[end] === docIds[start]; end++) {
|
|
5636
|
+
let slot = end - start;
|
|
5637
|
+
if (slot === entries.length) {
|
|
5638
|
+
const grown = new Uint32Array(slot * 2);
|
|
5639
|
+
grown.set(entries);
|
|
5640
|
+
entries = fieldRun.entries = grown;
|
|
5641
|
+
}
|
|
5642
|
+
const rank = ranks[fieldIndices[end]];
|
|
5643
|
+
while (slot > 0 && ranks[fieldIndices[entries[slot - 1]]] > rank) {
|
|
5644
|
+
entries[slot] = entries[slot - 1];
|
|
5645
|
+
slot--;
|
|
5646
|
+
}
|
|
5647
|
+
entries[slot] = end;
|
|
5648
|
+
}
|
|
5649
|
+
fieldRun.length = end - start;
|
|
5650
|
+
return entries;
|
|
5651
|
+
}
|
|
5652
|
+
|
|
6408
5653
|
// src/core/partition/field-scoring.ts
|
|
6409
5654
|
var NO_FIELD_VALUES = new Float64Array(0);
|
|
6410
5655
|
function differentPairsOf(searchable, k1s, bs) {
|
|
@@ -6432,7 +5677,8 @@ function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
|
6432
5677
|
lengthColumns: new Array(fieldCount).fill(null),
|
|
6433
5678
|
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
6434
5679
|
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
6435
|
-
boundPairs: [k1, b]
|
|
5680
|
+
boundPairs: [k1, b],
|
|
5681
|
+
nameRanks: fieldNameRanks(fieldNames)
|
|
6436
5682
|
};
|
|
6437
5683
|
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
6438
5684
|
const fieldName = fieldNames[fieldIndex];
|
|
@@ -6535,12 +5781,18 @@ function termGroupOf(groups, token) {
|
|
|
6535
5781
|
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
6536
5782
|
if (group !== void 0) {
|
|
6537
5783
|
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
6538
|
-
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true);
|
|
5784
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true).sort(byToken);
|
|
5785
|
+
}
|
|
5786
|
+
if (!exact) {
|
|
5787
|
+
const matches = index.fuzzyLookup(token, tolerance, prefixLength);
|
|
5788
|
+
return matches.length > 1 ? matches.sort(byToken) : matches;
|
|
6539
5789
|
}
|
|
6540
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
6541
5790
|
const postingList = index.lookup(token);
|
|
6542
5791
|
return postingList ? [{ token, postingList }] : [];
|
|
6543
5792
|
}
|
|
5793
|
+
function byToken(left, right) {
|
|
5794
|
+
return compareCodePoints(left.token, right.token);
|
|
5795
|
+
}
|
|
6544
5796
|
function resolveGroupMatches(context, group) {
|
|
6545
5797
|
const frequencyOf = (token) => context.globalFrequency(token) ?? context.index.documentFrequency(token);
|
|
6546
5798
|
const lookup = (token) => context.index.lookup(token);
|
|
@@ -6573,10 +5825,20 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
6573
5825
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
6574
5826
|
match.postingList
|
|
6575
5827
|
);
|
|
6576
|
-
|
|
6577
|
-
|
|
5828
|
+
let runStart = 0;
|
|
5829
|
+
let runEnd = 0;
|
|
5830
|
+
for (let at = 0; at < count; at++) {
|
|
5831
|
+
const internalId = docIds[at];
|
|
6578
5832
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
6579
5833
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
5834
|
+
let pi = at;
|
|
5835
|
+
if (at < runEnd) {
|
|
5836
|
+
pi = fieldRun.entries[at - runStart];
|
|
5837
|
+
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
5838
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
5839
|
+
runStart = at;
|
|
5840
|
+
runEnd = at + fieldRun.length;
|
|
5841
|
+
}
|
|
6580
5842
|
const fieldIndex = fieldNameIndices[pi];
|
|
6581
5843
|
if (fields.searchable[fieldIndex] === 0) continue;
|
|
6582
5844
|
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
@@ -6854,7 +6116,7 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
6854
6116
|
for (const [fieldPath, index] of source.geoIndexes) {
|
|
6855
6117
|
let targetIndex = target.geoIndexes.get(fieldPath);
|
|
6856
6118
|
if (targetIndex === void 0) {
|
|
6857
|
-
targetIndex =
|
|
6119
|
+
targetIndex = createLoadedGeoIndex();
|
|
6858
6120
|
target.geoIndexes.set(fieldPath, targetIndex);
|
|
6859
6121
|
}
|
|
6860
6122
|
for (const entry of index.serialize()) {
|
|
@@ -7281,15 +6543,30 @@ function firstEntryOf(docIds, count, internalId) {
|
|
|
7281
6543
|
}
|
|
7282
6544
|
return low;
|
|
7283
6545
|
}
|
|
6546
|
+
function fieldOrderedRunScore(scan, start, internalId, total, entryScore) {
|
|
6547
|
+
const { docIds, fieldNameIndices, count } = scan.columns;
|
|
6548
|
+
const ordered = orderRunByFieldName(docIds, fieldNameIndices, scan.fields.nameRanks, start, count);
|
|
6549
|
+
let score = total;
|
|
6550
|
+
for (let at = 0; at < fieldRun.length; at++) {
|
|
6551
|
+
const row = ordered[at];
|
|
6552
|
+
if (scan.fields.searchable[fieldNameIndices[row]] !== 1) continue;
|
|
6553
|
+
score += entryScore(scan, row, internalId);
|
|
6554
|
+
}
|
|
6555
|
+
return score;
|
|
6556
|
+
}
|
|
7284
6557
|
function samePairQueryOrderScore(scans, internalId) {
|
|
7285
6558
|
let score = 0;
|
|
7286
6559
|
for (const scan of scans) {
|
|
7287
6560
|
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
7288
6561
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
score
|
|
6562
|
+
const start = firstEntryOf(docIds, count, internalId);
|
|
6563
|
+
if (start + 1 < count && docIds[start + 1] === internalId) {
|
|
6564
|
+
score = fieldOrderedRunScore(scan, start, internalId, score, samePairEntryScore);
|
|
6565
|
+
continue;
|
|
7292
6566
|
}
|
|
6567
|
+
if (start === count || docIds[start] !== internalId) continue;
|
|
6568
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[start]] !== 1) continue;
|
|
6569
|
+
score += samePairEntryScore(scan, start, internalId);
|
|
7293
6570
|
}
|
|
7294
6571
|
return score;
|
|
7295
6572
|
}
|
|
@@ -7298,10 +6575,14 @@ function fieldPairQueryOrderScore(scans, internalId) {
|
|
|
7298
6575
|
for (const scan of scans) {
|
|
7299
6576
|
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
7300
6577
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7301
|
-
|
|
7302
|
-
|
|
7303
|
-
score
|
|
6578
|
+
const start = firstEntryOf(docIds, count, internalId);
|
|
6579
|
+
if (start + 1 < count && docIds[start + 1] === internalId) {
|
|
6580
|
+
score = fieldOrderedRunScore(scan, start, internalId, score, fieldPairEntryScore);
|
|
6581
|
+
continue;
|
|
7304
6582
|
}
|
|
6583
|
+
if (start === count || docIds[start] !== internalId) continue;
|
|
6584
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[start]] !== 1) continue;
|
|
6585
|
+
score += fieldPairEntryScore(scan, start, internalId);
|
|
7305
6586
|
}
|
|
7306
6587
|
return score;
|
|
7307
6588
|
}
|
|
@@ -7439,7 +6720,8 @@ function singleTermTopK(request) {
|
|
|
7439
6720
|
lengthColumns: fieldLengthColumns,
|
|
7440
6721
|
k1s,
|
|
7441
6722
|
bs,
|
|
7442
|
-
boundPairs
|
|
6723
|
+
boundPairs,
|
|
6724
|
+
nameRanks
|
|
7443
6725
|
} = request.fields;
|
|
7444
6726
|
const onePair = boundPairs.length === 2;
|
|
7445
6727
|
const onlyK1 = boundPairs[0];
|
|
@@ -7479,12 +6761,14 @@ function singleTermTopK(request) {
|
|
|
7479
6761
|
}
|
|
7480
6762
|
let score = 0;
|
|
7481
6763
|
let scored2 = false;
|
|
7482
|
-
|
|
7483
|
-
|
|
6764
|
+
const ordered = entry + 2 < end && docIds[entry + 2] === internalId ? orderRunByFieldName(docIds, fieldNameIndices, nameRanks, entry, end) : null;
|
|
6765
|
+
for (let at = 0; entry < end && docIds[entry] === internalId; at++, entry++) {
|
|
6766
|
+
const row = ordered === null ? entry : ordered[at];
|
|
6767
|
+
const fieldIndex = fieldNameIndices[row];
|
|
7484
6768
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
7485
6769
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
7486
6770
|
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7487
|
-
termFrequencies[
|
|
6771
|
+
termFrequencies[row],
|
|
7488
6772
|
idf,
|
|
7489
6773
|
fieldLength,
|
|
7490
6774
|
fieldAvgLengths[fieldIndex],
|
|
@@ -7493,7 +6777,6 @@ function singleTermTopK(request) {
|
|
|
7493
6777
|
)) * fieldBoosts[fieldIndex];
|
|
7494
6778
|
scored2 = true;
|
|
7495
6779
|
}
|
|
7496
|
-
entry++;
|
|
7497
6780
|
}
|
|
7498
6781
|
if (!scored2) continue;
|
|
7499
6782
|
totalMatched++;
|
|
@@ -7531,8 +6814,6 @@ function singleTermTopK(request) {
|
|
|
7531
6814
|
}
|
|
7532
6815
|
return { scored, totalMatched };
|
|
7533
6816
|
}
|
|
7534
|
-
|
|
7535
|
-
// src/core/partition/search.ts
|
|
7536
6817
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
7537
6818
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
7538
6819
|
}
|
|
@@ -7559,15 +6840,11 @@ function searchFulltext(state, params) {
|
|
|
7559
6840
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
7560
6841
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
7561
6842
|
const scoresAreZero = totalDocs === 0;
|
|
7562
|
-
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
7563
|
-
const scoreBuffer = state.scoreBuffer;
|
|
7564
|
-
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
7565
|
-
const components = collectComponents ? /* @__PURE__ */ new Map() : null;
|
|
7566
6843
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
7567
6844
|
const fieldNames = state.fieldNameTable.names;
|
|
7568
6845
|
const resolver = state.docStore.resolver();
|
|
7569
6846
|
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
7570
|
-
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
6847
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs, nameRanks } = fieldScoring;
|
|
7571
6848
|
const onePair = boundPairs.length === 2;
|
|
7572
6849
|
const onlyK1 = boundPairs[0];
|
|
7573
6850
|
const onlyB = boundPairs[1];
|
|
@@ -7584,6 +6861,10 @@ function searchFulltext(state, params) {
|
|
|
7584
6861
|
return { scored: [], totalMatched: 0 };
|
|
7585
6862
|
}
|
|
7586
6863
|
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
6864
|
+
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
6865
|
+
const scoreBuffer = state.scoreBuffer;
|
|
6866
|
+
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
6867
|
+
const components = collectComponents ? /* @__PURE__ */ new Map() : null;
|
|
7587
6868
|
function documentFrequency(token, list) {
|
|
7588
6869
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
7589
6870
|
}
|
|
@@ -7592,25 +6873,35 @@ function searchFulltext(state, params) {
|
|
|
7592
6873
|
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
7593
6874
|
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
7594
6875
|
}
|
|
7595
|
-
for (let
|
|
7596
|
-
const
|
|
7597
|
-
|
|
7598
|
-
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7603
|
-
frequency,
|
|
7604
|
-
idf,
|
|
7605
|
-
fieldLength,
|
|
7606
|
-
avgLen,
|
|
7607
|
-
onePair ? onlyK1 : k1s[fieldIndex],
|
|
7608
|
-
onePair ? onlyB : bs[fieldIndex]
|
|
6876
|
+
for (let start = 0; start < rows.docIds.length; start += fieldRun.length) {
|
|
6877
|
+
const entries = orderRunByFieldName(
|
|
6878
|
+
rows.docIds.values,
|
|
6879
|
+
rows.fieldIndices.values,
|
|
6880
|
+
nameRanks,
|
|
6881
|
+
start,
|
|
6882
|
+
rows.docIds.length
|
|
7609
6883
|
);
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
6884
|
+
for (let at = 0; at < fieldRun.length; at++) scorePhraseRow(rows, matched, entries[at], idf);
|
|
6885
|
+
}
|
|
6886
|
+
}
|
|
6887
|
+
function scorePhraseRow(rows, matched, row, idf) {
|
|
6888
|
+
const internalId = rows.docIds.values[row];
|
|
6889
|
+
if (!bitsetHas(matched, internalId)) return;
|
|
6890
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
6891
|
+
const frequency = rows.frequencies.values[row];
|
|
6892
|
+
const avgLen = averageLengths[fieldIndex];
|
|
6893
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
6894
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6895
|
+
frequency,
|
|
6896
|
+
idf,
|
|
6897
|
+
fieldLength,
|
|
6898
|
+
avgLen,
|
|
6899
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6900
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6901
|
+
);
|
|
6902
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
6903
|
+
if (components !== null) {
|
|
6904
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
7614
6905
|
}
|
|
7615
6906
|
}
|
|
7616
6907
|
const groupContext = {
|
|
@@ -7633,13 +6924,23 @@ function searchFulltext(state, params) {
|
|
|
7633
6924
|
const docFreq = documentFrequency(token, postingList);
|
|
7634
6925
|
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
7635
6926
|
}
|
|
7636
|
-
function scorePostings(match,
|
|
6927
|
+
function scorePostings(match, only) {
|
|
7637
6928
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7638
6929
|
match.postingList
|
|
7639
6930
|
);
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
6931
|
+
let runStart = 0;
|
|
6932
|
+
let runEnd = 0;
|
|
6933
|
+
if (only === null && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
6934
|
+
for (let at = 0; at < count; at++) {
|
|
6935
|
+
const internalId = docIds[at];
|
|
6936
|
+
let pi = at;
|
|
6937
|
+
if (at < runEnd) {
|
|
6938
|
+
pi = fieldRun.entries[at - runStart];
|
|
6939
|
+
} else if (at + 1 < count && docIds[at + 1] === internalId) {
|
|
6940
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, nameRanks, at, count)[0];
|
|
6941
|
+
runStart = at;
|
|
6942
|
+
runEnd = at + fieldRun.length;
|
|
6943
|
+
}
|
|
7643
6944
|
const fieldIndex = fieldNameIndices[pi];
|
|
7644
6945
|
if (searchable[fieldIndex] === 0) continue;
|
|
7645
6946
|
const avgLen = averageLengths[fieldIndex];
|
|
@@ -7656,11 +6957,19 @@ function searchFulltext(state, params) {
|
|
|
7656
6957
|
}
|
|
7657
6958
|
return;
|
|
7658
6959
|
}
|
|
7659
|
-
for (let
|
|
7660
|
-
const internalId = docIds[
|
|
6960
|
+
for (let at = 0; at < count; at++) {
|
|
6961
|
+
const internalId = docIds[at];
|
|
7661
6962
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7662
6963
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7663
|
-
if (
|
|
6964
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
6965
|
+
let pi = at;
|
|
6966
|
+
if (at < runEnd) {
|
|
6967
|
+
pi = fieldRun.entries[at - runStart];
|
|
6968
|
+
} else if (at + 1 < count && docIds[at + 1] === internalId) {
|
|
6969
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, nameRanks, at, count)[0];
|
|
6970
|
+
runStart = at;
|
|
6971
|
+
runEnd = at + fieldRun.length;
|
|
6972
|
+
}
|
|
7664
6973
|
const fieldIndex = fieldNameIndices[pi];
|
|
7665
6974
|
if (searchable[fieldIndex] === 0) continue;
|
|
7666
6975
|
const termFrequency = termFrequencies[pi];
|
|
@@ -7688,10 +6997,10 @@ function searchFulltext(state, params) {
|
|
|
7688
6997
|
}
|
|
7689
6998
|
}
|
|
7690
6999
|
}
|
|
7691
|
-
function scoreGroup(group,
|
|
7000
|
+
function scoreGroup(group, only) {
|
|
7692
7001
|
if (group === null) return;
|
|
7693
7002
|
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
7694
|
-
if (
|
|
7003
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
7695
7004
|
mergeGroupContribution(internalId, contribution);
|
|
7696
7005
|
}
|
|
7697
7006
|
}
|
|
@@ -7732,19 +7041,27 @@ function searchFulltext(state, params) {
|
|
|
7732
7041
|
}
|
|
7733
7042
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
7734
7043
|
}
|
|
7735
|
-
|
|
7736
|
-
for (
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7044
|
+
let anchor = resolved[0];
|
|
7045
|
+
for (const term of resolved) if (term.totalPostings < anchor.totalPostings) anchor = term;
|
|
7046
|
+
for (const match of anchor.matches) scorePostings(match, null);
|
|
7047
|
+
scoreGroup(anchor.group, null);
|
|
7048
|
+
const anchorDocuments = createBitSet(state.docStore.internalIdCapacity());
|
|
7049
|
+
for (let index = 0; index < scoreBuffer.touchedCount; index++) {
|
|
7050
|
+
bitsetSet(anchorDocuments, scoreBuffer.touched[index]);
|
|
7051
|
+
}
|
|
7052
|
+
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
7053
|
+
components?.clear();
|
|
7054
|
+
for (const term of resolved) {
|
|
7055
|
+
for (const match of term.matches) scorePostings(match, anchorDocuments);
|
|
7056
|
+
scoreGroup(term.group, anchorDocuments);
|
|
7740
7057
|
}
|
|
7741
7058
|
} else {
|
|
7742
7059
|
for (const qt of queryTokens) {
|
|
7743
7060
|
const group = termGroupOf(termGroups, qt.token);
|
|
7744
7061
|
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7745
|
-
scorePostings(resolveMatch(m.token, m.postingList),
|
|
7062
|
+
scorePostings(resolveMatch(m.token, m.postingList), null);
|
|
7746
7063
|
}
|
|
7747
|
-
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group),
|
|
7064
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), null);
|
|
7748
7065
|
}
|
|
7749
7066
|
}
|
|
7750
7067
|
if (phraseMatches !== null) {
|
|
@@ -8030,7 +7347,7 @@ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
|
8030
7347
|
for (const entry of payload.geo) {
|
|
8031
7348
|
let index = target.geoIndexes.get(entry.fieldPath);
|
|
8032
7349
|
if (index === void 0) {
|
|
8033
|
-
index =
|
|
7350
|
+
index = createLoadedGeoIndex();
|
|
8034
7351
|
target.geoIndexes.set(entry.fieldPath, index);
|
|
8035
7352
|
}
|
|
8036
7353
|
for (let i = 0; i < entry.docIds.length; i++) {
|
|
@@ -8264,7 +7581,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8264
7581
|
state.enumIndexes.set(path, idx);
|
|
8265
7582
|
}
|
|
8266
7583
|
for (const [path, entries] of Object.entries(data.fieldIndexes.geopoint)) {
|
|
8267
|
-
const geoIdx =
|
|
7584
|
+
const geoIdx = createLoadedGeoIndex();
|
|
8268
7585
|
const converted = entries.map((e) => ({
|
|
8269
7586
|
lat: e.lat,
|
|
8270
7587
|
lon: e.lon,
|
|
@@ -9668,22 +8985,22 @@ function createWriteLogRecorder() {
|
|
|
9668
8985
|
const current = [null, null];
|
|
9669
8986
|
return {
|
|
9670
8987
|
open(owner) {
|
|
9671
|
-
const
|
|
9672
|
-
current[owner] =
|
|
8988
|
+
const log2 = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
8989
|
+
current[owner] = log2;
|
|
9673
8990
|
return {
|
|
9674
8991
|
get partitionsReplaced() {
|
|
9675
|
-
return
|
|
8992
|
+
return log2.replaced;
|
|
9676
8993
|
},
|
|
9677
8994
|
get pendingDocuments() {
|
|
9678
|
-
return
|
|
8995
|
+
return log2.docIds.size;
|
|
9679
8996
|
},
|
|
9680
8997
|
takeWrittenDocIds() {
|
|
9681
|
-
const taken =
|
|
9682
|
-
|
|
8998
|
+
const taken = log2.docIds;
|
|
8999
|
+
log2.docIds = /* @__PURE__ */ new Set();
|
|
9683
9000
|
return taken;
|
|
9684
9001
|
},
|
|
9685
9002
|
close() {
|
|
9686
|
-
if (current[owner] ===
|
|
9003
|
+
if (current[owner] === log2) current[owner] = null;
|
|
9687
9004
|
}
|
|
9688
9005
|
};
|
|
9689
9006
|
},
|
|
@@ -9692,17 +9009,17 @@ function createWriteLogRecorder() {
|
|
|
9692
9009
|
current[1]?.docIds.add(docId);
|
|
9693
9010
|
},
|
|
9694
9011
|
segmentWritten(segment) {
|
|
9695
|
-
for (const
|
|
9696
|
-
if (
|
|
9012
|
+
for (const log2 of current) {
|
|
9013
|
+
if (log2 === null) continue;
|
|
9697
9014
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
9698
9015
|
const docId = segment.docStore.getExternalId(internalId);
|
|
9699
|
-
if (docId !== void 0)
|
|
9016
|
+
if (docId !== void 0) log2.docIds.add(docId);
|
|
9700
9017
|
}
|
|
9701
9018
|
}
|
|
9702
9019
|
},
|
|
9703
9020
|
partitionsReplaced() {
|
|
9704
|
-
for (const
|
|
9705
|
-
if (
|
|
9021
|
+
for (const log2 of current) {
|
|
9022
|
+
if (log2 !== null) log2.replaced = true;
|
|
9706
9023
|
}
|
|
9707
9024
|
}
|
|
9708
9025
|
};
|
|
@@ -11139,6 +10456,22 @@ async function runQueryAction(entry, action) {
|
|
|
11139
10456
|
return { count: result.totalMatched };
|
|
11140
10457
|
}
|
|
11141
10458
|
|
|
10459
|
+
// src/workers/vector-code.ts
|
|
10460
|
+
var vectorCode = codeLoader(() => import('./module-TCT66FWA.mjs'), "vector");
|
|
10461
|
+
var loadedVectorCode = vectorCode.loaded;
|
|
10462
|
+
var requireLoadedVectorCode = vectorCode.require;
|
|
10463
|
+
var loadVectorCode = vectorCode.load;
|
|
10464
|
+
var loadVectorCodeFor = vectorCode.loadFor;
|
|
10465
|
+
|
|
10466
|
+
// src/workers/schema-code.ts
|
|
10467
|
+
function loadSchemaCodeFor(schema) {
|
|
10468
|
+
const vectorCode2 = loadVectorCodeFor(schema);
|
|
10469
|
+
const geoCode2 = loadGeoCodeFor(schema);
|
|
10470
|
+
if (vectorCode2 === null) return geoCode2;
|
|
10471
|
+
if (geoCode2 === null) return vectorCode2;
|
|
10472
|
+
return Promise.all([vectorCode2, geoCode2]);
|
|
10473
|
+
}
|
|
10474
|
+
|
|
11142
10475
|
// src/core/partition/frozen/doc-table.ts
|
|
11143
10476
|
function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
11144
10477
|
const capacity = idTable.count;
|
|
@@ -12376,7 +11709,7 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12376
11709
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
12377
11710
|
}
|
|
12378
11711
|
}
|
|
12379
|
-
function
|
|
11712
|
+
function loaded2() {
|
|
12380
11713
|
if (registry === null) {
|
|
12381
11714
|
const built = createSurfaceRegistry();
|
|
12382
11715
|
const forms = load();
|
|
@@ -12395,10 +11728,10 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12395
11728
|
return {
|
|
12396
11729
|
heldBytes: () => table?.heldBytes() ?? 0,
|
|
12397
11730
|
reader: {
|
|
12398
|
-
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) :
|
|
12399
|
-
stemChangedTotalFor: (token) =>
|
|
12400
|
-
size: () =>
|
|
12401
|
-
serialize: () =>
|
|
11731
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded2().candidatesForPrefix(prefix),
|
|
11732
|
+
stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
|
|
11733
|
+
size: () => loaded2().size(),
|
|
11734
|
+
serialize: () => loaded2().serialize()
|
|
12402
11735
|
},
|
|
12403
11736
|
tableStaysOnDisk() {
|
|
12404
11737
|
return registry === null && tableOnDisk !== void 0;
|
|
@@ -12410,20 +11743,20 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12410
11743
|
waitingRemovals.push(ordinal);
|
|
12411
11744
|
},
|
|
12412
11745
|
holdsAny() {
|
|
12413
|
-
return
|
|
11746
|
+
return loaded2().size() > 0;
|
|
12414
11747
|
},
|
|
12415
11748
|
holds(spelling) {
|
|
12416
|
-
return
|
|
11749
|
+
return loaded2().holds(spelling);
|
|
12417
11750
|
},
|
|
12418
11751
|
subtractSpellings(ordinal, spellings) {
|
|
12419
11752
|
uncounted ??= new Uint8Array(documents.count);
|
|
12420
11753
|
uncounted[ordinal] = 1;
|
|
12421
11754
|
if (spellings.length === 0) return;
|
|
12422
|
-
const target =
|
|
11755
|
+
const target = loaded2();
|
|
12423
11756
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
12424
11757
|
},
|
|
12425
11758
|
addKeptForms(target, mergedOrdinals, analysis) {
|
|
12426
|
-
const source =
|
|
11759
|
+
const source = loaded2();
|
|
12427
11760
|
if (analysis === null) {
|
|
12428
11761
|
addForms(target, source.serialize());
|
|
12429
11762
|
return;
|
|
@@ -12825,7 +12158,7 @@ function storedFrequenciesOf(source) {
|
|
|
12825
12158
|
function buildGeoReaders(entries) {
|
|
12826
12159
|
const readers = /* @__PURE__ */ new Map();
|
|
12827
12160
|
for (const entry of entries) {
|
|
12828
|
-
const index =
|
|
12161
|
+
const index = createLoadedGeoIndex();
|
|
12829
12162
|
for (let i = 0; i < entry.docIds.length; i++) {
|
|
12830
12163
|
index.insert(entry.docIds[i], entry.latitudes[i], entry.longitudes[i]);
|
|
12831
12164
|
}
|
|
@@ -12881,6 +12214,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12881
12214
|
sharedSnapshot,
|
|
12882
12215
|
paged: pagedParts?.access ?? null,
|
|
12883
12216
|
arrays: source,
|
|
12217
|
+
tombstones,
|
|
12884
12218
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
12885
12219
|
docStore,
|
|
12886
12220
|
stats: buildStatsView(counts),
|
|
@@ -13664,7 +12998,7 @@ function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
|
13664
12998
|
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
13665
12999
|
} else if (fieldType === "geopoint") {
|
|
13666
13000
|
const point = value;
|
|
13667
|
-
indexOf(indexes.geoIndexes, fieldPath,
|
|
13001
|
+
indexOf(indexes.geoIndexes, fieldPath, createLoadedGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
13668
13002
|
}
|
|
13669
13003
|
}
|
|
13670
13004
|
function refuseRepeatedDocIds(documents) {
|
|
@@ -13815,7 +13149,7 @@ async function attachSegments(entry, action) {
|
|
|
13815
13149
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
13816
13150
|
const surfaceForms = segment.survivingSurfaceForms;
|
|
13817
13151
|
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
13818
|
-
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-
|
|
13152
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-NDSQLPMQ.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
13819
13153
|
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
13820
13154
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
13821
13155
|
}
|
|
@@ -13870,7 +13204,7 @@ function runSegmentAction(entry, action) {
|
|
|
13870
13204
|
case "attachSegments":
|
|
13871
13205
|
return attachSegments(entry, action);
|
|
13872
13206
|
case "adoptDiskSegments":
|
|
13873
|
-
return import('./module-
|
|
13207
|
+
return import('./module-NDSQLPMQ.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
13874
13208
|
case "compactSegments": {
|
|
13875
13209
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
13876
13210
|
return mergeFrozenSegments(
|
|
@@ -13904,47 +13238,6 @@ function runSegmentAction(entry, action) {
|
|
|
13904
13238
|
}
|
|
13905
13239
|
}
|
|
13906
13240
|
|
|
13907
|
-
// src/workers/vector-code.ts
|
|
13908
|
-
var VECTOR_TYPE_NAME = "vector";
|
|
13909
|
-
var loaded2 = null;
|
|
13910
|
-
var loading2 = null;
|
|
13911
|
-
function loadedVectorCode() {
|
|
13912
|
-
return loaded2;
|
|
13913
|
-
}
|
|
13914
|
-
function requireLoadedVectorCode() {
|
|
13915
|
-
if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
|
|
13916
|
-
return loaded2;
|
|
13917
|
-
}
|
|
13918
|
-
function loadVectorCode() {
|
|
13919
|
-
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
13920
|
-
if (loading2 === null) {
|
|
13921
|
-
loading2 = import('./module-3HNFCBJ5.mjs').then(
|
|
13922
|
-
(code) => {
|
|
13923
|
-
loaded2 = code;
|
|
13924
|
-
return code;
|
|
13925
|
-
},
|
|
13926
|
-
(err) => {
|
|
13927
|
-
loading2 = null;
|
|
13928
|
-
throw err;
|
|
13929
|
-
}
|
|
13930
|
-
);
|
|
13931
|
-
}
|
|
13932
|
-
return loading2;
|
|
13933
|
-
}
|
|
13934
|
-
function mayDeclareVectorField(schema, depth) {
|
|
13935
|
-
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
13936
|
-
for (const type of Object.values(schema)) {
|
|
13937
|
-
if (typeof type === "string" ? type.includes(VECTOR_TYPE_NAME) : mayDeclareVectorField(type, depth + 1)) {
|
|
13938
|
-
return true;
|
|
13939
|
-
}
|
|
13940
|
-
}
|
|
13941
|
-
return false;
|
|
13942
|
-
}
|
|
13943
|
-
function loadVectorCodeFor(schema) {
|
|
13944
|
-
if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
|
|
13945
|
-
return loadVectorCode();
|
|
13946
|
-
}
|
|
13947
|
-
|
|
13948
13241
|
// src/workers/vector-copies.ts
|
|
13949
13242
|
function createHeldVectorCopies() {
|
|
13950
13243
|
return { searchers: /* @__PURE__ */ new Map(), fields: /* @__PURE__ */ new Map(), builds: /* @__PURE__ */ new Map() };
|
|
@@ -14150,8 +13443,8 @@ function createDirectExecutor(options) {
|
|
|
14150
13443
|
);
|
|
14151
13444
|
}
|
|
14152
13445
|
case "createIndex": {
|
|
14153
|
-
const
|
|
14154
|
-
if (
|
|
13446
|
+
const loading2 = loadSchemaCodeFor(action.config.schema);
|
|
13447
|
+
if (loading2 !== null) await loading2;
|
|
14155
13448
|
const language = getLanguage(action.config.language ?? "english");
|
|
14156
13449
|
const manager = addIndex(
|
|
14157
13450
|
action.indexName,
|
|
@@ -14202,16 +13495,19 @@ function createDirectExecutor(options) {
|
|
|
14202
13495
|
case "insert": {
|
|
14203
13496
|
const entry = requireIndex(action.indexName);
|
|
14204
13497
|
entry.manager.insert(action.docId, action.document, action.skipClone ? { skipClone: true } : void 0);
|
|
13498
|
+
options?.onDocumentWrite?.(action);
|
|
14205
13499
|
return void 0;
|
|
14206
13500
|
}
|
|
14207
13501
|
case "remove": {
|
|
14208
13502
|
const entry = requireIndex(action.indexName);
|
|
14209
13503
|
entry.manager.remove(action.docId, action.removedSpellings);
|
|
13504
|
+
options?.onDocumentWrite?.(action);
|
|
14210
13505
|
return void 0;
|
|
14211
13506
|
}
|
|
14212
13507
|
case "update": {
|
|
14213
13508
|
const entry = requireIndex(action.indexName);
|
|
14214
13509
|
entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
|
|
13510
|
+
options?.onDocumentWrite?.(action);
|
|
14215
13511
|
return void 0;
|
|
14216
13512
|
}
|
|
14217
13513
|
case "query":
|
|
@@ -14258,6 +13554,7 @@ function createDirectExecutor(options) {
|
|
|
14258
13554
|
entry.manager.resetVectorIndexes(vectorIndexesFor(action.indexName, entry.config));
|
|
14259
13555
|
entry.vectorIndexes = entry.manager.getVectorIndexes();
|
|
14260
13556
|
entry.vectorCopies = createHeldVectorCopies();
|
|
13557
|
+
options?.onDocumentWrite?.(action);
|
|
14261
13558
|
return void 0;
|
|
14262
13559
|
}
|
|
14263
13560
|
case "serialize": {
|
|
@@ -14334,7 +13631,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
14334
13631
|
// src/partitioning/rewrite-spill-of.ts
|
|
14335
13632
|
async function rewriteSpillOf(storage, manager, target) {
|
|
14336
13633
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
14337
|
-
const { openRewriteSpill } = await import('./module-
|
|
13634
|
+
const { openRewriteSpill } = await import('./module-NDSQLPMQ.mjs');
|
|
14338
13635
|
return openRewriteSpill(storage, manager, target);
|
|
14339
13636
|
}
|
|
14340
13637
|
|
|
@@ -14424,16 +13721,16 @@ async function rebuildTextIndexes(manager, hooks, target) {
|
|
|
14424
13721
|
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
14425
13722
|
const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
|
|
14426
13723
|
while (hooks.isCurrent()) {
|
|
14427
|
-
const
|
|
13724
|
+
const log2 = manager.openWriteLog();
|
|
14428
13725
|
let swapped = false;
|
|
14429
13726
|
try {
|
|
14430
|
-
swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
|
|
13727
|
+
swapped = await rebuildOnce({ manager, log: log2, hooks, language, partialWordAnalysis, options, spill });
|
|
14431
13728
|
if (swapped) {
|
|
14432
13729
|
hooks.onRewriteSpill?.(spill);
|
|
14433
13730
|
return true;
|
|
14434
13731
|
}
|
|
14435
13732
|
} finally {
|
|
14436
|
-
|
|
13733
|
+
log2.close();
|
|
14437
13734
|
if (!swapped) await spill?.abandon();
|
|
14438
13735
|
}
|
|
14439
13736
|
}
|
|
@@ -14937,12 +14234,12 @@ function recoveryFailure(metadata, err) {
|
|
|
14937
14234
|
});
|
|
14938
14235
|
}
|
|
14939
14236
|
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
14940
|
-
const
|
|
14941
|
-
if (
|
|
14237
|
+
const loadingSchemaCode = loadData ? loadSchemaCodeFor(metadata.schema) : null;
|
|
14238
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
14942
14239
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
14943
14240
|
if (existing !== void 0) {
|
|
14944
14241
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
14945
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
14242
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-ZKGJFENM.mjs');
|
|
14946
14243
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
14947
14244
|
}
|
|
14948
14245
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -15962,12 +15259,12 @@ function validateVectorDimensions(vectors, vecIndexes) {
|
|
|
15962
15259
|
}
|
|
15963
15260
|
}
|
|
15964
15261
|
}
|
|
15965
|
-
function prepareDocumentVectors(document,
|
|
15966
|
-
if (
|
|
15262
|
+
function prepareDocumentVectors(document, vectorFieldPaths) {
|
|
15263
|
+
if (vectorFieldPaths.size === 0) {
|
|
15967
15264
|
return { partitionDoc: document, extractedVectors: /* @__PURE__ */ new Map() };
|
|
15968
15265
|
}
|
|
15969
15266
|
const extractedVectors = /* @__PURE__ */ new Map();
|
|
15970
|
-
for (const fieldPath of
|
|
15267
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
15971
15268
|
const vec = extractVectorFromDoc(document, fieldPath);
|
|
15972
15269
|
if (vec) {
|
|
15973
15270
|
extractedVectors.set(fieldPath, vec);
|
|
@@ -16181,16 +15478,16 @@ function documentWeight(action) {
|
|
|
16181
15478
|
}
|
|
16182
15479
|
return 1;
|
|
16183
15480
|
}
|
|
15481
|
+
var NO_VECTOR_FIELDS = /* @__PURE__ */ new Set();
|
|
16184
15482
|
function vectorFieldPathsOf(state, indexName) {
|
|
16185
|
-
|
|
16186
|
-
if (entry === void 0) return /* @__PURE__ */ new Set();
|
|
16187
|
-
return new Set(extractVectorFieldsFromSchema(entry.config.schema).keys());
|
|
15483
|
+
return state.indexRegistry.get(indexName)?.vectorFieldPaths ?? NO_VECTOR_FIELDS;
|
|
16188
15484
|
}
|
|
16189
15485
|
function withoutVectors(document, fieldPaths) {
|
|
15486
|
+
if (fieldPaths.size === 0) return document;
|
|
16190
15487
|
return prepareDocumentVectors(document, fieldPaths).partitionDoc;
|
|
16191
15488
|
}
|
|
16192
15489
|
function detachCallerDocuments(state, action) {
|
|
16193
|
-
if (
|
|
15490
|
+
if (action.type !== "insert" && action.type !== "update" && action.type !== "mergeSegments") return action;
|
|
16194
15491
|
const fieldPaths = vectorFieldPathsOf(state, action.indexName);
|
|
16195
15492
|
if (action.type === "insert") {
|
|
16196
15493
|
const stripped = withoutVectors(action.document, fieldPaths);
|
|
@@ -16201,17 +15498,14 @@ function detachCallerDocuments(state, action) {
|
|
|
16201
15498
|
const stripped = withoutVectors(action.document, fieldPaths);
|
|
16202
15499
|
return { ...action, document: stripped !== action.document ? stripped : structuredClone(stripped) };
|
|
16203
15500
|
}
|
|
16204
|
-
|
|
16205
|
-
|
|
16206
|
-
|
|
16207
|
-
|
|
16208
|
-
|
|
16209
|
-
|
|
16210
|
-
|
|
16211
|
-
|
|
16212
|
-
};
|
|
16213
|
-
}
|
|
16214
|
-
return action;
|
|
15501
|
+
return {
|
|
15502
|
+
...action,
|
|
15503
|
+
segments: action.segments.map((segment) => {
|
|
15504
|
+
const documents = segment.documents.map((document) => withoutVectors(document, fieldPaths));
|
|
15505
|
+
const cloned = action.skipClone === true || fieldPaths.size > 0 ? documents : structuredClone(documents);
|
|
15506
|
+
return { ...segment, documents: cloned };
|
|
15507
|
+
})
|
|
15508
|
+
};
|
|
16215
15509
|
}
|
|
16216
15510
|
function partitionLedger(state, indexName, partitionId) {
|
|
16217
15511
|
let byPartition = state.segmentLedger.get(indexName);
|
|
@@ -16227,6 +15521,10 @@ function partitionLedger(state, indexName, partitionId) {
|
|
|
16227
15521
|
return entries;
|
|
16228
15522
|
}
|
|
16229
15523
|
function recordSegmentBroadcast(state, action) {
|
|
15524
|
+
if (action.type === "clear") {
|
|
15525
|
+
state.segmentLedger.delete(action.indexName);
|
|
15526
|
+
return;
|
|
15527
|
+
}
|
|
16230
15528
|
if (action.type === "attachSegments") {
|
|
16231
15529
|
for (const segment of action.segments) {
|
|
16232
15530
|
partitionLedger(state, action.indexName, segment.partitionId).push({
|
|
@@ -16310,30 +15608,36 @@ async function replicateToWorkers(state, action) {
|
|
|
16310
15608
|
await dispatchToWorkers(state, action);
|
|
16311
15609
|
return;
|
|
16312
15610
|
}
|
|
16313
|
-
|
|
16314
|
-
const
|
|
16315
|
-
|
|
16316
|
-
if (loading3 !== void 0) {
|
|
16317
|
-
queueForCopies(state, detached);
|
|
16318
|
-
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
|
|
16319
|
-
return;
|
|
16320
|
-
}
|
|
16321
|
-
const queue = state.replicationQueues.get(action.indexName);
|
|
16322
|
-
if (queue !== void 0 && queue.pendingDocuments + documentWeight(detached) > MAX_PENDING_REPLICATION_DOCUMENTS) {
|
|
16323
|
-
await new Promise((resume) => queue.drainWaiters.push(resume));
|
|
16324
|
-
}
|
|
16325
|
-
queueForCopies(state, detached);
|
|
15611
|
+
recordMainWrite(state, action);
|
|
15612
|
+
const waiting = replicationRoomWait(state, action.indexName);
|
|
15613
|
+
if (waiting !== null) await waiting;
|
|
16326
15614
|
}
|
|
16327
15615
|
function queueForCopies(state, action) {
|
|
16328
15616
|
if (!("indexName" in action)) return;
|
|
16329
|
-
const
|
|
16330
|
-
if (
|
|
16331
|
-
|
|
15617
|
+
const loading2 = state.copyLoadBuffers.get(action.indexName);
|
|
15618
|
+
if (loading2 !== void 0) {
|
|
15619
|
+
loading2.push(action);
|
|
16332
15620
|
return;
|
|
16333
15621
|
}
|
|
16334
15622
|
if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
|
|
16335
15623
|
enqueueReplication(state, action.indexName, action);
|
|
16336
15624
|
}
|
|
15625
|
+
function recordMainWrite(state, action) {
|
|
15626
|
+
if (!("indexName" in action) || !copiesWillReceive(state, action.indexName)) return;
|
|
15627
|
+
queueForCopies(state, detachCallerDocuments(state, action));
|
|
15628
|
+
}
|
|
15629
|
+
function replicationRoomWait(state, indexName) {
|
|
15630
|
+
const loading2 = state.copyLoadBuffers.get(indexName);
|
|
15631
|
+
if (loading2 !== void 0) {
|
|
15632
|
+
const lastYield = state.copyLoadBufferYieldMarks.get(loading2) ?? 0;
|
|
15633
|
+
if (loading2.length - lastYield < LOAD_BUFFER_YIELD_INTERVAL) return null;
|
|
15634
|
+
state.copyLoadBufferYieldMarks.set(loading2, loading2.length);
|
|
15635
|
+
return yieldToEventLoop2();
|
|
15636
|
+
}
|
|
15637
|
+
const queue = state.replicationQueues.get(indexName);
|
|
15638
|
+
if (queue === void 0 || queue.pendingDocuments < MAX_PENDING_REPLICATION_DOCUMENTS) return null;
|
|
15639
|
+
return new Promise((resume) => queue.drainWaiters.push(resume));
|
|
15640
|
+
}
|
|
16337
15641
|
function settled(queue) {
|
|
16338
15642
|
return Promise.all([...queue.inFlight, queue.tail]).then(() => void 0);
|
|
16339
15643
|
}
|
|
@@ -16410,6 +15714,7 @@ function freezeLiveTail(state, indexName, manager, partitionId, floor) {
|
|
|
16410
15714
|
}
|
|
16411
15715
|
}
|
|
16412
15716
|
function flushGrownTails(state, indexName) {
|
|
15717
|
+
if (!state.scaledOutIndexes.has(indexName)) return;
|
|
16413
15718
|
const manager = state.executor.getManager(indexName);
|
|
16414
15719
|
if (manager === void 0) return;
|
|
16415
15720
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
@@ -16432,25 +15737,10 @@ function mainCompositeOf(manager, partitionId) {
|
|
|
16432
15737
|
const partition = manager.getPartition(partitionId);
|
|
16433
15738
|
return isCompositePartition(partition) ? partition : null;
|
|
16434
15739
|
}
|
|
16435
|
-
function holdUnbroadcastSegments(state, indexName, segmentIds) {
|
|
16436
|
-
let held = state.unbroadcastSegments.get(indexName);
|
|
16437
|
-
if (held === void 0) {
|
|
16438
|
-
held = /* @__PURE__ */ new Set();
|
|
16439
|
-
state.unbroadcastSegments.set(indexName, held);
|
|
16440
|
-
}
|
|
16441
|
-
for (const segmentId of segmentIds) held.add(segmentId);
|
|
16442
|
-
}
|
|
16443
|
-
function releaseUnbroadcastSegments(state, indexName, segmentIds) {
|
|
16444
|
-
const held = state.unbroadcastSegments.get(indexName);
|
|
16445
|
-
if (held === void 0) return;
|
|
16446
|
-
for (const segmentId of segmentIds) held.delete(segmentId);
|
|
16447
|
-
if (held.size === 0) state.unbroadcastSegments.delete(indexName);
|
|
16448
|
-
}
|
|
16449
15740
|
function candidateSegmentIds(state, indexName, manager, partitionId, policy) {
|
|
16450
|
-
const unbroadcast = state.unbroadcastSegments.get(indexName);
|
|
16451
15741
|
const composite = mainCompositeOf(manager, partitionId);
|
|
16452
15742
|
if (composite !== null) {
|
|
16453
|
-
const sizes = composite.frozenSegmentSizes().
|
|
15743
|
+
const sizes = composite.frozenSegmentSizes().map((size) => ({ segmentId: size.segmentId, documentCount: size.liveDocumentCount }));
|
|
16454
15744
|
const picks = pickSegmentIds(sizes, policy);
|
|
16455
15745
|
if (picks !== null) return picks;
|
|
16456
15746
|
}
|
|
@@ -16484,6 +15774,17 @@ async function compactOnWorker(state, indexName, partitionId, segmentIds) {
|
|
|
16484
15774
|
lease.release();
|
|
16485
15775
|
}
|
|
16486
15776
|
}
|
|
15777
|
+
function ledgerHoldsAll(state, indexName, partitionId, segmentIds) {
|
|
15778
|
+
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
15779
|
+
if (entries === void 0) return false;
|
|
15780
|
+
const held = new Set(entries.map((entry) => entry.segmentId));
|
|
15781
|
+
return segmentIds.every((segmentId) => held.has(segmentId));
|
|
15782
|
+
}
|
|
15783
|
+
function segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain) {
|
|
15784
|
+
if (state.executor.getManager(indexName) !== manager) return false;
|
|
15785
|
+
if (mergedOnMain === null) return ledgerHoldsAll(state, indexName, partitionId, segmentIds);
|
|
15786
|
+
return mainCompositeOf(manager, partitionId) === mergedOnMain && mainHoldsAll(mergedOnMain, segmentIds);
|
|
15787
|
+
}
|
|
16487
15788
|
function forgetLedgerSegments(state, indexName, partitionId, segmentIds) {
|
|
16488
15789
|
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
16489
15790
|
if (entries === void 0) return;
|
|
@@ -16509,8 +15810,10 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
16509
15810
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
16510
15811
|
return false;
|
|
16511
15812
|
}
|
|
16512
|
-
const
|
|
16513
|
-
|
|
15813
|
+
const mergedOnMain = mainCanMerge ? composite : null;
|
|
15814
|
+
if (!segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain)) return false;
|
|
15815
|
+
const removedSpellings = mergedOnMain?.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
|
|
15816
|
+
recordMainWrite(state, {
|
|
16514
15817
|
type: "swapSegments",
|
|
16515
15818
|
indexName,
|
|
16516
15819
|
partitionId,
|
|
@@ -16552,19 +15855,34 @@ function maybeCompactSegments(state, indexName, policy = "ingest") {
|
|
|
16552
15855
|
});
|
|
16553
15856
|
state.compactionsInFlight.set(indexName, run);
|
|
16554
15857
|
}
|
|
16555
|
-
function
|
|
16556
|
-
cancelIdleMerge(state, indexName);
|
|
15858
|
+
function armIdleMerge(state, indexName, dueAt, delayMs) {
|
|
16557
15859
|
const timer = setTimeout(() => {
|
|
15860
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
15861
|
+
if (pending === void 0 || pending.timer !== timer) return;
|
|
15862
|
+
const remainingMs = pending.dueAt - Date.now();
|
|
15863
|
+
if (remainingMs > 0) {
|
|
15864
|
+
armIdleMerge(state, indexName, pending.dueAt, remainingMs);
|
|
15865
|
+
return;
|
|
15866
|
+
}
|
|
16558
15867
|
state.idleMergeTimers.delete(indexName);
|
|
16559
15868
|
maybeCompactSegments(state, indexName, "idle");
|
|
16560
|
-
},
|
|
15869
|
+
}, delayMs);
|
|
16561
15870
|
if (typeof timer.unref === "function") timer.unref();
|
|
16562
|
-
state.idleMergeTimers.set(indexName, timer);
|
|
15871
|
+
state.idleMergeTimers.set(indexName, { timer, dueAt });
|
|
15872
|
+
}
|
|
15873
|
+
function scheduleIdleMerge(state, indexName) {
|
|
15874
|
+
const dueAt = Date.now() + IDLE_MERGE_DELAY_MS;
|
|
15875
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
15876
|
+
if (pending !== void 0) {
|
|
15877
|
+
pending.dueAt = dueAt;
|
|
15878
|
+
return;
|
|
15879
|
+
}
|
|
15880
|
+
armIdleMerge(state, indexName, dueAt, IDLE_MERGE_DELAY_MS);
|
|
16563
15881
|
}
|
|
16564
15882
|
function cancelIdleMerge(state, indexName) {
|
|
16565
|
-
const
|
|
16566
|
-
if (
|
|
16567
|
-
clearTimeout(timer);
|
|
15883
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
15884
|
+
if (pending === void 0) return;
|
|
15885
|
+
clearTimeout(pending.timer);
|
|
16568
15886
|
state.idleMergeTimers.delete(indexName);
|
|
16569
15887
|
}
|
|
16570
15888
|
async function awaitCompactions(state) {
|
|
@@ -16715,10 +16033,10 @@ function createWorkerExecutor(worker, config) {
|
|
|
16715
16033
|
}
|
|
16716
16034
|
const requestId = createRequestId();
|
|
16717
16035
|
const taggedAction = { ...action, requestId };
|
|
16718
|
-
return new Promise((resolve,
|
|
16036
|
+
return new Promise((resolve, reject) => {
|
|
16719
16037
|
const entry = {
|
|
16720
16038
|
resolve,
|
|
16721
|
-
reject
|
|
16039
|
+
reject,
|
|
16722
16040
|
timeoutId: setTimeout(() => {
|
|
16723
16041
|
entry.timeoutId = setTimeout(() => declareTimedOut(requestId, entry), 0);
|
|
16724
16042
|
}, requestTimeout)
|
|
@@ -17506,13 +16824,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17506
16824
|
desyncedIndexes: /* @__PURE__ */ new Set(),
|
|
17507
16825
|
copyReleaseHolds: /* @__PURE__ */ new Map(),
|
|
17508
16826
|
copyLoadBuffers: /* @__PURE__ */ new Map(),
|
|
16827
|
+
copyLoadBufferYieldMarks: /* @__PURE__ */ new WeakMap(),
|
|
17509
16828
|
copyTransitions: /* @__PURE__ */ new Map(),
|
|
17510
16829
|
droppedCopies: /* @__PURE__ */ new Map(),
|
|
17511
16830
|
lastAccessAt: /* @__PURE__ */ new Map(),
|
|
17512
16831
|
copyReloadCounts: /* @__PURE__ */ new Map(),
|
|
17513
16832
|
replicationQueues: /* @__PURE__ */ new Map(),
|
|
17514
16833
|
segmentLedger: /* @__PURE__ */ new Map(),
|
|
17515
|
-
unbroadcastSegments: /* @__PURE__ */ new Map(),
|
|
17516
16834
|
compactionsInFlight: /* @__PURE__ */ new Map(),
|
|
17517
16835
|
idleMergeTimers: /* @__PURE__ */ new Map(),
|
|
17518
16836
|
sharedVectorFields: /* @__PURE__ */ new Map(),
|
|
@@ -17667,7 +16985,6 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17667
16985
|
state.lastAccessAt.delete(indexName);
|
|
17668
16986
|
state.replicationQueues.delete(indexName);
|
|
17669
16987
|
state.segmentLedger.delete(indexName);
|
|
17670
|
-
state.unbroadcastSegments.delete(indexName);
|
|
17671
16988
|
}
|
|
17672
16989
|
async function replicate(action) {
|
|
17673
16990
|
await replicateToWorkers(state, action);
|
|
@@ -17678,10 +16995,31 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17678
16995
|
scheduleIdleMerge(state, action.indexName);
|
|
17679
16996
|
}
|
|
17680
16997
|
}
|
|
16998
|
+
function afterMainWrites(indexName, growth) {
|
|
16999
|
+
if (growth === "segments") maybeCompactSegments(state, indexName);
|
|
17000
|
+
if (growth === "liveTail") flushGrownTails(state, indexName);
|
|
17001
|
+
if (state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) {
|
|
17002
|
+
scheduleIdleMerge(state, indexName);
|
|
17003
|
+
}
|
|
17004
|
+
}
|
|
17005
|
+
function copiesFollow(indexName) {
|
|
17006
|
+
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName) || state.replicationQueues.has(indexName);
|
|
17007
|
+
}
|
|
17008
|
+
function settleMainWrites(indexName, growth = "liveTail") {
|
|
17009
|
+
if (growth !== "segments" && !copiesFollow(indexName)) return null;
|
|
17010
|
+
const waiting = replicationRoomWait(state, indexName);
|
|
17011
|
+
if (waiting === null) {
|
|
17012
|
+
afterMainWrites(indexName, growth);
|
|
17013
|
+
return null;
|
|
17014
|
+
}
|
|
17015
|
+
return waiting.then(() => afterMainWrites(indexName, growth));
|
|
17016
|
+
}
|
|
17681
17017
|
return {
|
|
17682
17018
|
scaleOutReadyIndexes: () => scaleOutReadyIndexes(state),
|
|
17683
17019
|
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
17684
17020
|
replicateToWorkers: replicate,
|
|
17021
|
+
recordMainWrite: (action) => recordMainWrite(state, action),
|
|
17022
|
+
settleMainWrites,
|
|
17685
17023
|
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
17686
17024
|
queueDiskAdoption: (action) => queueForCopies(state, action),
|
|
17687
17025
|
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
@@ -17691,8 +17029,6 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17691
17029
|
isIndexBusy: (indexName) => isIndexBusy(state, indexName),
|
|
17692
17030
|
buildSegments: (requests) => buildSegments(state, requests),
|
|
17693
17031
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
17694
|
-
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
17695
|
-
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
17696
17032
|
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25),
|
|
17697
17033
|
hasWorkerPool: () => state.workerPool !== null,
|
|
17698
17034
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
@@ -17705,21 +17041,21 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17705
17041
|
stopRequestThreads: () => stopRequestThreads(state),
|
|
17706
17042
|
desyncIndex: (indexName) => {
|
|
17707
17043
|
state.desyncedIndexes.add(indexName);
|
|
17708
|
-
const
|
|
17044
|
+
const loading2 = state.copyTransitions.get(indexName);
|
|
17709
17045
|
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
17710
17046
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
17711
|
-
return wasScaledOut ||
|
|
17047
|
+
return wasScaledOut || loading2 !== void 0 && loading2.kind !== "drop";
|
|
17712
17048
|
},
|
|
17713
17049
|
async releaseIndexCopies(indexName) {
|
|
17714
17050
|
const hold = state.copyReleaseHolds.get(indexName);
|
|
17715
17051
|
if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
|
|
17716
17052
|
else hold.holders += 1;
|
|
17717
17053
|
state.desyncedIndexes.add(indexName);
|
|
17718
|
-
const { releaseIndexCopies } = await import('./release-
|
|
17054
|
+
const { releaseIndexCopies } = await import('./release-HYXPVJVS.mjs');
|
|
17719
17055
|
return releaseIndexCopies(state, indexName, copyReleaseSteps);
|
|
17720
17056
|
},
|
|
17721
17057
|
async endCopyRelease(indexName, reloads) {
|
|
17722
|
-
const { endCopyRelease } = await import('./release-
|
|
17058
|
+
const { endCopyRelease } = await import('./release-HYXPVJVS.mjs');
|
|
17723
17059
|
await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
|
|
17724
17060
|
},
|
|
17725
17061
|
resyncIndex,
|
|
@@ -17768,12 +17104,13 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17768
17104
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
17769
17105
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
17770
17106
|
}
|
|
17771
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
17107
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-GVZEE46O.mjs');
|
|
17772
17108
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
17773
17109
|
const executor = createDirectExecutor({
|
|
17774
17110
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
17775
17111
|
vectorStorage: filesystemDurability ? "disk" : "memory",
|
|
17776
|
-
onVectorGraphRepaired: (indexName) => void durability?.manager.checkpointAfterInFlight(indexName)
|
|
17112
|
+
onVectorGraphRepaired: (indexName) => void durability?.manager.checkpointAfterInFlight(indexName),
|
|
17113
|
+
onDocumentWrite: (action) => orchestrator.recordMainWrite(action)
|
|
17777
17114
|
});
|
|
17778
17115
|
const pluginRegistry = createPluginRegistry();
|
|
17779
17116
|
if (config?.plugins) {
|
|
@@ -17912,7 +17249,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17912
17249
|
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
17913
17250
|
{ eventHandlers, mutationCtx },
|
|
17914
17251
|
indexName,
|
|
17915
|
-
|
|
17252
|
+
durabilityWiring?.reloadIndexAndCopies(orchestrator, durability, indexName) ?? Promise.resolve()
|
|
17916
17253
|
),
|
|
17917
17254
|
onError(error) {
|
|
17918
17255
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
@@ -17954,7 +17291,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17954
17291
|
requireManager,
|
|
17955
17292
|
resumeChange: async (indexName) => {
|
|
17956
17293
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
17957
|
-
await (await import('./analysis-change-entry-
|
|
17294
|
+
await (await import('./analysis-change-entry-5BFUVMHJ.mjs')).resumeChangeOf(engineCore, indexName);
|
|
17958
17295
|
},
|
|
17959
17296
|
onClose: hooks?.onIndexClose,
|
|
17960
17297
|
onOpen: (indexName) => {
|
|
@@ -18039,7 +17376,7 @@ async function startEngineCore(core, config) {
|
|
|
18039
17376
|
checkHeapAfterRecovery(core);
|
|
18040
17377
|
}
|
|
18041
17378
|
if (core.invalidation) await core.invalidation.start();
|
|
18042
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
17379
|
+
if (core.durability) await (await import('./analysis-change-entry-5BFUVMHJ.mjs')).resumeEveryChange(core);
|
|
18043
17380
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
18044
17381
|
} catch (err) {
|
|
18045
17382
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -18342,8 +17679,8 @@ async function announceIndexDropped(core, name, config) {
|
|
|
18342
17679
|
}
|
|
18343
17680
|
async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
|
|
18344
17681
|
const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
|
|
18345
|
-
const
|
|
18346
|
-
if (
|
|
17682
|
+
const loadingSchemaCode = loadSchemaCodeFor(requestedConfig.schema);
|
|
17683
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
18347
17684
|
core.guardShutdown();
|
|
18348
17685
|
validateIndexName(name);
|
|
18349
17686
|
if (core.indexRegistry.has(name)) {
|
|
@@ -18454,7 +17791,7 @@ async function dropEngineIndex(core, name) {
|
|
|
18454
17791
|
const entry = core.requireIndex(name);
|
|
18455
17792
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
18456
17793
|
const abandonChange = async () => {
|
|
18457
|
-
await (await import('./analysis-change-entry-
|
|
17794
|
+
await (await import('./analysis-change-entry-5BFUVMHJ.mjs')).abandonChangeOf(core, name);
|
|
18458
17795
|
};
|
|
18459
17796
|
if (changeRecorded()) await abandonChange();
|
|
18460
17797
|
await core.indexState.drop(name, async () => {
|
|
@@ -18521,7 +17858,7 @@ function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterNa
|
|
|
18521
17858
|
}
|
|
18522
17859
|
}
|
|
18523
17860
|
}
|
|
18524
|
-
function
|
|
17861
|
+
function validateVector(vector, targetField, expectedDimensions) {
|
|
18525
17862
|
if (!(vector instanceof Float32Array)) {
|
|
18526
17863
|
throw new NarsilError(
|
|
18527
17864
|
ErrorCodes.EMBEDDING_FAILED,
|
|
@@ -18634,7 +17971,7 @@ async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
|
18634
17971
|
let vector;
|
|
18635
17972
|
try {
|
|
18636
17973
|
const raw = await adapter.embed(text, "document", signal);
|
|
18637
|
-
vector =
|
|
17974
|
+
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
18638
17975
|
} catch (err) {
|
|
18639
17976
|
throw wrapAdapterError(err, targetField);
|
|
18640
17977
|
}
|
|
@@ -18705,7 +18042,7 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
18705
18042
|
}
|
|
18706
18043
|
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
18707
18044
|
const { docIndex } = needsEmbedding[j];
|
|
18708
|
-
const validated =
|
|
18045
|
+
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
18709
18046
|
setFieldValue(documents[docIndex], targetField, validated);
|
|
18710
18047
|
let fieldSet = embedded.get(docIndex);
|
|
18711
18048
|
if (!fieldSet) {
|
|
@@ -19010,14 +18347,8 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
19010
18347
|
} catch (err) {
|
|
19011
18348
|
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
19012
18349
|
}
|
|
19013
|
-
|
|
19014
|
-
|
|
19015
|
-
indexName,
|
|
19016
|
-
docId: resolvedDocId,
|
|
19017
|
-
document: partitionDoc,
|
|
19018
|
-
requestId: `replicate-insert-${resolvedDocId}`,
|
|
19019
|
-
skipClone: options?.skipClone
|
|
19020
|
-
});
|
|
18350
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
18351
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19021
18352
|
for (const fieldPath of extractedVectors.keys()) {
|
|
19022
18353
|
const vecIndex = insertVecIndexes.get(fieldPath);
|
|
19023
18354
|
if (vecIndex) {
|
|
@@ -19131,7 +18462,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
19131
18462
|
const entry = ctx.requireIndex(indexName);
|
|
19132
18463
|
const manager = ctx.requireManager(indexName);
|
|
19133
18464
|
const vecIndexes = manager.getVectorIndexes();
|
|
19134
|
-
const
|
|
18465
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
19135
18466
|
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
19136
18467
|
const strict = entry.config.strict === true;
|
|
19137
18468
|
const required = entry.config.required;
|
|
@@ -19166,7 +18497,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
19166
18497
|
}
|
|
19167
18498
|
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
19168
18499
|
documents[i],
|
|
19169
|
-
|
|
18500
|
+
vectorFieldPaths
|
|
19170
18501
|
);
|
|
19171
18502
|
if (extractedVectors.size > 0) {
|
|
19172
18503
|
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
@@ -19228,6 +18559,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
19228
18559
|
}
|
|
19229
18560
|
succeeded.push(doc.docId);
|
|
19230
18561
|
}
|
|
18562
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
18563
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19231
18564
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
19232
18565
|
if (applied < admitted.length) {
|
|
19233
18566
|
ctx.checkHeapPressure(indexName);
|
|
@@ -19272,75 +18605,34 @@ function buildSegmentRequests(indexName, docIds, documents, partitionCount, skip
|
|
|
19272
18605
|
}
|
|
19273
18606
|
return { requests, memberIndexes };
|
|
19274
18607
|
}
|
|
19275
|
-
function
|
|
19276
|
-
|
|
19277
|
-
|
|
19278
|
-
const snapshot = segment.snapshot ?? (segment.payload === null ? null : freezeSegmentShared(segment.payload, segment.documents, segment.segmentId));
|
|
19279
|
-
if (snapshot === null) return null;
|
|
19280
|
-
frozen.push({ partitionId: segment.partitionId, snapshot });
|
|
19281
|
-
}
|
|
19282
|
-
return frozen;
|
|
19283
|
-
}
|
|
19284
|
-
function tryFreezeSegmentsForAttach(segments) {
|
|
18608
|
+
function sharedSnapshotOf(segment) {
|
|
18609
|
+
if (segment.snapshot !== null) return segment.snapshot;
|
|
18610
|
+
if (segment.payload === null) return null;
|
|
19285
18611
|
try {
|
|
19286
|
-
return
|
|
18612
|
+
return freezeSegmentShared(segment.payload, segment.documents, segment.segmentId);
|
|
19287
18613
|
} catch (err) {
|
|
19288
18614
|
console.warn("Segment freeze failed, replicating by merge instead:", err);
|
|
19289
18615
|
return null;
|
|
19290
18616
|
}
|
|
19291
18617
|
}
|
|
19292
|
-
function
|
|
19293
|
-
const
|
|
19294
|
-
|
|
19295
|
-
|
|
19296
|
-
|
|
19297
|
-
|
|
19298
|
-
|
|
19299
|
-
|
|
19300
|
-
}
|
|
19301
|
-
function snapshotSegments(segments) {
|
|
19302
|
-
const shared = [];
|
|
19303
|
-
for (const segment of segments) {
|
|
19304
|
-
if (segment.snapshot !== null) shared.push({ partitionId: segment.partitionId, snapshot: segment.snapshot });
|
|
19305
|
-
}
|
|
19306
|
-
return shared;
|
|
19307
|
-
}
|
|
19308
|
-
function attachSegments2(orchestrator, indexName, segments) {
|
|
19309
|
-
return orchestrator.replicateToWorkers({
|
|
19310
|
-
type: "attachSegments",
|
|
19311
|
-
indexName,
|
|
19312
|
-
segments,
|
|
19313
|
-
requestId: `attach-segments-${indexName}-${segments.length}`
|
|
19314
|
-
});
|
|
19315
|
-
}
|
|
19316
|
-
async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
|
|
19317
|
-
const frozen = tryFreezeSegmentsForAttach(segments);
|
|
19318
|
-
if (frozen !== null) {
|
|
19319
|
-
await attachSegments2(orchestrator, indexName, frozen);
|
|
19320
|
-
return;
|
|
18618
|
+
function copyActionOfSegment(indexName, segment, skipClone) {
|
|
18619
|
+
const snapshot = sharedSnapshotOf(segment);
|
|
18620
|
+
if (snapshot !== null) {
|
|
18621
|
+
return {
|
|
18622
|
+
type: "attachSegments",
|
|
18623
|
+
indexName,
|
|
18624
|
+
segments: [{ partitionId: segment.partitionId, snapshot }],
|
|
18625
|
+
requestId: `attach-segments-${indexName}-${segment.segmentId}`
|
|
18626
|
+
};
|
|
19321
18627
|
}
|
|
19322
|
-
|
|
19323
|
-
|
|
19324
|
-
const plain = payloadSegments(segments);
|
|
19325
|
-
if (plain.length === 0) return;
|
|
19326
|
-
await orchestrator.replicateToWorkers({
|
|
18628
|
+
if (segment.payload === null) return null;
|
|
18629
|
+
return {
|
|
19327
18630
|
type: "mergeSegments",
|
|
19328
18631
|
indexName,
|
|
19329
|
-
segments:
|
|
19330
|
-
requestId: `merge-segments-${indexName}-${
|
|
18632
|
+
segments: [{ partitionId: segment.partitionId, payload: segment.payload, documents: segment.documents }],
|
|
18633
|
+
requestId: `merge-segments-${indexName}-${segment.segmentId}`,
|
|
19331
18634
|
skipClone: skipClone === true ? true : void 0
|
|
19332
|
-
}
|
|
19333
|
-
}
|
|
19334
|
-
async function replicateAsSegments(ctx, indexName, docIds, documents, skipClone) {
|
|
19335
|
-
if (docIds.length < MIN_DOCUMENTS_FOR_SEGMENTS) return false;
|
|
19336
|
-
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
19337
|
-
if (workers <= 0) return false;
|
|
19338
|
-
const manager = ctx.requireManager(indexName);
|
|
19339
|
-
const { requests } = buildSegmentRequests(indexName, docIds, documents, manager.partitionCount, skipClone);
|
|
19340
|
-
const built = await ctx.orchestrator.buildSegments(requests);
|
|
19341
|
-
if (built === null || built.length === 0) return false;
|
|
19342
|
-
await broadcastBuiltSegments(ctx.orchestrator, indexName, built, skipClone);
|
|
19343
|
-
return true;
|
|
18635
|
+
};
|
|
19344
18636
|
}
|
|
19345
18637
|
|
|
19346
18638
|
// src/engine/mutations/insert-batch-segments.ts
|
|
@@ -19348,26 +18640,11 @@ function mainStoreDocument(doc, options) {
|
|
|
19348
18640
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
19349
18641
|
return structuredClone(doc.partitionDoc);
|
|
19350
18642
|
}
|
|
19351
|
-
async function replicateDocuments(ctx, indexName, docs, options) {
|
|
19352
|
-
for (const doc of docs) {
|
|
19353
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
19354
|
-
type: "insert",
|
|
19355
|
-
indexName,
|
|
19356
|
-
docId: doc.docId,
|
|
19357
|
-
document: doc.partitionDoc,
|
|
19358
|
-
requestId: `replicate-insert-${doc.docId}`,
|
|
19359
|
-
skipClone: options?.skipClone
|
|
19360
|
-
});
|
|
19361
|
-
}
|
|
19362
|
-
}
|
|
19363
18643
|
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
19364
18644
|
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
19365
|
-
const succeededIds = new Set(applied.succeeded);
|
|
19366
|
-
const replicable = admitted.filter((doc) => succeededIds.has(doc.docId) && !applied.buffered.has(doc.docId));
|
|
19367
|
-
await replicateDocuments(ctx, indexName, replicable, options);
|
|
19368
18645
|
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
19369
18646
|
}
|
|
19370
|
-
function attachOnce(manager, attachment) {
|
|
18647
|
+
function attachOnce(ctx, manager, attachment) {
|
|
19371
18648
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
19372
18649
|
if (attachment.attached) return;
|
|
19373
18650
|
try {
|
|
@@ -19379,11 +18656,12 @@ function attachOnce(manager, attachment) {
|
|
|
19379
18656
|
attachment.failure = error;
|
|
19380
18657
|
throw error;
|
|
19381
18658
|
}
|
|
18659
|
+
if (attachment.copyAction !== null) ctx.orchestrator.recordMainWrite(attachment.copyAction);
|
|
19382
18660
|
}
|
|
19383
18661
|
function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
19384
18662
|
const manager = ctx.requireManager(indexName);
|
|
19385
18663
|
const attachToIndex = async () => {
|
|
19386
|
-
attachOnce(manager, attachment);
|
|
18664
|
+
attachOnce(ctx, manager, attachment);
|
|
19387
18665
|
progress.applied = true;
|
|
19388
18666
|
try {
|
|
19389
18667
|
insertDocumentVectors(
|
|
@@ -19407,13 +18685,14 @@ function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
|
19407
18685
|
};
|
|
19408
18686
|
return indexChangeOutsideExternalWaits(attachToIndex);
|
|
19409
18687
|
}
|
|
19410
|
-
function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
18688
|
+
function chunksOfWholeSegments(indexName, built, frozen, memberIndexes, admitted, options) {
|
|
19411
18689
|
const chunks = [];
|
|
19412
18690
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
19413
18691
|
for (let i = 0; i < built.length; i++) {
|
|
19414
18692
|
const attachment = {
|
|
19415
18693
|
partitionId: built[i].partitionId,
|
|
19416
18694
|
segments: [frozen[i]],
|
|
18695
|
+
copyAction: copyActionOfSegment(indexName, built[i], options?.skipClone),
|
|
19417
18696
|
attached: false,
|
|
19418
18697
|
failure: void 0
|
|
19419
18698
|
};
|
|
@@ -19430,12 +18709,12 @@ function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
|
19430
18709
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
19431
18710
|
return chunks;
|
|
19432
18711
|
}
|
|
19433
|
-
async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, failed) {
|
|
18712
|
+
async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, options, failed) {
|
|
19434
18713
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
19435
18714
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
19436
18715
|
const errorOf = /* @__PURE__ */ new Map();
|
|
19437
18716
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
19438
|
-
const chunks = chunksOfWholeSegments(built, frozen, memberIndexes, admitted);
|
|
18717
|
+
const chunks = chunksOfWholeSegments(indexName, built, frozen, memberIndexes, admitted, options);
|
|
19439
18718
|
for (let index = 0; index < chunks.length; index++) {
|
|
19440
18719
|
const { docs, attachmentOf } = chunks[index];
|
|
19441
18720
|
const progress = docs.map(() => ({ applied: false }));
|
|
@@ -19470,40 +18749,23 @@ async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexe
|
|
|
19470
18749
|
}
|
|
19471
18750
|
succeededIds.add(doc.docId);
|
|
19472
18751
|
}
|
|
18752
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName, "segments");
|
|
18753
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19473
18754
|
if (index + 1 < chunks.length) {
|
|
19474
18755
|
ctx.checkHeapPressure(indexName);
|
|
19475
18756
|
await new Promise((r) => setTimeout(r, 0));
|
|
19476
18757
|
}
|
|
19477
18758
|
}
|
|
19478
18759
|
const succeeded = [];
|
|
19479
|
-
const failedDocIds = /* @__PURE__ */ new Set();
|
|
19480
18760
|
for (const doc of admitted) {
|
|
19481
18761
|
if (succeededIds.has(doc.docId)) {
|
|
19482
18762
|
succeeded.push(doc.docId);
|
|
19483
18763
|
continue;
|
|
19484
18764
|
}
|
|
19485
18765
|
if (!errorOf.has(doc.docId)) continue;
|
|
19486
|
-
failedDocIds.add(doc.docId);
|
|
19487
18766
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
19488
18767
|
}
|
|
19489
|
-
return { succeeded, touchedVectorFields
|
|
19490
|
-
}
|
|
19491
|
-
async function broadcastSegments(ctx, indexName, built, memberIndexes, admitted, failedDocIds, options) {
|
|
19492
|
-
const clean = [];
|
|
19493
|
-
const retryDocs = [];
|
|
19494
|
-
for (let i = 0; i < built.length; i++) {
|
|
19495
|
-
const members = memberIndexes[i].map((m) => admitted[m]);
|
|
19496
|
-
if (members.some((doc) => failedDocIds.has(doc.docId))) {
|
|
19497
|
-
retryDocs.push(...members.filter((doc) => !failedDocIds.has(doc.docId)));
|
|
19498
|
-
continue;
|
|
19499
|
-
}
|
|
19500
|
-
clean.push(built[i]);
|
|
19501
|
-
}
|
|
19502
|
-
if (clean.length > 0) {
|
|
19503
|
-
await broadcastBuiltSegments(ctx.orchestrator, indexName, clean, options?.skipClone);
|
|
19504
|
-
}
|
|
19505
|
-
await replicateDocuments(ctx, indexName, retryDocs, options);
|
|
19506
|
-
return clean.map((segment) => segment.segmentId);
|
|
18768
|
+
return { succeeded, touchedVectorFields };
|
|
19507
18769
|
}
|
|
19508
18770
|
function frozenSegmentOf(segment, members, options) {
|
|
19509
18771
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -19557,20 +18819,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
19557
18819
|
for (const doc of admitted) {
|
|
19558
18820
|
if (manager.has(doc.docId)) return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
19559
18821
|
}
|
|
19560
|
-
|
|
19561
|
-
ctx.orchestrator.holdUnbroadcastSegments(indexName, segmentIds);
|
|
19562
|
-
const recorded = await recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, failed);
|
|
19563
|
-
const broadcast = await broadcastSegments(
|
|
19564
|
-
ctx,
|
|
19565
|
-
indexName,
|
|
19566
|
-
built,
|
|
19567
|
-
memberIndexes,
|
|
19568
|
-
admitted,
|
|
19569
|
-
recorded.failedDocIds,
|
|
19570
|
-
options
|
|
19571
|
-
);
|
|
19572
|
-
ctx.orchestrator.releaseUnbroadcastSegments(indexName, broadcast);
|
|
19573
|
-
return { succeeded: recorded.succeeded, touchedVectorFields: recorded.touchedVectorFields };
|
|
18822
|
+
return recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, options, failed);
|
|
19574
18823
|
}
|
|
19575
18824
|
async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
19576
18825
|
const failed = [];
|
|
@@ -19598,7 +18847,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19598
18847
|
return insertBatchViaSegments(ctx, indexName, documents, options);
|
|
19599
18848
|
}
|
|
19600
18849
|
const succeeded = [];
|
|
19601
|
-
const succeededDocs = [];
|
|
19602
18850
|
const failed = [];
|
|
19603
18851
|
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
19604
18852
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
@@ -19607,7 +18855,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19607
18855
|
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
19608
18856
|
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
19609
18857
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
19610
|
-
const bufferedDocIds = /* @__PURE__ */ new Set();
|
|
19611
18858
|
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
19612
18859
|
if (ctx.abortController.signal.aborted) break;
|
|
19613
18860
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
@@ -19635,9 +18882,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19635
18882
|
continue;
|
|
19636
18883
|
}
|
|
19637
18884
|
if (application.status === "buffered") {
|
|
19638
|
-
bufferedDocIds.add(doc.docId);
|
|
19639
18885
|
succeeded.push(doc.docId);
|
|
19640
|
-
succeededDocs.push(doc.partitionDoc);
|
|
19641
18886
|
continue;
|
|
19642
18887
|
}
|
|
19643
18888
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
@@ -19651,7 +18896,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19651
18896
|
}
|
|
19652
18897
|
}
|
|
19653
18898
|
succeeded.push(doc.docId);
|
|
19654
|
-
succeededDocs.push(doc.partitionDoc);
|
|
19655
18899
|
}
|
|
19656
18900
|
}
|
|
19657
18901
|
const prepared = [];
|
|
@@ -19685,32 +18929,13 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19685
18929
|
}
|
|
19686
18930
|
}
|
|
19687
18931
|
if (prepared.length > 0) await applyPrepared(prepared);
|
|
18932
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
18933
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19688
18934
|
if (chunkEnd < documents.length) {
|
|
19689
18935
|
ctx.checkHeapPressure(indexName);
|
|
19690
18936
|
await new Promise((r) => setTimeout(r, 0));
|
|
19691
18937
|
}
|
|
19692
18938
|
}
|
|
19693
|
-
const replicableIds = [];
|
|
19694
|
-
const replicableDocs = [];
|
|
19695
|
-
for (let i = 0; i < succeeded.length; i++) {
|
|
19696
|
-
if (bufferedDocIds.has(succeeded[i])) continue;
|
|
19697
|
-
replicableIds.push(succeeded[i]);
|
|
19698
|
-
replicableDocs.push(succeededDocs[i]);
|
|
19699
|
-
}
|
|
19700
|
-
const replicatedAsSegments = copiesBuildSegments && await replicateAsSegments(ctx, indexName, replicableIds, replicableDocs, options?.skipClone);
|
|
19701
|
-
if (!replicatedAsSegments) {
|
|
19702
|
-
for (let i = 0; i < replicableIds.length; i++) {
|
|
19703
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
19704
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
19705
|
-
type: "insert",
|
|
19706
|
-
indexName,
|
|
19707
|
-
docId: replicableIds[i],
|
|
19708
|
-
document: replicableDocs[i],
|
|
19709
|
-
requestId: `replicate-insert-${replicableIds[i]}`,
|
|
19710
|
-
skipClone: options?.skipClone
|
|
19711
|
-
});
|
|
19712
|
-
}
|
|
19713
|
-
}
|
|
19714
18939
|
for (const fieldPath of touchedVectorFields) {
|
|
19715
18940
|
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
19716
18941
|
if (vecIndex) {
|
|
@@ -19766,19 +18991,14 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
19766
18991
|
return indexChangeOutsideExternalWaits(removeFromIndex);
|
|
19767
18992
|
}
|
|
19768
18993
|
async function settleRemoval(ctx, indexName, prepared) {
|
|
19769
|
-
const { docId
|
|
18994
|
+
const { docId } = prepared;
|
|
19770
18995
|
try {
|
|
19771
18996
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
19772
18997
|
} catch (err) {
|
|
19773
18998
|
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
19774
18999
|
}
|
|
19775
|
-
|
|
19776
|
-
|
|
19777
|
-
indexName,
|
|
19778
|
-
docId,
|
|
19779
|
-
requestId: `replicate-remove-${docId}`,
|
|
19780
|
-
removedSpellings
|
|
19781
|
-
});
|
|
19000
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName, "none");
|
|
19001
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19782
19002
|
}
|
|
19783
19003
|
async function removeDocument(ctx, indexName, docId, options) {
|
|
19784
19004
|
ctx.guardShutdown();
|
|
@@ -19876,14 +19096,8 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
19876
19096
|
} catch (err) {
|
|
19877
19097
|
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
19878
19098
|
}
|
|
19879
|
-
|
|
19880
|
-
|
|
19881
|
-
indexName,
|
|
19882
|
-
docId: prepared.docId,
|
|
19883
|
-
document: prepared.partitionDoc,
|
|
19884
|
-
requestId: `replicate-update-${prepared.docId}`,
|
|
19885
|
-
removedSpellings: prepared.removedSpellings
|
|
19886
|
-
});
|
|
19099
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
19100
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19887
19101
|
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
19888
19102
|
for (const [fieldPath, vec] of prepared.extractedVectors) {
|
|
19889
19103
|
if (vec === null) continue;
|
|
@@ -20104,7 +19318,6 @@ async function withRenewedSyncEpoch(core, indexName, change) {
|
|
|
20104
19318
|
function clearEngineIndex(core, indexName) {
|
|
20105
19319
|
return withRenewedSyncEpoch(core, indexName, async () => {
|
|
20106
19320
|
await core.executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
20107
|
-
await core.orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
20108
19321
|
core.watermarkNotifier.forget(indexName);
|
|
20109
19322
|
const durable = core.durability?.manager;
|
|
20110
19323
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
@@ -20201,7 +19414,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20201
19414
|
const { restoresInFlight } = core;
|
|
20202
19415
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20203
19416
|
const restored = previous.then(async () => {
|
|
20204
|
-
const { runRestore } = await import('./restore-queue-
|
|
19417
|
+
const { runRestore } = await import('./restore-queue-LQBBTODR.mjs');
|
|
20205
19418
|
await runRestore(core, restoreHost, indexName, data, engine());
|
|
20206
19419
|
});
|
|
20207
19420
|
const settled2 = restored.then(
|
|
@@ -20232,7 +19445,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20232
19445
|
return {
|
|
20233
19446
|
snapshot(indexName, options) {
|
|
20234
19447
|
return withOpenIndex2(indexName, async () => {
|
|
20235
|
-
const { createSnapshot } = await import('./snapshot-
|
|
19448
|
+
const { createSnapshot } = await import('./snapshot-5FHPPCNZ.mjs');
|
|
20236
19449
|
return createSnapshot(snapshotDepsOf(indexName), options?.queryHistory === true ? core : null);
|
|
20237
19450
|
});
|
|
20238
19451
|
},
|
|
@@ -20245,8 +19458,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
20245
19458
|
exportDocuments(indexName) {
|
|
20246
19459
|
return withOpenIndex2(indexName, async () => {
|
|
20247
19460
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
20248
|
-
import('./snapshot-
|
|
20249
|
-
import('./write-
|
|
19461
|
+
import('./snapshot-5FHPPCNZ.mjs'),
|
|
19462
|
+
import('./write-ULXAN5B4.mjs')
|
|
20250
19463
|
]);
|
|
20251
19464
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
20252
19465
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -20254,7 +19467,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
20254
19467
|
},
|
|
20255
19468
|
importDocumentExport(indexName, data) {
|
|
20256
19469
|
core.guardShutdown();
|
|
20257
|
-
const imported = import('./import-
|
|
19470
|
+
const imported = import('./import-5ESVMFDJ.mjs').then(
|
|
20258
19471
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
20259
19472
|
);
|
|
20260
19473
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -22275,7 +21488,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22275
21488
|
const entry = requireIndex(indexName);
|
|
22276
21489
|
await withOpenIndexWrite(indexName, async () => {
|
|
22277
21490
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
22278
|
-
return (await import('./analysis-change-
|
|
21491
|
+
return (await import('./analysis-change-ZKGJFENM.mjs')).resumeAnalysisChange(core, indexName);
|
|
22279
21492
|
});
|
|
22280
21493
|
},
|
|
22281
21494
|
async changeAnalysis(indexName, change) {
|
|
@@ -22283,7 +21496,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22283
21496
|
requireIndex(indexName);
|
|
22284
21497
|
const changed = withOpenIndex2(
|
|
22285
21498
|
indexName,
|
|
22286
|
-
async () => (await import('./analysis-change-
|
|
21499
|
+
async () => (await import('./analysis-change-ZKGJFENM.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
22287
21500
|
);
|
|
22288
21501
|
await emitWriteEventAfter(core, indexName, changed);
|
|
22289
21502
|
},
|
|
@@ -22318,7 +21531,7 @@ function createNarsilFromCore(core, config) {
|
|
|
22318
21531
|
const rebalanced = withOpenIndex2(
|
|
22319
21532
|
indexName,
|
|
22320
21533
|
() => withRenewedSyncEpoch(core, indexName, async () => {
|
|
22321
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
21534
|
+
const { executeRebalance } = await import('./analysis-change-ZKGJFENM.mjs');
|
|
22322
21535
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
22323
21536
|
releaseWrites();
|
|
22324
21537
|
await running;
|
|
@@ -22403,8 +21616,8 @@ function createNarsilFromCore(core, config) {
|
|
|
22403
21616
|
guardShutdown();
|
|
22404
21617
|
requireIndex(indexName);
|
|
22405
21618
|
const manager = executor.getManager(indexName);
|
|
22406
|
-
const
|
|
22407
|
-
return manager === void 0 ||
|
|
21619
|
+
const vectorCode2 = loadedVectorCode();
|
|
21620
|
+
return manager === void 0 || vectorCode2 === null ? [] : vectorCode2.getVectorMaintenanceStatus(manager);
|
|
22408
21621
|
},
|
|
22409
21622
|
on(event, handler) {
|
|
22410
21623
|
const key = event;
|
|
@@ -22442,7 +21655,7 @@ async function createNarsil(config) {
|
|
|
22442
21655
|
await startEngineCore(core, runnableConfig);
|
|
22443
21656
|
const engine = createNarsilFromCore(core, runnableConfig);
|
|
22444
21657
|
if (runnableConfig?.querySuggestions !== void 0 || core.durability !== null) {
|
|
22445
|
-
await (await import('./engine-hooks-
|
|
21658
|
+
await (await import('./engine-hooks-HBVS6YM4.mjs')).startQuerySuggestions(core, engine, runnableConfig);
|
|
22446
21659
|
}
|
|
22447
21660
|
return engine;
|
|
22448
21661
|
}
|
|
@@ -22450,4 +21663,4 @@ async function createNarsil(config) {
|
|
|
22450
21663
|
// src/index.ts
|
|
22451
21664
|
var VERSION = "0.1.0";
|
|
22452
21665
|
|
|
22453
|
-
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, REBALANCE_CHUNK_SIZE, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, dropEngineIndex, emitEngineEvent, emitWriteEventAfter, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineCoreOf, engineVersionParts, extractVectorFromDoc, fieldIndexRemapOf, freezeSegmentShared, generateId,
|
|
21666
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, REBALANCE_CHUNK_SIZE, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, dropEngineIndex, emitEngineEvent, emitWriteEventAfter, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineCoreOf, engineVersionParts, extractVectorFromDoc, fieldIndexRemapOf, freezeSegmentShared, generateId, getNormalizationCacheSize, holdSyncEpochChange, holdsADiscardedField, insertDocumentVectors, isCompositePartition, loadSchemaCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, packChecksummedEnvelope, packEnvelopeWithChecksum, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerPartialWordSearch, registerPatternSearch, registerSynonyms, removeDocumentBatch, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, resetNormalizationCache, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, vectorsEqual, wrapFrozenTokenTable, yieldToEventLoop2 as yieldToEventLoop };
|