@delali/narsil 0.3.1-next.155 → 0.3.1-next.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/dist/{analysis-change-3TFIHT5E.mjs → analysis-change-CKEMM2MC.mjs} +25 -18
- package/dist/{react-native/chunks/analysis-change-entry-JLL42PH6.mjs → analysis-change-entry-SEDMJ6MW.mjs} +1 -1
- package/dist/browser/{analysis-change-TC36BU47.mjs → analysis-change-ZKGJFENM.mjs} +42 -32
- package/dist/{analysis-change-entry-KA6JAO4X.mjs → browser/analysis-change-entry-5BFUVMHJ.mjs} +1 -1
- package/dist/browser/{chunk-KKAMCABR.mjs → chunk-25MTVDYD.mjs} +1 -1
- package/dist/browser/{chunk-K5AXJBLL.mjs → chunk-3NLMB5CC.mjs} +3 -3
- package/dist/browser/{chunk-23XRXIBW.mjs → chunk-5YWB3E2W.mjs} +8 -1
- package/dist/browser/chunk-AI5UI46D.mjs +12 -0
- package/dist/browser/{chunk-A5KUEGDY.mjs → chunk-BLT4RHCJ.mjs} +3 -3
- package/dist/browser/{chunk-RRO3EV4C.mjs → chunk-C26SPCGQ.mjs} +8 -7
- package/dist/browser/{chunk-ABCXENU5.mjs → chunk-C4FJ3WDY.mjs} +1 -1
- package/dist/browser/{chunk-3UIEB4QF.mjs → chunk-CXNTHHA7.mjs} +2 -2
- package/dist/browser/{chunk-JIJXMI42.mjs → chunk-DUH6QUVU.mjs} +547 -2
- package/dist/browser/chunk-EKMGCRSR.mjs +61 -0
- package/dist/browser/{chunk-JVBP5EYO.mjs → chunk-GFAZPSWK.mjs} +7 -7
- package/dist/browser/{chunk-CJSYBQ4J.mjs → chunk-HUCSHDTB.mjs} +1 -1
- package/dist/browser/{chunk-RND5CLX3.mjs → chunk-K3IPRX6D.mjs} +52 -21
- package/dist/browser/chunk-MY2XKH22.mjs +40 -0
- package/dist/browser/{chunk-SMEQ36NT.mjs → chunk-N23GL72Q.mjs} +629 -1416
- package/dist/browser/{chunk-LFIWRCEA.mjs → chunk-NXQG7DRA.mjs} +29 -6
- package/dist/browser/{chunk-26A5JFT2.mjs → chunk-TK63WFXK.mjs} +2 -1
- package/dist/browser/{chunk-4EWUGZYZ.mjs → chunk-U4YDIBUT.mjs} +2 -2
- package/dist/browser/{chunk-RQBUNZQW.mjs → chunk-UATIJ4KU.mjs} +2 -2
- package/dist/browser/{chunk-BXM5XOCA.mjs → chunk-UKOWD2K5.mjs} +3 -3
- package/dist/browser/{chunk-FNIFPFB7.mjs → chunk-UTMHUHUV.mjs} +2 -2
- package/dist/browser/{chunk-56W3HQVJ.mjs → chunk-V43N6TZO.mjs} +5 -5
- package/dist/browser/{chunk-K4GCSZLI.mjs → chunk-VLJQW4J6.mjs} +2985 -2985
- package/dist/browser/{chunk-IKDKX5VQ.mjs → chunk-VY5KW3HN.mjs} +3 -3
- package/dist/browser/{chunk-5TCXK5SB.mjs → chunk-WN464PTL.mjs} +1 -1
- package/dist/browser/{chunk-SUDNZKPL.mjs → chunk-X7TXYUKO.mjs} +1 -1
- package/dist/browser/{chunk-XY4VVBBO.mjs → chunk-Y6IBAUNO.mjs} +2 -86
- package/dist/browser/{cold-module-NEUGHUPA.mjs → cold-module-YNUUDNOM.mjs} +18 -15
- package/dist/browser/{durability-wiring-CVGPRO6D.mjs → durability-wiring-GVZEE46O.mjs} +71 -27
- package/dist/browser/{engine-hooks-YXMPHZKZ.mjs → engine-hooks-HBVS6YM4.mjs} +8 -6
- package/dist/browser/geo-index-OTV7VPIR.mjs +1387 -0
- package/dist/browser/{import-OM6QAW45.mjs → import-5ESVMFDJ.mjs} +9 -7
- package/dist/browser/{module-E33X4AOK.mjs → module-NDSQLPMQ.mjs} +14 -11
- package/dist/browser/{module-3HNFCBJ5.mjs → module-TCT66FWA.mjs} +6 -6
- package/dist/browser/{phrase-module-ZXXU7S3B.mjs → phrase-module-HAT3TLZN.mjs} +1 -1
- package/dist/{release-BH3DKW3T.mjs → browser/release-HYXPVJVS.mjs} +0 -1
- package/dist/browser/{restore-hooks-IF2ZROUP.mjs → restore-hooks-IE3GGYHG.mjs} +11 -9
- package/dist/browser/{restore-queue-ZBS7CJA6.mjs → restore-queue-LQBBTODR.mjs} +3 -3
- package/dist/browser/{retention-ASWUHAYI.mjs → retention-TY3L3BFN.mjs} +11 -9
- package/dist/browser/{snapshot-PN7ZW6Q4.mjs → snapshot-5FHPPCNZ.mjs} +27 -24
- package/dist/browser/{start-ZIRB7MQ2.mjs → start-2UZTOBG5.mjs} +12 -10
- package/dist/browser/{write-HMDYCTV2.mjs → write-ULXAN5B4.mjs} +14 -11
- package/dist/{capabilities-Da4ywVVl.d.ts → capabilities-D3AoBJoF.d.ts} +2 -2
- package/dist/{chunk-KBKEXQZA.mjs → chunk-24FNVRKB.mjs} +1 -1
- package/dist/{chunk-VHBNF4MA.mjs → chunk-2ROUFMQX.mjs} +2 -2
- package/dist/{chunk-AILVAOLX.mjs → chunk-4B7JIGQD.mjs} +2 -2
- package/dist/{chunk-DCNKVEDB.mjs → chunk-4LMSZAK6.mjs} +16 -6
- package/dist/{chunk-RSVULW4Q.mjs → chunk-4ZTHGTVX.mjs} +4 -4
- package/dist/{chunk-VQEUR6T6.mjs → chunk-5VV4R6P7.mjs} +3040 -3040
- package/dist/chunk-6HTGOXIK.mjs +875 -0
- package/dist/chunk-7GFE3PEY.mjs +40 -0
- package/dist/{chunk-JUKYYXSP.mjs → chunk-7V5IUP2T.mjs} +1 -31
- package/dist/{chunk-4BYWRYPM.mjs → chunk-A4L7YTKX.mjs} +52 -89
- package/dist/{chunk-J4IWBIVY.mjs → chunk-BNL5OPRV.mjs} +6 -6
- package/dist/{chunk-BY6ZDGYS.mjs → chunk-C7XQYG6U.mjs} +2 -2
- package/dist/{chunk-LC3FDWTX.mjs → chunk-CITFCZKF.mjs} +3 -2
- package/dist/{chunk-KXCVQWOR.mjs → chunk-CQZXZ75V.mjs} +3 -97
- package/dist/{chunk-SY3EWKQY.mjs → chunk-CWS55TEX.mjs} +2 -2
- package/dist/{chunk-3HHQLEEQ.mjs → chunk-EYD7K3H6.mjs} +7 -7
- package/dist/chunk-FB6463IN.mjs +1499 -0
- package/dist/{chunk-KFLK5NA2.mjs → chunk-FCJ553HS.mjs} +1 -1
- package/dist/{chunk-KMUNHGBQ.mjs → chunk-FEZVDOWX.mjs} +169 -253
- package/dist/{chunk-SG3JBNHY.mjs → chunk-FJKKCV4D.mjs} +1 -1
- package/dist/{chunk-BDRIVYOK.mjs → chunk-FUD5XSZ5.mjs} +1 -1
- package/dist/chunk-G6HRSGOU.mjs +74 -0
- package/dist/{chunk-S2EQU2GE.mjs → chunk-GPWMUPCA.mjs} +10 -10
- package/dist/{chunk-ZHCUZZE5.mjs → chunk-HHBUCIUY.mjs} +1 -1
- package/dist/{chunk-YEDIDCR7.mjs → chunk-I5B7RXU7.mjs} +91 -22
- package/dist/{chunk-2MANFZJ3.mjs → chunk-IIRIG7H5.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-KWRW5U65.mjs → chunk-J4IJTTEX.mjs} +1 -1
- package/dist/{chunk-Q622CTMC.mjs → chunk-JSV6OQ4C.mjs} +1 -1
- package/dist/{chunk-NF4DT3IH.mjs → chunk-JTYPJF3T.mjs} +8 -1
- package/dist/chunk-KCQYFO4K.mjs +62 -0
- package/dist/{chunk-Q7G3BV3Y.mjs → chunk-KKQC526K.mjs} +19 -11
- package/dist/{chunk-PCRA7HTI.mjs → chunk-LDNCLRZU.mjs} +1 -1
- package/dist/{chunk-U4BS74YN.mjs → chunk-MW6BR4QP.mjs} +2 -2
- package/dist/{chunk-DIBHI6W5.mjs → chunk-MWD5VPN7.mjs} +2 -2
- package/dist/{chunk-UZRZKNNZ.mjs → chunk-ODYKO5TO.mjs} +1 -1
- package/dist/{chunk-PHQNACDT.mjs → chunk-OG7UGJ6F.mjs} +2 -2
- package/dist/{chunk-QH3YIQP4.mjs → chunk-OQLD2BSW.mjs} +2 -2
- package/dist/chunk-ORN5BLL7.mjs +11 -0
- package/dist/{chunk-L7ZD4L3K.mjs → chunk-OYABFKTW.mjs} +7 -6
- package/dist/{chunk-KBDBPVLN.mjs → chunk-P5LANSH6.mjs} +1 -1
- package/dist/{chunk-ZAUJFLEN.mjs → chunk-PIH5SJ4N.mjs} +122 -1697
- package/dist/{chunk-DUOLGBTE.mjs → chunk-PSZI4O24.mjs} +51 -4
- package/dist/{chunk-4KRC2APY.mjs → chunk-PV2CG7EJ.mjs} +5 -567
- package/dist/{chunk-6L3XJXGU.mjs → chunk-PWGYQXML.mjs} +1 -1
- package/dist/{chunk-5FVU47TB.mjs → chunk-QLDABJZM.mjs} +3 -3
- package/dist/{chunk-KPIV4RUC.mjs → chunk-RCX4WNLC.mjs} +2 -2
- package/dist/{chunk-3HB3PEC6.mjs → chunk-RIHIVFIQ.mjs} +6 -6
- package/dist/chunk-RLOES5SX.mjs +31 -0
- package/dist/{chunk-H6VIGE62.mjs → chunk-RWURUDSP.mjs} +10 -9
- package/dist/{chunk-ZTW5WTGG.mjs → chunk-SMOOSIAW.mjs} +4 -4
- package/dist/{chunk-MT2NRTA6.mjs → chunk-SOI6LTHM.mjs} +3 -3
- package/dist/chunk-SQJWAIF4.mjs +12 -0
- package/dist/{react-native/chunks/chunk-PPEPTUVV.mjs → chunk-TBC3XUL2.mjs} +29 -6
- package/dist/chunk-TDIK6NTV.mjs +10 -0
- package/dist/{react-native/chunks/chunk-JV6VQY2S.mjs → chunk-U55XJEIG.mjs} +2 -1
- package/dist/{chunk-H4KHPLHJ.mjs → chunk-UD5HIS4U.mjs} +1 -1
- package/dist/{chunk-HW5TNXBO.mjs → chunk-UJDJB3NE.mjs} +40 -31
- package/dist/{chunk-QUJHMPTP.mjs → chunk-VSLSGASN.mjs} +2 -2
- package/dist/{chunk-Z2BE2MAM.mjs → chunk-VZ2ONHEE.mjs} +32 -15
- package/dist/chunk-W5EB5XKI.mjs +741 -0
- package/dist/{chunk-3R4UZWQG.mjs → chunk-W7OSSKH5.mjs} +50 -20
- package/dist/{chunk-SMVH6JBE.mjs → chunk-XXJZMTZT.mjs} +1 -1
- package/dist/{chunk-VGF7CFRP.mjs → chunk-ZPZ3YGXV.mjs} +5 -4
- package/dist/{client-DRv7UVeY.d.ts → client-kqSMevVi.d.ts} +15 -3
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +4 -0
- package/dist/{cold-module-7XXCEGTZ.mjs → cold-module-VOJXMNZB.mjs} +31 -24
- package/dist/{config-DMwffLKn.d.ts → config-B4v_Htpu.d.ts} +1 -1
- package/dist/{config-CoSig5Ox.d.ts → config-CTP01fns.d.ts} +1 -1
- package/dist/direct-executor-V5DRIECI.mjs +34 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +94 -80
- package/dist/{durability-wiring-NCKUC5EO.mjs → durability-wiring-3TD62MBI.mjs} +57 -42
- package/dist/{engine-CFxPYqEk.d.ts → engine-CkPn2PWE.d.ts} +1 -1
- package/dist/{engine-hooks-DHCMOADY.mjs → engine-hooks-HA3SBOBR.mjs} +24 -17
- package/dist/geo-index-MRH5JMIR.mjs +1387 -0
- package/dist/{import-NBFXFCSG.mjs → import-GPKLTSKA.mjs} +24 -17
- package/dist/index.browser.mjs +7 -5
- package/dist/index.d.ts +131 -6
- package/dist/index.mjs +23 -16
- package/dist/index.react-native.mjs +12 -10
- package/dist/invalidation/filesystem.mjs +2 -2
- package/dist/{long-query-thread-J4DVRM4A.mjs → long-query-thread-QDFQSJD6.mjs} +26 -21
- package/dist/{module-DWKPBSMN.mjs → module-JYHFTVPS.mjs} +28 -20
- package/dist/{module-UN76TF7I.mjs → module-N55W4Q5C.mjs} +9 -9
- package/dist/{paged-partition-archive-DPNWVSK2.mjs → paged-partition-archive-RLRGOXDX.mjs} +33 -27
- package/dist/partial-words.mjs +8 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +32 -22
- package/dist/{phrase-module-ZI4WSQVH.mjs → phrase-module-XW5MKWP5.mjs} +3 -2
- package/dist/query-suggestions/merge-worker.mjs +25 -18
- package/dist/query-suggestions.browser.mjs +13 -11
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +34 -27
- package/dist/react-native/chunks/{analysis-change-6BPDPH6K.mjs → analysis-change-QUVDNABS.mjs} +46 -36
- package/dist/{browser/analysis-change-entry-4MH3CZXH.mjs → react-native/chunks/analysis-change-entry-OC4XASBF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-7PXRWYUU.mjs → chunk-2PRHILX3.mjs} +53 -19
- package/dist/{chunk-R6A773WK.mjs → react-native/chunks/chunk-3WI35N75.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GIZTJXYW.mjs → chunk-4AV5PPH6.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-4Y53T4YE.mjs → chunk-4COWCEY7.mjs} +189 -178
- package/dist/react-native/chunks/{chunk-OL3WDV53.mjs → chunk-4NWR3NGS.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-C3PTWYKT.mjs → chunk-5MSX5FRT.mjs} +5 -5
- package/dist/react-native/chunks/{chunk-6WA3C3DV.mjs → chunk-6OFDUYVX.mjs} +8 -1
- package/dist/react-native/chunks/{chunk-4LCEN2IU.mjs → chunk-6ZELFCK5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-CUH5F57W.mjs → chunk-BITYFZSZ.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TYIRMPD3.mjs → chunk-DMIFXZ3Q.mjs} +1 -1
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +12 -0
- package/dist/{chunk-YTJCGKM4.mjs → react-native/chunks/chunk-EJ33LCHV.mjs} +29 -6
- package/dist/react-native/chunks/{chunk-EIOYM5TO.mjs → chunk-ENLMYKSX.mjs} +3024 -3054
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +40 -0
- package/dist/{chunk-4UMZ7KQL.mjs → react-native/chunks/chunk-FTZHMRD2.mjs} +2 -1
- package/dist/react-native/chunks/{chunk-ALOIBXWN.mjs → chunk-IAENKE2D.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-QJXTNPLQ.mjs → chunk-KSBYEMCB.mjs} +1 -1
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +164 -0
- package/dist/react-native/chunks/{chunk-HUOYZJ3Q.mjs → chunk-OCSLSH27.mjs} +1561 -933
- package/dist/react-native/chunks/chunk-OXM66LO6.mjs +62 -0
- package/dist/react-native/chunks/{chunk-KTTAU63H.mjs → chunk-SK2PWYDH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-PEHS2UO3.mjs → chunk-TORU2OHQ.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-VXGE4ZXL.mjs → chunk-WVWJNUMN.mjs} +51 -21
- package/dist/react-native/chunks/{chunk-KJIG57UD.mjs → chunk-WW2X2AEI.mjs} +7 -7
- package/dist/react-native/chunks/{chunk-JLIIWJUV.mjs → chunk-YZW4CHM6.mjs} +50 -11
- package/dist/react-native/chunks/{chunk-GJFXPHXI.mjs → chunk-ZNZ6CPAZ.mjs} +2 -87
- package/dist/react-native/chunks/{chunk-7GRCBBME.mjs → chunk-ZVQMBF2H.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-N2WUOWLL.mjs → chunk-ZWAIN4GG.mjs} +7 -7
- package/dist/react-native/chunks/{cold-module-5PHF6DZR.mjs → cold-module-N6KFNNZA.mjs} +20 -17
- package/dist/react-native/chunks/{durability-wiring-KO565FZC.mjs → durability-wiring-RN64S2TL.mjs} +76 -32
- package/dist/react-native/chunks/{engine-hooks-MVRRHH6B.mjs → engine-hooks-CWA5CKCA.mjs} +13 -11
- package/dist/react-native/chunks/geo-index-PX3BIHAH.mjs +1387 -0
- package/dist/react-native/chunks/{import-YVZA4V4V.mjs → import-5UAWBRJP.mjs} +14 -12
- package/dist/react-native/chunks/{module-ZL5ZG6YB.mjs → module-6RP4ZCBW.mjs} +17 -14
- package/dist/react-native/chunks/{module-H5MVPSP2.mjs → module-TTEVPRCW.mjs} +8 -8
- package/dist/react-native/chunks/{phrase-module-RTVKEWJT.mjs → phrase-module-QK22HSQ2.mjs} +1 -1
- package/dist/react-native/chunks/{release-D7ZTIANM.mjs → release-2SIHZWDO.mjs} +0 -1
- package/dist/react-native/chunks/{restore-hooks-SILFPU6B.mjs → restore-hooks-ENFMDTWC.mjs} +17 -15
- package/dist/react-native/chunks/{restore-queue-HSYP2AEC.mjs → restore-queue-4R6AASE5.mjs} +3 -3
- package/dist/react-native/chunks/{retention-7I7UAWYK.mjs → retention-5GTFT3GR.mjs} +18 -16
- package/dist/react-native/chunks/{snapshot-EZYB6SLF.mjs → snapshot-5LGPQXFS.mjs} +32 -29
- package/dist/react-native/chunks/{start-SHVZ72I5.mjs → start-ZQJVZ5DN.mjs} +18 -16
- package/dist/react-native/chunks/{write-JUESXHDL.mjs → write-6BRYIIZ7.mjs} +17 -14
- package/dist/react-native/partial-words.mjs +7 -6
- package/dist/react-native/pattern.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +19 -17
- package/dist/react-native/sync.mjs +47 -13
- package/dist/react-native/synonyms.mjs +92 -24
- package/dist/react.d.ts +7 -7
- package/dist/{browser/release-76MXRPGV.mjs → release-SMR4WJD4.mjs} +0 -1
- package/dist/{restore-hooks-HZ3ILAMO.mjs → restore-hooks-HXLIEB4A.mjs} +28 -21
- package/dist/{restore-queue-JVGENGGE.mjs → restore-queue-LZH7LMLR.mjs} +3 -3
- package/dist/{retention-HROPV2QU.mjs → retention-3K7ARGAG.mjs} +29 -22
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +55 -41
- package/dist/{snapshot-4HS2G2UM.mjs → snapshot-O7ICP7NJ.mjs} +46 -39
- package/dist/{start-UTWHGH42.mjs → start-S4PFP6RS.mjs} +31 -24
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +3 -3
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +46 -12
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +46 -12
- package/dist/synonyms.mjs +7 -6
- package/dist/{tasks-qrJojgST.d.ts → tasks-CoNjxbop.d.ts} +2 -2
- package/dist/{thread-6MOXY2KD.mjs → thread-YLJ3ZPQ3.mjs} +25 -20
- package/dist/{types-BZCGjfLn.d.ts → types-DPMjfOr_.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +17 -6
- package/dist/{write-GTTWWRRG.mjs → write-2OBJGLFS.mjs} +27 -19
- package/openapi.json +71 -0
- package/package.json +15 -8
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-HNVVEQP7.mjs +0 -44
- package/dist/chunk-WQRN64Q6.mjs +0 -196
- package/dist/direct-executor-G5TK7NLA.mjs +0 -29
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +0 -191
- package/dist/browser/{chunk-4WCVMOU3.mjs → chunk-XAWNZ4G2.mjs} +10 -10
|
@@ -1,25 +1,27 @@
|
|
|
1
|
+
import { createRequestId } from './chunk-NNE4VDMM.mjs';
|
|
1
2
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-IO2SMUQD.mjs';
|
|
2
|
-
import { createDocIdMap, turnsAreBudgeted, turnIsSpent, yieldTurn, detectRuntime, indexChangeOutsideExternalWaits, changeIndexOutsideExternalWaits, countWrite, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, yieldBackgroundTurn, backgroundTurnIsSpent, reservedHeapBytes, reserveHeapBytes, MIN_OVERSAMPLE, noteThreadSlotGone, threadSlotOfWorker, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, collectAndReturnPages, admittedHeapBytes } from './chunk-
|
|
3
|
+
import { createDocIdMap, turnsAreBudgeted, turnIsSpent, yieldTurn, detectRuntime, indexChangeOutsideExternalWaits, changeIndexOutsideExternalWaits, noteFallback, countWrite, nativeSearchCoreIsEnabled, loadNativeCore, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, yieldBackgroundTurn, backgroundTurnIsSpent, reservedHeapBytes, reserveHeapBytes, MIN_OVERSAMPLE, noteThreadSlotGone, threadSlotOfWorker, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, collectAndReturnPages, admittedHeapBytes, NATIVE_OK } from './chunk-YZW4CHM6.mjs';
|
|
4
|
+
import { splitWorkerBudget, resolveWorkerCount, resolveRequestThreadCount, DEFAULT_REQUEST_TIMEOUT_MS, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-LU3Y3P63.mjs';
|
|
3
5
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION } from './chunk-AM7RSBPF.mjs';
|
|
4
6
|
import { reactNativeHost } from './chunk-MVXELMJX.mjs';
|
|
5
|
-
import { createRequestId } from './chunk-NNE4VDMM.mjs';
|
|
6
|
-
import { splitWorkerBudget, resolveWorkerCount, resolveRequestThreadCount, DEFAULT_REQUEST_TIMEOUT_MS, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-LU3Y3P63.mjs';
|
|
7
7
|
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, GEO_UNIT_METRES, DEFAULT_PATTERN_WORK_CAP, MAX_FILTER_DEPTH, MATCHER_STATE_BUDGET_BYTES, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH, TYPO_CLAIM_SLOTS_PER_WORD, PATTERN_WORK_FLUSH_UNITS } from './chunk-RDWG33MM.mjs';
|
|
8
|
-
import {
|
|
8
|
+
import { expandTermPrefix, suggestDisplayTerms, sameFields, partitionDocumentFrequency, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-KU646XXT.mjs';
|
|
9
9
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-UNUFZ2QY.mjs';
|
|
10
10
|
import { emitSyncChange, withOpenIndex, oneAtATime } from './chunk-JDOEEMEF.mjs';
|
|
11
|
-
import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder, tokenize } from './chunk-
|
|
11
|
+
import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder, tokenize, installNativeAnalysis, NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS, NATIVE_ANALYSIS_HELD_TERMS } from './chunk-2PRHILX3.mjs';
|
|
12
12
|
import { requireWithinFileLimit, HEADER_SIZE, payloadByteLength, checksumOfChunks, packEnvelopeWithEngineVersion, INDEX_CONFIGURATION_ENVELOPE_VERSION, readHeapStatistics, readOldGenerationStatistics } from './chunk-YH7QY657.mjs';
|
|
13
13
|
import { fnv1a, FNV_OFFSET_BASIS, fnv1aContinue, fnv1aContinueTaggedNumbers } from './chunk-CHPTFFJM.mjs';
|
|
14
|
-
import { flattenSchema, partialFieldsOf, schemaFieldsOf, validateSchema, requireCreatableSchema, encodeDocument, extractVectorFieldsFromSchema } from './chunk-
|
|
14
|
+
import { isLatitude, isLongitude, flattenSchema, partialFieldsOf, getNestedValue, schemaFieldsOf, tokenizeOptions, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateDocument, validateDocumentStrict, textFieldsChanged, encodeDocument, extractVectorFieldsFromSchema, getFlatSchema, getOrCreateFieldNameIndex, validateRequiredFields, requireDocumentWithinSizeLimit } from './chunk-4COWCEY7.mjs';
|
|
15
15
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-6VBXNFLE.mjs';
|
|
16
|
-
import { parsedTypeOf, isSingleStringFieldType, isWordIndexedBase, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, PROTOTYPE_POLLUTION_KEYS, isPlainObject,
|
|
16
|
+
import { parsedTypeOf, isSingleStringFieldType, isWordIndexedBase, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, PROTOTYPE_POLLUTION_KEYS, isPlainObject, isWordIndexedFieldType, isStringValuedBase, MAX_NESTING_DEPTH, withSortableOption, isGeopointOrVectorType, isListBase, VECTOR_PATTERN } from './chunk-C5G6TJ7A.mjs';
|
|
17
17
|
import { resolveIndexAnalysis, getLanguage } from './chunk-MJDQU5MW.mjs';
|
|
18
|
-
import { singleFoldTable, appendFoldedCodePoint, toReducedSortValue, compareComparableValues, defaultSortMode, readSortField, truncateSortString, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, compareSortValues } from './chunk-
|
|
19
|
-
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints, codePointCount } from './chunk-
|
|
18
|
+
import { singleFoldTable, appendFoldedCodePoint, toReducedSortValue, compareComparableValues, defaultSortMode, readSortField, truncateSortString, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, compareSortValues } from './chunk-3WI35N75.mjs';
|
|
19
|
+
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints, codePointCount } from './chunk-EJ33LCHV.mjs';
|
|
20
|
+
import { highWord, SMALLEST_NORMAL_HIGH_WORD, MAGNITUDE_MASK, lowWord, INFINITE_HIGH_WORD, withHighWord } from './chunk-FHPW7P2O.mjs';
|
|
21
|
+
import { MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS } from './chunk-DXRSEWA2.mjs';
|
|
20
22
|
import { bitsetHas, createBitSet, GrowableUint32, bitsetIsEmpty, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet } from './chunk-LVIKKAA6.mjs';
|
|
21
23
|
import { BM25_DISPLAYED_NAME_LENGTH, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-AAAETEJK.mjs';
|
|
22
|
-
import { TARGET_BLOCK_ENTRIES, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-
|
|
24
|
+
import { TARGET_BLOCK_ENTRIES, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, FIELD_RUN_ENTRIES_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-FTZHMRD2.mjs';
|
|
23
25
|
import { ErrorCodes, NarsilError } from './chunk-DZ7H7D2C.mjs';
|
|
24
26
|
import { Decoder, decode, encode } from '@msgpack/msgpack';
|
|
25
27
|
|
|
@@ -455,7 +457,7 @@ function createMemoryMonitor(options) {
|
|
|
455
457
|
collectionCount++;
|
|
456
458
|
return options.read() ?? figures;
|
|
457
459
|
}
|
|
458
|
-
function
|
|
460
|
+
function record2(figures) {
|
|
459
461
|
const wasFull = latest?.full ?? false;
|
|
460
462
|
let full = nextFullState(wasFull, figures, options.fractions);
|
|
461
463
|
let settled2 = figures;
|
|
@@ -473,7 +475,7 @@ function createMemoryMonitor(options) {
|
|
|
473
475
|
lastSampleAt = Date.now();
|
|
474
476
|
const figures = options.read();
|
|
475
477
|
if (figures !== null) {
|
|
476
|
-
|
|
478
|
+
record2(figures);
|
|
477
479
|
}
|
|
478
480
|
}
|
|
479
481
|
function latestFigures() {
|
|
@@ -487,7 +489,7 @@ function createMemoryMonitor(options) {
|
|
|
487
489
|
lastSampleAt = Date.now();
|
|
488
490
|
const figures = options.read();
|
|
489
491
|
if (figures === null) return latestFigures();
|
|
490
|
-
|
|
492
|
+
record2(figuresWithoutGarbageBeforeRefusal(figures));
|
|
491
493
|
return latestFigures();
|
|
492
494
|
},
|
|
493
495
|
refusedAfterCollection(figures) {
|
|
@@ -1316,6 +1318,11 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
1316
1318
|
}
|
|
1317
1319
|
}
|
|
1318
1320
|
|
|
1321
|
+
// src/core/document-copy.browser.ts
|
|
1322
|
+
function copyStoredValue(value) {
|
|
1323
|
+
return structuredClone(value);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1319
1326
|
// src/core/projection.ts
|
|
1320
1327
|
function requireFieldPathList(paths, option) {
|
|
1321
1328
|
if (paths === void 0) return;
|
|
@@ -1366,9 +1373,9 @@ function readProjectedPath(source, path) {
|
|
|
1366
1373
|
for (const segment of path) {
|
|
1367
1374
|
if (typeof current !== "object" || current === null || Array.isArray(current))
|
|
1368
1375
|
return { found: false, value: void 0 };
|
|
1369
|
-
const
|
|
1370
|
-
if (!Object.hasOwn(
|
|
1371
|
-
current =
|
|
1376
|
+
const record2 = current;
|
|
1377
|
+
if (!Object.hasOwn(record2, segment)) return { found: false, value: void 0 };
|
|
1378
|
+
current = record2[segment];
|
|
1372
1379
|
}
|
|
1373
1380
|
return { found: true, value: current };
|
|
1374
1381
|
}
|
|
@@ -1389,8 +1396,8 @@ function writePath(target, path, value) {
|
|
|
1389
1396
|
}
|
|
1390
1397
|
function cloneExcept(value, path, exclude) {
|
|
1391
1398
|
const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
|
|
1392
|
-
if (!reachesDeeper) return
|
|
1393
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return
|
|
1399
|
+
if (!reachesDeeper) return copyStoredValue(value);
|
|
1400
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return copyStoredValue(value);
|
|
1394
1401
|
const source = value;
|
|
1395
1402
|
const kept = {};
|
|
1396
1403
|
for (const key of Object.keys(source)) {
|
|
@@ -1429,7 +1436,7 @@ function projectionKeepsField(projection, fieldPath) {
|
|
|
1429
1436
|
return !projection.exclude.some((excluded) => covers(excluded, path));
|
|
1430
1437
|
}
|
|
1431
1438
|
function cloneProjected(document, projection) {
|
|
1432
|
-
if (projection === void 0 || projection.kind === "full") return
|
|
1439
|
+
if (projection === void 0 || projection.kind === "full") return copyStoredValue(document);
|
|
1433
1440
|
if (projection.kind === "none") return {};
|
|
1434
1441
|
if (projection.include === null) return cloneExcept(document, [], projection.exclude);
|
|
1435
1442
|
return projectIncludedPaths(projection.include, projection.exclude, (path) => readProjectedPath(document, path));
|
|
@@ -1577,9 +1584,9 @@ function createDocumentStore() {
|
|
|
1577
1584
|
for (const fieldName of Object.keys(fieldLengths)) {
|
|
1578
1585
|
let column = lengthColumns.get(fieldName);
|
|
1579
1586
|
if (column === void 0 || column.length <= internalId) {
|
|
1580
|
-
const
|
|
1581
|
-
if (column !== void 0)
|
|
1582
|
-
column =
|
|
1587
|
+
const grown2 = new Uint32Array(Math.max(internalId + 1, (column?.length ?? 0) * 2, 64));
|
|
1588
|
+
if (column !== void 0) grown2.set(column);
|
|
1589
|
+
column = grown2;
|
|
1583
1590
|
lengthColumns.set(fieldName, column);
|
|
1584
1591
|
}
|
|
1585
1592
|
column[internalId] = fieldLengths[fieldName];
|
|
@@ -2010,9 +2017,9 @@ var PositionPool = class {
|
|
|
2010
2017
|
reserve(needed) {
|
|
2011
2018
|
if (needed <= this.values.length) return;
|
|
2012
2019
|
const capacity = grownCapacity(this.values.length, needed);
|
|
2013
|
-
const
|
|
2014
|
-
|
|
2015
|
-
this.values =
|
|
2020
|
+
const grown2 = new Uint32Array(capacity);
|
|
2021
|
+
grown2.set(this.values.subarray(0, this.end));
|
|
2022
|
+
this.values = grown2;
|
|
2016
2023
|
}
|
|
2017
2024
|
};
|
|
2018
2025
|
|
|
@@ -2054,9 +2061,9 @@ var PostingPool2 = class extends PostingPool {
|
|
|
2054
2061
|
if (this.copy !== null || this.replaced) return;
|
|
2055
2062
|
const capacity = this.docIds.length;
|
|
2056
2063
|
if (this.end < capacity * POOL_GROWTH_START_FRACTION) return;
|
|
2057
|
-
const
|
|
2058
|
-
if (
|
|
2059
|
-
const copy = new PostingCopy(this,
|
|
2064
|
+
const grown2 = grownCapacity(capacity, this.end + 1);
|
|
2065
|
+
if (grown2 <= capacity) return;
|
|
2066
|
+
const copy = new PostingCopy(this, grown2);
|
|
2060
2067
|
this.copy = copy;
|
|
2061
2068
|
void copyBetweenTurns(copy);
|
|
2062
2069
|
}
|
|
@@ -2098,9 +2105,9 @@ var PositionPool2 = class extends PositionPool {
|
|
|
2098
2105
|
if (this.copy !== null || this.replaced) return;
|
|
2099
2106
|
const capacity = this.values.length;
|
|
2100
2107
|
if (this.end < capacity * POOL_GROWTH_START_FRACTION) return;
|
|
2101
|
-
const
|
|
2102
|
-
if (
|
|
2103
|
-
const copy = new PositionCopy(this,
|
|
2108
|
+
const grown2 = grownCapacity(capacity, this.end + 1);
|
|
2109
|
+
if (grown2 <= capacity) return;
|
|
2110
|
+
const copy = new PositionCopy(this, grown2);
|
|
2104
2111
|
this.copy = copy;
|
|
2105
2112
|
void copyBetweenTurns(copy);
|
|
2106
2113
|
}
|
|
@@ -2472,7 +2479,7 @@ var LivePostings = class {
|
|
|
2472
2479
|
terms = new TermColumns(TERM_TABLE_START_CAPACITY);
|
|
2473
2480
|
pool = new PostingPool2(POSTING_POOL_START_ENTRIES);
|
|
2474
2481
|
positions = new PositionPool2(POSITION_POOL_START_VALUES);
|
|
2475
|
-
retired = new
|
|
2482
|
+
retired = new Uint32Array(RETIRED_DOCUMENTS_START_CAPACITY >>> 5);
|
|
2476
2483
|
batchMode = false;
|
|
2477
2484
|
dirty = /* @__PURE__ */ new Set();
|
|
2478
2485
|
cachedViews = /* @__PURE__ */ new Map();
|
|
@@ -2559,7 +2566,7 @@ var LivePostings = class {
|
|
|
2559
2566
|
this.repackIfSparse();
|
|
2560
2567
|
}
|
|
2561
2568
|
isRetired(internalId) {
|
|
2562
|
-
return
|
|
2569
|
+
return bitsetHas(this.retired, internalId);
|
|
2563
2570
|
}
|
|
2564
2571
|
beginBatch() {
|
|
2565
2572
|
this.batchMode = true;
|
|
@@ -2583,7 +2590,7 @@ var LivePostings = class {
|
|
|
2583
2590
|
this.pool = new PostingPool2(POSTING_POOL_START_ENTRIES);
|
|
2584
2591
|
this.pool.epoch = epoch;
|
|
2585
2592
|
this.positions = new PositionPool2(POSITION_POOL_START_VALUES);
|
|
2586
|
-
this.retired = new
|
|
2593
|
+
this.retired = new Uint32Array(RETIRED_DOCUMENTS_START_CAPACITY >>> 5);
|
|
2587
2594
|
this.dirty.clear();
|
|
2588
2595
|
this.cachedViews.clear();
|
|
2589
2596
|
this.compactedSlack = 0;
|
|
@@ -2592,12 +2599,13 @@ var LivePostings = class {
|
|
|
2592
2599
|
return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
|
|
2593
2600
|
}
|
|
2594
2601
|
retire(internalId) {
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
this.retired
|
|
2602
|
+
const word = internalId >>> 5;
|
|
2603
|
+
if (word >= this.retired.length) {
|
|
2604
|
+
const grown2 = new Uint32Array(Math.max(word + 1, this.retired.length * 2));
|
|
2605
|
+
grown2.set(this.retired);
|
|
2606
|
+
this.retired = grown2;
|
|
2599
2607
|
}
|
|
2600
|
-
this.retired
|
|
2608
|
+
bitsetSet(this.retired, internalId);
|
|
2601
2609
|
}
|
|
2602
2610
|
purgeRetired() {
|
|
2603
2611
|
const terms = this.terms;
|
|
@@ -2609,7 +2617,7 @@ var LivePostings = class {
|
|
|
2609
2617
|
compact(ordinal) {
|
|
2610
2618
|
if (this.terms.deadDocuments[ordinal] === 0) return;
|
|
2611
2619
|
const retired = this.retired;
|
|
2612
|
-
this.filterEntries(ordinal, (id) =>
|
|
2620
|
+
this.filterEntries(ordinal, (id) => bitsetHas(retired, id));
|
|
2613
2621
|
this.terms.deadDocuments[ordinal] = 0;
|
|
2614
2622
|
}
|
|
2615
2623
|
filterEntries(ordinal, drop) {
|
|
@@ -2819,6 +2827,7 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
2819
2827
|
return entries;
|
|
2820
2828
|
}
|
|
2821
2829
|
return {
|
|
2830
|
+
livePostings: postings,
|
|
2822
2831
|
insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
|
|
2823
2832
|
postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
|
|
2824
2833
|
},
|
|
@@ -3324,420 +3333,54 @@ function patternIndexOf(indexes, fieldPath) {
|
|
|
3324
3333
|
return index;
|
|
3325
3334
|
}
|
|
3326
3335
|
|
|
3327
|
-
// src/
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
const
|
|
3331
|
-
|
|
3332
|
-
return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
|
|
3333
|
-
}
|
|
3334
|
-
function latitudeCell(lat) {
|
|
3335
|
-
return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
|
|
3336
|
-
}
|
|
3337
|
-
function longitudeCell(lon) {
|
|
3338
|
-
return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
|
|
3339
|
-
}
|
|
3340
|
-
function interleave(row, column, bits) {
|
|
3341
|
-
let code = 0;
|
|
3342
|
-
for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
|
|
3343
|
-
return code;
|
|
3344
|
-
}
|
|
3345
|
-
function locationCode(lat, lon) {
|
|
3346
|
-
return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
|
|
3347
|
-
}
|
|
3348
|
-
function appendBoxRanges(box, out) {
|
|
3349
|
-
const south = latitudeCell(box.south);
|
|
3350
|
-
const north = latitudeCell(box.north);
|
|
3351
|
-
const west = longitudeCell(box.west);
|
|
3352
|
-
const east = longitudeCell(box.east);
|
|
3353
|
-
let level = LOCATION_CODE_BITS_PER_AXIS;
|
|
3354
|
-
while (level > 0) {
|
|
3355
|
-
const shift2 = LOCATION_CODE_BITS_PER_AXIS - level;
|
|
3356
|
-
const rows = (north >>> shift2) - (south >>> shift2) + 1;
|
|
3357
|
-
const columns = (east >>> shift2) - (west >>> shift2) + 1;
|
|
3358
|
-
if (rows * columns <= MAX_COVERING_CELLS) break;
|
|
3359
|
-
level--;
|
|
3360
|
-
}
|
|
3361
|
-
const shift = LOCATION_CODE_BITS_PER_AXIS - level;
|
|
3362
|
-
const codesPerCell = 4 ** shift;
|
|
3363
|
-
for (let row = south >>> shift; row <= north >>> shift; row++) {
|
|
3364
|
-
for (let column = west >>> shift; column <= east >>> shift; column++) {
|
|
3365
|
-
const from = interleave(row, column, level) * codesPerCell;
|
|
3366
|
-
out.push({ from, to: from + codesPerCell });
|
|
3367
|
-
}
|
|
3368
|
-
}
|
|
3369
|
-
}
|
|
3370
|
-
function coveringRanges(boxes) {
|
|
3371
|
-
const ranges = [];
|
|
3372
|
-
for (const box of boxes) appendBoxRanges(box, ranges);
|
|
3373
|
-
ranges.sort((left, right) => left.from - right.from);
|
|
3374
|
-
const merged = [];
|
|
3375
|
-
for (const range of ranges) {
|
|
3376
|
-
const last = merged[merged.length - 1];
|
|
3377
|
-
if (last !== void 0 && range.from <= last.to) {
|
|
3378
|
-
if (range.to > last.to) last.to = range.to;
|
|
3379
|
-
} else {
|
|
3380
|
-
merged.push({ from: range.from, to: range.to });
|
|
3381
|
-
}
|
|
3336
|
+
// src/schema/declared-types.ts
|
|
3337
|
+
function mayDeclare(schema, typeName, depth) {
|
|
3338
|
+
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
3339
|
+
for (const type of Object.values(schema)) {
|
|
3340
|
+
if (typeof type === "string" ? type.includes(typeName) : mayDeclare(type, typeName, depth + 1)) return true;
|
|
3382
3341
|
}
|
|
3383
|
-
return
|
|
3342
|
+
return false;
|
|
3384
3343
|
}
|
|
3385
|
-
function
|
|
3386
|
-
|
|
3387
|
-
return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
|
|
3388
|
-
}
|
|
3389
|
-
if (west < -MAX_LONGITUDE_DEGREES) {
|
|
3390
|
-
return [
|
|
3391
|
-
{ south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
|
|
3392
|
-
{ south, north, west: -MAX_LONGITUDE_DEGREES, east }
|
|
3393
|
-
];
|
|
3394
|
-
}
|
|
3395
|
-
if (east > MAX_LONGITUDE_DEGREES) {
|
|
3396
|
-
return [
|
|
3397
|
-
{ south, north, west, east: MAX_LONGITUDE_DEGREES },
|
|
3398
|
-
{ south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
|
|
3399
|
-
];
|
|
3400
|
-
}
|
|
3401
|
-
return [{ south, north, west, east }];
|
|
3402
|
-
}
|
|
3403
|
-
function radiusBoxes(lat, lon, distanceMeters) {
|
|
3404
|
-
const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
|
|
3405
|
-
const angleDegrees = angle / DEG_TO_RAD;
|
|
3406
|
-
const south = lat - angleDegrees;
|
|
3407
|
-
const north = lat + angleDegrees;
|
|
3408
|
-
if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
|
|
3409
|
-
return [
|
|
3410
|
-
{
|
|
3411
|
-
south: Math.max(south, -MAX_LATITUDE_DEGREES),
|
|
3412
|
-
north: Math.min(north, MAX_LATITUDE_DEGREES),
|
|
3413
|
-
west: -MAX_LONGITUDE_DEGREES,
|
|
3414
|
-
east: MAX_LONGITUDE_DEGREES
|
|
3415
|
-
}
|
|
3416
|
-
];
|
|
3417
|
-
}
|
|
3418
|
-
const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
|
|
3419
|
-
return longitudeBoxes(south, north, lon - spread, lon + spread);
|
|
3420
|
-
}
|
|
3421
|
-
|
|
3422
|
-
// src/geo/haversine.ts
|
|
3423
|
-
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
3424
|
-
const phi1 = lat1 * DEG_TO_RAD;
|
|
3425
|
-
const phi2 = lat2 * DEG_TO_RAD;
|
|
3426
|
-
const deltaPhi = (lat2 - lat1) * DEG_TO_RAD;
|
|
3427
|
-
const deltaLambda = (lon2 - lon1) * DEG_TO_RAD;
|
|
3428
|
-
const halfDeltaPhi = Math.sin(deltaPhi / 2);
|
|
3429
|
-
const halfDeltaLambda = Math.sin(deltaLambda / 2);
|
|
3430
|
-
const a = halfDeltaPhi * halfDeltaPhi + Math.cos(phi1) * Math.cos(phi2) * halfDeltaLambda * halfDeltaLambda;
|
|
3431
|
-
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
|
3432
|
-
return EARTH_RADIUS_METERS * c;
|
|
3433
|
-
}
|
|
3434
|
-
|
|
3435
|
-
// src/geo/edge-tree.ts
|
|
3436
|
-
function buildLongitudeEdgeTree(lons) {
|
|
3437
|
-
const n = lons.length;
|
|
3438
|
-
const edges = [];
|
|
3439
|
-
for (let i = 0; i < n; i++) {
|
|
3440
|
-
if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
|
|
3441
|
-
}
|
|
3442
|
-
const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
|
|
3443
|
-
edges.sort((left, right) => lowOf(left) - lowOf(right));
|
|
3444
|
-
const count = edges.length;
|
|
3445
|
-
const tree = {
|
|
3446
|
-
from: new Int32Array(count),
|
|
3447
|
-
to: new Int32Array(count),
|
|
3448
|
-
low: new Float64Array(count),
|
|
3449
|
-
high: new Float64Array(count),
|
|
3450
|
-
subtreeHigh: new Float64Array(count)
|
|
3451
|
-
};
|
|
3452
|
-
edges.forEach((i, at) => {
|
|
3453
|
-
const j = i === 0 ? n - 1 : i - 1;
|
|
3454
|
-
tree.from[at] = i;
|
|
3455
|
-
tree.to[at] = j;
|
|
3456
|
-
tree.low[at] = Math.min(lons[i], lons[j]);
|
|
3457
|
-
tree.high[at] = Math.max(lons[i], lons[j]);
|
|
3458
|
-
});
|
|
3459
|
-
const fillSubtreeHigh = (start, end) => {
|
|
3460
|
-
if (start >= end) return Number.NEGATIVE_INFINITY;
|
|
3461
|
-
const mid = start + end >>> 1;
|
|
3462
|
-
const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
|
|
3463
|
-
tree.subtreeHigh[mid] = highest;
|
|
3464
|
-
return highest;
|
|
3465
|
-
};
|
|
3466
|
-
fillSubtreeHigh(0, count);
|
|
3467
|
-
return tree;
|
|
3468
|
-
}
|
|
3469
|
-
function forEachEdgeSpanning(tree, lon, visit) {
|
|
3470
|
-
const walk = (start, end) => {
|
|
3471
|
-
if (start >= end) return;
|
|
3472
|
-
const mid = start + end >>> 1;
|
|
3473
|
-
if (tree.subtreeHigh[mid] <= lon) return;
|
|
3474
|
-
walk(start, mid);
|
|
3475
|
-
if (tree.low[mid] > lon) return;
|
|
3476
|
-
if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
|
|
3477
|
-
walk(mid + 1, end);
|
|
3478
|
-
};
|
|
3479
|
-
walk(0, tree.from.length);
|
|
3344
|
+
function schemaMayDeclare(schema, typeName) {
|
|
3345
|
+
return mayDeclare(schema, typeName, 1);
|
|
3480
3346
|
}
|
|
3481
3347
|
|
|
3482
|
-
// src/
|
|
3483
|
-
function
|
|
3484
|
-
|
|
3485
|
-
|
|
3486
|
-
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3348
|
+
// src/workers/code-loader.ts
|
|
3349
|
+
function codeLoader(importCode, typeName) {
|
|
3350
|
+
let loaded2 = null;
|
|
3351
|
+
let loading2 = null;
|
|
3352
|
+
function load() {
|
|
3353
|
+
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
3354
|
+
if (loading2 === null) {
|
|
3355
|
+
loading2 = importCode().then(
|
|
3356
|
+
(code) => {
|
|
3357
|
+
loaded2 = code;
|
|
3358
|
+
return code;
|
|
3359
|
+
},
|
|
3360
|
+
(err) => {
|
|
3361
|
+
loading2 = null;
|
|
3362
|
+
throw err;
|
|
3363
|
+
}
|
|
3498
3364
|
);
|
|
3499
3365
|
}
|
|
3500
|
-
|
|
3501
|
-
throw new NarsilError(
|
|
3502
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
3503
|
-
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
3504
|
-
{ position, lat, lon }
|
|
3505
|
-
);
|
|
3506
|
-
});
|
|
3507
|
-
}
|
|
3508
|
-
function signedArea(polygon) {
|
|
3509
|
-
let area = 0;
|
|
3510
|
-
for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
|
|
3511
|
-
area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
|
|
3512
|
-
}
|
|
3513
|
-
return area;
|
|
3514
|
-
}
|
|
3515
|
-
function crossesAntimeridian(polygon) {
|
|
3516
|
-
let west = Number.POSITIVE_INFINITY;
|
|
3517
|
-
let east = Number.NEGATIVE_INFINITY;
|
|
3518
|
-
for (const point of polygon) {
|
|
3519
|
-
if (point.lon < west) west = point.lon;
|
|
3520
|
-
if (point.lon > east) east = point.lon;
|
|
3521
|
-
}
|
|
3522
|
-
return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
|
|
3523
|
-
}
|
|
3524
|
-
function eastward(lon) {
|
|
3525
|
-
return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
|
|
3526
|
-
}
|
|
3527
|
-
function polygonContainment(polygon) {
|
|
3528
|
-
const n = polygon.length;
|
|
3529
|
-
if (n < MIN_POLYGON_POINTS) return () => false;
|
|
3530
|
-
const unwrap = crossesAntimeridian(polygon);
|
|
3531
|
-
const lats = new Float64Array(n);
|
|
3532
|
-
const lons = new Float64Array(n);
|
|
3533
|
-
for (let i = 0; i < n; i++) {
|
|
3534
|
-
lats[i] = polygon[i].lat;
|
|
3535
|
-
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
3536
|
-
}
|
|
3537
|
-
const edges = buildLongitudeEdgeTree(lons);
|
|
3538
|
-
return (lat, rawLon) => {
|
|
3539
|
-
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
3540
|
-
let inside = false;
|
|
3541
|
-
forEachEdgeSpanning(edges, lon, (i, j) => {
|
|
3542
|
-
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
|
|
3543
|
-
if (lat < intersectLat) inside = !inside;
|
|
3544
|
-
});
|
|
3545
|
-
return inside;
|
|
3546
|
-
};
|
|
3547
|
-
}
|
|
3548
|
-
function polygonBoxes(polygon) {
|
|
3549
|
-
if (polygon.length < MIN_POLYGON_POINTS) return [];
|
|
3550
|
-
const unwrap = crossesAntimeridian(polygon);
|
|
3551
|
-
let south = Number.POSITIVE_INFINITY;
|
|
3552
|
-
let north = Number.NEGATIVE_INFINITY;
|
|
3553
|
-
let west = Number.POSITIVE_INFINITY;
|
|
3554
|
-
let east = Number.NEGATIVE_INFINITY;
|
|
3555
|
-
for (const point of polygon) {
|
|
3556
|
-
const lon = unwrap ? eastward(point.lon) : point.lon;
|
|
3557
|
-
south = Math.min(south, point.lat);
|
|
3558
|
-
north = Math.max(north, point.lat);
|
|
3559
|
-
west = Math.min(west, lon);
|
|
3560
|
-
east = Math.max(east, lon);
|
|
3561
|
-
}
|
|
3562
|
-
if (west >= MAX_LONGITUDE_DEGREES)
|
|
3563
|
-
return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
|
|
3564
|
-
return longitudeBoxes(south, north, west, east);
|
|
3565
|
-
}
|
|
3566
|
-
|
|
3567
|
-
// src/geo/vincenty.ts
|
|
3568
|
-
var WGS84_A = 6378137;
|
|
3569
|
-
var WGS84_F = 1 / 298.257223563;
|
|
3570
|
-
var WGS84_B = WGS84_A * (1 - WGS84_F);
|
|
3571
|
-
function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
3572
|
-
const phi1 = lat1 * DEG_TO_RAD;
|
|
3573
|
-
const phi2 = lat2 * DEG_TO_RAD;
|
|
3574
|
-
const L = (lon2 - lon1) * DEG_TO_RAD;
|
|
3575
|
-
const U1 = Math.atan((1 - WGS84_F) * Math.tan(phi1));
|
|
3576
|
-
const U2 = Math.atan((1 - WGS84_F) * Math.tan(phi2));
|
|
3577
|
-
const sinU1 = Math.sin(U1);
|
|
3578
|
-
const cosU1 = Math.cos(U1);
|
|
3579
|
-
const sinU2 = Math.sin(U2);
|
|
3580
|
-
const cosU2 = Math.cos(U2);
|
|
3581
|
-
let lambda = L;
|
|
3582
|
-
let prevLambda;
|
|
3583
|
-
let sinSigma = 0;
|
|
3584
|
-
let cosSigma = 0;
|
|
3585
|
-
let sigma = 0;
|
|
3586
|
-
let sinAlpha = 0;
|
|
3587
|
-
let cos2Alpha = 0;
|
|
3588
|
-
let cos2SigmaM = 0;
|
|
3589
|
-
let iterations = VINCENTY_MAX_ITERATIONS;
|
|
3590
|
-
do {
|
|
3591
|
-
const sinLambda = Math.sin(lambda);
|
|
3592
|
-
const cosLambda = Math.cos(lambda);
|
|
3593
|
-
const term1 = cosU2 * sinLambda;
|
|
3594
|
-
const term2 = cosU1 * sinU2 - sinU1 * cosU2 * cosLambda;
|
|
3595
|
-
sinSigma = Math.sqrt(term1 * term1 + term2 * term2);
|
|
3596
|
-
if (sinSigma === 0) return 0;
|
|
3597
|
-
cosSigma = sinU1 * sinU2 + cosU1 * cosU2 * cosLambda;
|
|
3598
|
-
sigma = Math.atan2(sinSigma, cosSigma);
|
|
3599
|
-
sinAlpha = cosU1 * cosU2 * sinLambda / sinSigma;
|
|
3600
|
-
cos2Alpha = 1 - sinAlpha * sinAlpha;
|
|
3601
|
-
cos2SigmaM = cos2Alpha === 0 ? 0 : cosSigma - 2 * sinU1 * sinU2 / cos2Alpha;
|
|
3602
|
-
const C = WGS84_F / 16 * cos2Alpha * (4 + WGS84_F * (4 - 3 * cos2Alpha));
|
|
3603
|
-
prevLambda = lambda;
|
|
3604
|
-
lambda = L + (1 - C) * WGS84_F * sinAlpha * (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM)));
|
|
3605
|
-
} while (Math.abs(lambda - prevLambda) > VINCENTY_CONVERGENCE_THRESHOLD && --iterations > 0);
|
|
3606
|
-
if (iterations === 0) {
|
|
3607
|
-
return haversineDistance(lat1, lon1, lat2, lon2);
|
|
3608
|
-
}
|
|
3609
|
-
const uSq = cos2Alpha * (WGS84_A * WGS84_A - WGS84_B * WGS84_B) / (WGS84_B * WGS84_B);
|
|
3610
|
-
const A = 1 + uSq / 16384 * (4096 + uSq * (-768 + uSq * (320 - 175 * uSq)));
|
|
3611
|
-
const B = uSq / 1024 * (256 + uSq * (-128 + uSq * (74 - 47 * uSq)));
|
|
3612
|
-
const deltaSigma = B * sinSigma * (cos2SigmaM + B / 4 * (cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM) - B / 6 * cos2SigmaM * (-3 + 4 * sinSigma * sinSigma) * (-3 + 4 * cos2SigmaM * cos2SigmaM)));
|
|
3613
|
-
return WGS84_B * A * (sigma - deltaSigma);
|
|
3614
|
-
}
|
|
3615
|
-
|
|
3616
|
-
// src/geo/geo-index.ts
|
|
3617
|
-
var REMOVED_DOC_ID = -1;
|
|
3618
|
-
function firstAtOrAfter(entries, code) {
|
|
3619
|
-
let lo = 0;
|
|
3620
|
-
let hi = entries.length;
|
|
3621
|
-
while (lo < hi) {
|
|
3622
|
-
const mid = lo + hi >>> 1;
|
|
3623
|
-
if (entries[mid].code < code) lo = mid + 1;
|
|
3624
|
-
else hi = mid;
|
|
3625
|
-
}
|
|
3626
|
-
return lo;
|
|
3627
|
-
}
|
|
3628
|
-
function dropRemoved(entries) {
|
|
3629
|
-
let write = 0;
|
|
3630
|
-
for (let read = 0; read < entries.length; read++) {
|
|
3631
|
-
if (entries[read].docId !== REMOVED_DOC_ID) entries[write++] = entries[read];
|
|
3632
|
-
}
|
|
3633
|
-
entries.length = write;
|
|
3634
|
-
}
|
|
3635
|
-
function mergeSortedPending(entries, pending) {
|
|
3636
|
-
pending.sort((left2, right2) => left2.code - right2.code || left2.docId - right2.docId);
|
|
3637
|
-
const mainLength = entries.length;
|
|
3638
|
-
for (const entry of pending) entries.push(entry);
|
|
3639
|
-
let left = mainLength - 1;
|
|
3640
|
-
let right = pending.length - 1;
|
|
3641
|
-
let write = entries.length - 1;
|
|
3642
|
-
while (right >= 0) {
|
|
3643
|
-
const later = left >= 0 ? entries[left] : void 0;
|
|
3644
|
-
const next = pending[right];
|
|
3645
|
-
if (later !== void 0 && (later.code > next.code || later.code === next.code && later.docId > next.docId)) {
|
|
3646
|
-
entries[write--] = later;
|
|
3647
|
-
left--;
|
|
3648
|
-
} else {
|
|
3649
|
-
entries[write--] = next;
|
|
3650
|
-
right--;
|
|
3651
|
-
}
|
|
3652
|
-
}
|
|
3653
|
-
}
|
|
3654
|
-
function createGeoIndex() {
|
|
3655
|
-
let entries = [];
|
|
3656
|
-
let pending = [];
|
|
3657
|
-
const byDocument = /* @__PURE__ */ new Map();
|
|
3658
|
-
let removedCount = 0;
|
|
3659
|
-
function ensureSorted() {
|
|
3660
|
-
if (removedCount > 0) {
|
|
3661
|
-
dropRemoved(entries);
|
|
3662
|
-
dropRemoved(pending);
|
|
3663
|
-
removedCount = 0;
|
|
3664
|
-
}
|
|
3665
|
-
if (pending.length === 0) return;
|
|
3666
|
-
if (entries.length === 0) {
|
|
3667
|
-
pending.sort((left, right) => left.code - right.code || left.docId - right.docId);
|
|
3668
|
-
entries = pending;
|
|
3669
|
-
} else {
|
|
3670
|
-
mergeSortedPending(entries, pending);
|
|
3671
|
-
}
|
|
3672
|
-
pending = [];
|
|
3366
|
+
return loading2;
|
|
3673
3367
|
}
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
if (removedCount >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) ensureSorted();
|
|
3680
|
-
}
|
|
3681
|
-
function matching(boxes, passes, inside) {
|
|
3682
|
-
ensureSorted();
|
|
3683
|
-
const within = /* @__PURE__ */ new Set();
|
|
3684
|
-
for (const range of coveringRanges(boxes)) {
|
|
3685
|
-
for (let at = firstAtOrAfter(entries, range.from); at < entries.length && entries[at].code < range.to; at++) {
|
|
3686
|
-
if (passes(entries[at])) within.add(entries[at].docId);
|
|
3687
|
-
}
|
|
3688
|
-
}
|
|
3689
|
-
if (inside) return within;
|
|
3690
|
-
const outside = /* @__PURE__ */ new Set();
|
|
3691
|
-
for (const entry of entries) {
|
|
3692
|
-
if (!within.has(entry.docId)) outside.add(entry.docId);
|
|
3693
|
-
}
|
|
3694
|
-
return outside;
|
|
3695
|
-
}
|
|
3696
|
-
const index = {
|
|
3697
|
-
insert(internalId, lat, lon) {
|
|
3698
|
-
const existing = byDocument.get(internalId);
|
|
3699
|
-
if (existing !== void 0) removeEntry(existing);
|
|
3700
|
-
const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
|
|
3701
|
-
pending.push(entry);
|
|
3702
|
-
byDocument.set(internalId, entry);
|
|
3703
|
-
},
|
|
3704
|
-
remove(internalId) {
|
|
3705
|
-
const existing = byDocument.get(internalId);
|
|
3706
|
-
if (existing !== void 0) removeEntry(existing);
|
|
3707
|
-
},
|
|
3708
|
-
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
3709
|
-
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
3710
|
-
return matching(
|
|
3711
|
-
radiusBoxes(lat, lon, distanceMeters),
|
|
3712
|
-
(entry) => distanceFn(lat, lon, entry.lat, entry.lon) <= distanceMeters,
|
|
3713
|
-
inside
|
|
3714
|
-
);
|
|
3715
|
-
},
|
|
3716
|
-
polygonQuery(points, inside) {
|
|
3717
|
-
const contains = polygonContainment(points);
|
|
3718
|
-
return matching(polygonBoxes(points), (entry) => contains(entry.lat, entry.lon), inside);
|
|
3719
|
-
},
|
|
3720
|
-
documentsBitset(capacity) {
|
|
3721
|
-
const bits = createBitSet(capacity);
|
|
3722
|
-
for (const docId of byDocument.keys()) bitsetSet(bits, docId);
|
|
3723
|
-
return bits;
|
|
3724
|
-
},
|
|
3725
|
-
clear() {
|
|
3726
|
-
entries = [];
|
|
3727
|
-
pending = [];
|
|
3728
|
-
byDocument.clear();
|
|
3729
|
-
removedCount = 0;
|
|
3730
|
-
},
|
|
3731
|
-
serialize() {
|
|
3732
|
-
ensureSorted();
|
|
3733
|
-
return entries.map((entry) => ({ lat: entry.lat, lon: entry.lon, docId: entry.docId }));
|
|
3368
|
+
return {
|
|
3369
|
+
loaded: () => loaded2,
|
|
3370
|
+
require() {
|
|
3371
|
+
if (loaded2 === null) throw new Error(`The engine reached ${typeName} code before it loaded that code`);
|
|
3372
|
+
return loaded2;
|
|
3734
3373
|
},
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
for (const entry of data) index.insert(entry.docId, entry.lat, entry.lon);
|
|
3738
|
-
}
|
|
3374
|
+
load,
|
|
3375
|
+
loadFor: (schema) => loaded2 !== null || !schemaMayDeclare(schema, typeName) ? null : load()
|
|
3739
3376
|
};
|
|
3740
|
-
|
|
3377
|
+
}
|
|
3378
|
+
|
|
3379
|
+
// src/geo/geo-code.ts
|
|
3380
|
+
var geoCode = codeLoader(() => import('./geo-index-PX3BIHAH.mjs'), "geopoint");
|
|
3381
|
+
var loadGeoCodeFor = geoCode.loadFor;
|
|
3382
|
+
function createLoadedGeoIndex() {
|
|
3383
|
+
return geoCode.require().createGeoIndex(PENDING_REMOVAL_MERGE_FLOOR, PENDING_REMOVAL_MERGE_DIVISOR);
|
|
3741
3384
|
}
|
|
3742
3385
|
|
|
3743
3386
|
// src/core/field-index/boolean.ts
|
|
@@ -3821,7 +3464,7 @@ function reversedUnits(value) {
|
|
|
3821
3464
|
for (let at = value.length - 1; at >= 0; at--) reversed += value[at];
|
|
3822
3465
|
return reversed;
|
|
3823
3466
|
}
|
|
3824
|
-
function
|
|
3467
|
+
function firstAtOrAfter(sorted, key) {
|
|
3825
3468
|
let lo = 0;
|
|
3826
3469
|
let hi = sorted.length;
|
|
3827
3470
|
while (lo < hi) {
|
|
@@ -3831,10 +3474,10 @@ function firstAtOrAfter2(sorted, key) {
|
|
|
3831
3474
|
}
|
|
3832
3475
|
return lo;
|
|
3833
3476
|
}
|
|
3834
|
-
function settledColumn(column,
|
|
3477
|
+
function settledColumn(column, holds2) {
|
|
3835
3478
|
const { keys, values } = column;
|
|
3836
3479
|
const order = [];
|
|
3837
|
-
for (let at = 0; at < keys.length; at++) if (
|
|
3480
|
+
for (let at = 0; at < keys.length; at++) if (holds2(values[at])) order.push(at);
|
|
3838
3481
|
order.sort((left, right) => compareUnits(keys[left], keys[right]) || compareUnits(values[left], values[right]));
|
|
3839
3482
|
const settled2 = { keys: [], values: [] };
|
|
3840
3483
|
for (const at of order) {
|
|
@@ -3846,8 +3489,8 @@ function settledColumn(column, holds) {
|
|
|
3846
3489
|
return settled2;
|
|
3847
3490
|
}
|
|
3848
3491
|
var ValueAffixes = class {
|
|
3849
|
-
constructor(
|
|
3850
|
-
this.holds =
|
|
3492
|
+
constructor(holds2, values) {
|
|
3493
|
+
this.holds = holds2;
|
|
3851
3494
|
for (const value of values) this.add(value);
|
|
3852
3495
|
}
|
|
3853
3496
|
forward = { keys: [], values: [] };
|
|
@@ -3879,7 +3522,7 @@ var ValueAffixes = class {
|
|
|
3879
3522
|
}
|
|
3880
3523
|
matching(column, start) {
|
|
3881
3524
|
const found = [];
|
|
3882
|
-
for (let at =
|
|
3525
|
+
for (let at = firstAtOrAfter(column.keys, start); at < column.keys.length; at++) {
|
|
3883
3526
|
if (!column.keys[at].startsWith(start)) break;
|
|
3884
3527
|
found.push(column.values[at]);
|
|
3885
3528
|
}
|
|
@@ -4117,7 +3760,7 @@ function removeFromPending(pending, marked, byDocument) {
|
|
|
4117
3760
|
}
|
|
4118
3761
|
pending.length = write;
|
|
4119
3762
|
}
|
|
4120
|
-
function
|
|
3763
|
+
function mergeSortedPending(entries, pending) {
|
|
4121
3764
|
pending.sort((a, b) => a.value - b.value);
|
|
4122
3765
|
const mainLength = entries.length;
|
|
4123
3766
|
for (const entry of pending) entries.push(entry);
|
|
@@ -4153,7 +3796,7 @@ function createNumericIndex() {
|
|
|
4153
3796
|
pending.sort((a, b) => a.value - b.value);
|
|
4154
3797
|
entries = pending;
|
|
4155
3798
|
} else {
|
|
4156
|
-
|
|
3799
|
+
mergeSortedPending(entries, pending);
|
|
4157
3800
|
}
|
|
4158
3801
|
pending = [];
|
|
4159
3802
|
}
|
|
@@ -4341,7 +3984,7 @@ function ensureFieldIndex(state, fieldPath, base) {
|
|
|
4341
3984
|
} else if ((base === "enum" || base === "enum[]") && !state.enumIndexes.has(fieldPath)) {
|
|
4342
3985
|
state.enumIndexes.set(fieldPath, createEnumIndex());
|
|
4343
3986
|
} else if (base === "geopoint" && !state.geoIndexes.has(fieldPath)) {
|
|
4344
|
-
state.geoIndexes.set(fieldPath,
|
|
3987
|
+
state.geoIndexes.set(fieldPath, createLoadedGeoIndex());
|
|
4345
3988
|
}
|
|
4346
3989
|
}
|
|
4347
3990
|
function getOrCreateNumericIndex(state, fieldPath) {
|
|
@@ -4924,6 +4567,33 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
4924
4567
|
"polygon"
|
|
4925
4568
|
];
|
|
4926
4569
|
|
|
4570
|
+
// src/geo/polygon-ring.ts
|
|
4571
|
+
function requirePolygonRing(points) {
|
|
4572
|
+
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
4573
|
+
throw new NarsilError(
|
|
4574
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4575
|
+
`A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
|
|
4576
|
+
{ points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
|
|
4577
|
+
);
|
|
4578
|
+
}
|
|
4579
|
+
points.forEach((point, position) => {
|
|
4580
|
+
const { lat, lon } = point ?? {};
|
|
4581
|
+
if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
|
|
4582
|
+
throw new NarsilError(
|
|
4583
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4584
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
4585
|
+
{ position }
|
|
4586
|
+
);
|
|
4587
|
+
}
|
|
4588
|
+
if (isLatitude(lat) && isLongitude(lon)) return;
|
|
4589
|
+
throw new NarsilError(
|
|
4590
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4591
|
+
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
4592
|
+
{ position, lat, lon }
|
|
4593
|
+
);
|
|
4594
|
+
});
|
|
4595
|
+
}
|
|
4596
|
+
|
|
4927
4597
|
// src/filters/operands/geo.ts
|
|
4928
4598
|
function requireFlag(fieldPath, operator, flag, value) {
|
|
4929
4599
|
if (value === void 0 || typeof value === "boolean") return;
|
|
@@ -5287,11 +4957,11 @@ function noteUnknownExpressionKey(expression, walk) {
|
|
|
5287
4957
|
return;
|
|
5288
4958
|
}
|
|
5289
4959
|
}
|
|
5290
|
-
function setOwn(
|
|
4960
|
+
function setOwn(record2, key, value) {
|
|
5291
4961
|
if (key === PROTOTYPE_KEY) {
|
|
5292
|
-
Object.defineProperty(
|
|
4962
|
+
Object.defineProperty(record2, key, { value, enumerable: true, writable: true, configurable: true });
|
|
5293
4963
|
} else {
|
|
5294
|
-
|
|
4964
|
+
record2[key] = value;
|
|
5295
4965
|
}
|
|
5296
4966
|
}
|
|
5297
4967
|
function ownedOperandOf(value, fieldPath, operator, depth) {
|
|
@@ -5302,9 +4972,9 @@ function ownedOperandOf(value, fieldPath, operator, depth) {
|
|
|
5302
4972
|
for (let at = 0; at < value.length; at++) owned2.push(ownedOperandOf(value[at], fieldPath, operator, depth + 1));
|
|
5303
4973
|
return owned2;
|
|
5304
4974
|
}
|
|
5305
|
-
const
|
|
4975
|
+
const record2 = value;
|
|
5306
4976
|
const owned = {};
|
|
5307
|
-
for (const key of Object.keys(
|
|
4977
|
+
for (const key of Object.keys(record2)) setOwn(owned, key, ownedOperandOf(record2[key], fieldPath, operator, depth + 1));
|
|
5308
4978
|
return owned;
|
|
5309
4979
|
}
|
|
5310
4980
|
function ownedFieldFilterOf(fieldPath, filter) {
|
|
@@ -5532,20 +5202,20 @@ function createValueStore(initialKind) {
|
|
|
5532
5202
|
function setNumber(internalId, value) {
|
|
5533
5203
|
if (numbers === null) return;
|
|
5534
5204
|
if (internalId >= numbers.length) {
|
|
5535
|
-
const
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
numbers =
|
|
5205
|
+
const grown2 = new Float64Array(grow2(numbers.length, internalId));
|
|
5206
|
+
grown2.fill(Number.NaN, numbers.length);
|
|
5207
|
+
grown2.set(numbers);
|
|
5208
|
+
numbers = grown2;
|
|
5539
5209
|
}
|
|
5540
5210
|
numbers[internalId] = value;
|
|
5541
5211
|
}
|
|
5542
5212
|
function setBoolean(internalId, value) {
|
|
5543
5213
|
if (booleans === null) return;
|
|
5544
5214
|
if (internalId >= booleans.length) {
|
|
5545
|
-
const
|
|
5546
|
-
|
|
5547
|
-
|
|
5548
|
-
booleans =
|
|
5215
|
+
const grown2 = new Uint8Array(grow2(booleans.length, internalId));
|
|
5216
|
+
grown2.fill(BOOLEAN_MISSING, booleans.length);
|
|
5217
|
+
grown2.set(booleans);
|
|
5218
|
+
booleans = grown2;
|
|
5549
5219
|
}
|
|
5550
5220
|
booleans[internalId] = value;
|
|
5551
5221
|
}
|
|
@@ -5640,7 +5310,7 @@ function liveInternalIds(docStore) {
|
|
|
5640
5310
|
return ids;
|
|
5641
5311
|
}
|
|
5642
5312
|
function createSortColumnSet(docStore) {
|
|
5643
|
-
const
|
|
5313
|
+
const columns2 = /* @__PURE__ */ new Map();
|
|
5644
5314
|
function rebuildThreshold() {
|
|
5645
5315
|
const count = docStore.count();
|
|
5646
5316
|
const fraction = count >> SORT_COLUMN_REBUILD_FRACTION_SHIFT;
|
|
@@ -5665,7 +5335,7 @@ function createSortColumnSet(docStore) {
|
|
|
5665
5335
|
dirty: /* @__PURE__ */ new Set(),
|
|
5666
5336
|
dirtyStream: null
|
|
5667
5337
|
};
|
|
5668
|
-
|
|
5338
|
+
columns2.set(key, entry);
|
|
5669
5339
|
return entry;
|
|
5670
5340
|
}
|
|
5671
5341
|
function buildDirtyStream(entry) {
|
|
@@ -5719,11 +5389,11 @@ function createSortColumnSet(docStore) {
|
|
|
5719
5389
|
}
|
|
5720
5390
|
return {
|
|
5721
5391
|
holds(field, fieldType, mode) {
|
|
5722
|
-
return
|
|
5392
|
+
return columns2.has(columnKeyOf(field, fieldType, mode));
|
|
5723
5393
|
},
|
|
5724
5394
|
column(field, fieldType, mode) {
|
|
5725
5395
|
const key = columnKeyOf(field, fieldType, mode);
|
|
5726
|
-
let entry =
|
|
5396
|
+
let entry = columns2.get(key);
|
|
5727
5397
|
if (entry === void 0) {
|
|
5728
5398
|
entry = backfill(key, field, fieldType, mode);
|
|
5729
5399
|
} else if (entry.dirty.size > rebuildThreshold()) {
|
|
@@ -5732,14 +5402,14 @@ function createSortColumnSet(docStore) {
|
|
|
5732
5402
|
return viewOf(entry);
|
|
5733
5403
|
},
|
|
5734
5404
|
record(internalId, document) {
|
|
5735
|
-
for (const entry of
|
|
5405
|
+
for (const entry of columns2.values()) {
|
|
5736
5406
|
entry.store.set(internalId, toReducedSortValue(readSortField(document, entry.field), entry.mode));
|
|
5737
5407
|
entry.dirty.add(internalId);
|
|
5738
5408
|
entry.dirtyStream = null;
|
|
5739
5409
|
}
|
|
5740
5410
|
},
|
|
5741
5411
|
forget(internalId) {
|
|
5742
|
-
for (const entry of
|
|
5412
|
+
for (const entry of columns2.values()) {
|
|
5743
5413
|
entry.store.clear(internalId);
|
|
5744
5414
|
entry.dirty.delete(internalId);
|
|
5745
5415
|
entry.dirtyStream = null;
|
|
@@ -5747,16 +5417,16 @@ function createSortColumnSet(docStore) {
|
|
|
5747
5417
|
},
|
|
5748
5418
|
refresh() {
|
|
5749
5419
|
const threshold = rebuildThreshold();
|
|
5750
|
-
for (const entry of
|
|
5420
|
+
for (const entry of columns2.values()) {
|
|
5751
5421
|
if (entry.dirty.size > threshold) rebuild(entry);
|
|
5752
5422
|
}
|
|
5753
5423
|
},
|
|
5754
5424
|
fieldCount() {
|
|
5755
|
-
return
|
|
5425
|
+
return columns2.size;
|
|
5756
5426
|
},
|
|
5757
5427
|
estimateBytes() {
|
|
5758
5428
|
let bytes = 0;
|
|
5759
|
-
for (const entry of
|
|
5429
|
+
for (const entry of columns2.values()) {
|
|
5760
5430
|
bytes += entry.store.estimateBytes() + estimateOrderBytes(entry.order);
|
|
5761
5431
|
}
|
|
5762
5432
|
return bytes;
|
|
@@ -5825,26 +5495,26 @@ function compareColumnValue(column, aId, bId, direction) {
|
|
|
5825
5495
|
if (!rankIsBetweenValues(aRank)) return 0;
|
|
5826
5496
|
return compareComparableValues(column.valueOf(aId), column.valueOf(bId), direction);
|
|
5827
5497
|
}
|
|
5828
|
-
function compareCandidates(
|
|
5829
|
-
for (let i = 0; i <
|
|
5830
|
-
const comparison = compareColumnValue(
|
|
5498
|
+
function compareCandidates(columns2, directions, a, b) {
|
|
5499
|
+
for (let i = 0; i < columns2.length; i++) {
|
|
5500
|
+
const comparison = compareColumnValue(columns2[i], a.internalId, b.internalId, directions[i]);
|
|
5831
5501
|
if (comparison !== 0) return comparison;
|
|
5832
5502
|
}
|
|
5833
5503
|
return compareCodePoints(a.externalId, b.externalId);
|
|
5834
5504
|
}
|
|
5835
|
-
function compareWithAnchor(
|
|
5836
|
-
for (let i = 0; i <
|
|
5505
|
+
function compareWithAnchor(columns2, directions, candidate, anchorKey, anchorId) {
|
|
5506
|
+
for (let i = 0; i < columns2.length; i++) {
|
|
5837
5507
|
const comparison = compareComparableValues(candidate.key[i], anchorKey[i] ?? null, directions[i]);
|
|
5838
5508
|
if (comparison !== 0) return comparison;
|
|
5839
5509
|
}
|
|
5840
5510
|
return compareCodePoints(candidate.externalId, anchorId);
|
|
5841
5511
|
}
|
|
5842
|
-
function insertOrdered(page, entry,
|
|
5512
|
+
function insertOrdered(page, entry, columns2, directions) {
|
|
5843
5513
|
let low = 0;
|
|
5844
5514
|
let high = page.length;
|
|
5845
5515
|
while (low < high) {
|
|
5846
5516
|
const mid = low + high >>> 1;
|
|
5847
|
-
if (compareCandidates(
|
|
5517
|
+
if (compareCandidates(columns2, directions, page[mid], entry) <= 0) low = mid + 1;
|
|
5848
5518
|
else high = mid;
|
|
5849
5519
|
}
|
|
5850
5520
|
page.splice(low, 0, entry);
|
|
@@ -5934,11 +5604,11 @@ function selectSortedPage(state, request) {
|
|
|
5934
5604
|
if (limit <= 0 || fields.length === 0) return [];
|
|
5935
5605
|
const set = state.sortColumns;
|
|
5936
5606
|
if (set === null) return [];
|
|
5937
|
-
const
|
|
5607
|
+
const columns2 = fields.map(
|
|
5938
5608
|
(field, index) => set.column(field, fieldTypes[index], modes[index] ?? defaultSortMode(directions[index] ?? "asc"))
|
|
5939
5609
|
);
|
|
5940
5610
|
const resolver = state.docStore.resolver();
|
|
5941
|
-
const leading =
|
|
5611
|
+
const leading = columns2[0];
|
|
5942
5612
|
const leadingDirection = directions[0];
|
|
5943
5613
|
const anchorRank = anchorKey === null || anchorId === null ? null : rankOfValue(leading.order, anchorKey[0] ?? null);
|
|
5944
5614
|
const next = createStream(leading, leadingDirection, anchorRank);
|
|
@@ -5953,17 +5623,17 @@ function selectSortedPage(state, request) {
|
|
|
5953
5623
|
if (page.length === limit && rankOrdersAfter(leadingRank, page[limit - 1].leadingRank, leadingDirection)) {
|
|
5954
5624
|
break;
|
|
5955
5625
|
}
|
|
5956
|
-
const key = new Array(
|
|
5957
|
-
for (let i = 0; i <
|
|
5626
|
+
const key = new Array(columns2.length);
|
|
5627
|
+
for (let i = 0; i < columns2.length; i++) key[i] = columns2[i].valueOf(internalId);
|
|
5958
5628
|
const candidate = { internalId, externalId, leadingRank, key };
|
|
5959
5629
|
if (anchorKey !== null && anchorId !== null) {
|
|
5960
|
-
if (compareWithAnchor(
|
|
5630
|
+
if (compareWithAnchor(columns2, directions, candidate, anchorKey, anchorId) <= 0) continue;
|
|
5961
5631
|
}
|
|
5962
5632
|
if (page.length === limit) {
|
|
5963
|
-
if (compareCandidates(
|
|
5633
|
+
if (compareCandidates(columns2, directions, candidate, page[limit - 1]) >= 0) continue;
|
|
5964
5634
|
page.pop();
|
|
5965
5635
|
}
|
|
5966
|
-
insertOrdered(page, candidate,
|
|
5636
|
+
insertOrdered(page, candidate, columns2, directions);
|
|
5967
5637
|
}
|
|
5968
5638
|
return page.map((entry) => ({ id: entry.externalId, key: entry.key }));
|
|
5969
5639
|
}
|
|
@@ -6046,17 +5716,21 @@ function presentBitsetOf(state, fieldPath, type, capacity) {
|
|
|
6046
5716
|
if (base === "number") present = state.numericIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6047
5717
|
else if (base === "boolean") present = state.booleanIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6048
5718
|
else if (base === "enum") present = state.enumIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6049
|
-
else if (base === "geopoint") present = state
|
|
5719
|
+
else if (base === "geopoint") present = geoDocumentsBitset(state, fieldPath, capacity);
|
|
6050
5720
|
else if (base !== void 0 && keepsValueSize(base)) {
|
|
6051
5721
|
present = state.sizeIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
6052
5722
|
}
|
|
6053
5723
|
return present ?? createBitSet(capacity);
|
|
6054
5724
|
}
|
|
6055
|
-
function
|
|
5725
|
+
function bitsetOfDocuments(internalIds, capacity) {
|
|
6056
5726
|
const bits = createBitSet(capacity);
|
|
6057
|
-
for (const internalId of
|
|
5727
|
+
for (const internalId of internalIds) bitsetSet(bits, internalId);
|
|
6058
5728
|
return bits;
|
|
6059
5729
|
}
|
|
5730
|
+
function geoDocumentsBitset(state, fieldPath, capacity) {
|
|
5731
|
+
const geoIndex = state.geoIndexes.get(fieldPath);
|
|
5732
|
+
return geoIndex === void 0 ? void 0 : bitsetOfDocuments(geoIndex.documentIds(), capacity);
|
|
5733
|
+
}
|
|
6060
5734
|
function buildFilterContext(state, schema, meter) {
|
|
6061
5735
|
const flat = getFlatSchema(state, schema);
|
|
6062
5736
|
const capacity = state.docStore.internalIdCapacity();
|
|
@@ -6108,7 +5782,7 @@ function buildFilterContext(state, schema, meter) {
|
|
|
6108
5782
|
},
|
|
6109
5783
|
capacity,
|
|
6110
5784
|
get allDocIdsBitset() {
|
|
6111
|
-
cachedAllDocIdsBitset ??=
|
|
5785
|
+
cachedAllDocIdsBitset ??= bitsetOfDocuments(state.docStore.allInternalIds(), capacity);
|
|
6112
5786
|
return cachedAllDocIdsBitset;
|
|
6113
5787
|
}
|
|
6114
5788
|
};
|
|
@@ -6182,7 +5856,7 @@ function requireLoadedPhraseCode() {
|
|
|
6182
5856
|
}
|
|
6183
5857
|
function loadPhraseCode() {
|
|
6184
5858
|
if (loading === null) {
|
|
6185
|
-
loading = import('./phrase-module-
|
|
5859
|
+
loading = import('./phrase-module-QK22HSQ2.mjs').then(
|
|
6186
5860
|
(code) => {
|
|
6187
5861
|
loaded = code;
|
|
6188
5862
|
return code;
|
|
@@ -6216,10 +5890,82 @@ function analyseQueryPhrases(term, language, analysis) {
|
|
|
6216
5890
|
);
|
|
6217
5891
|
}
|
|
6218
5892
|
|
|
5893
|
+
// src/math/log.ts
|
|
5894
|
+
var EXPONENT_BIAS = 1023;
|
|
5895
|
+
var EXPONENT_SHIFT = 20;
|
|
5896
|
+
var SUBNORMAL_SCALE_EXPONENT = 54;
|
|
5897
|
+
var ROUNDING_OFFSET = 614244;
|
|
5898
|
+
var HALVING_BIT = 1048576;
|
|
5899
|
+
var ONE_HIGH_WORD = 1072693248;
|
|
5900
|
+
var SIGNIFICAND_MASK = 1048575;
|
|
5901
|
+
var NEAR_ONE_LIMIT = 3;
|
|
5902
|
+
var LOWER_SPLIT = 398458;
|
|
5903
|
+
var UPPER_SPLIT = 440401;
|
|
5904
|
+
var LN2_HI = 0.6931471803691238;
|
|
5905
|
+
var LN2_LO = 19082149292705877e-26;
|
|
5906
|
+
var TWO54 = 18014398509481984;
|
|
5907
|
+
var TWO = 2;
|
|
5908
|
+
var THIRD = 0.3333333333333333;
|
|
5909
|
+
var HALF = 0.5;
|
|
5910
|
+
var LG1 = 0.6666666666666735;
|
|
5911
|
+
var LG2 = 0.3999999999940942;
|
|
5912
|
+
var LG3 = 0.2857142874366239;
|
|
5913
|
+
var LG4 = 0.22222198432149784;
|
|
5914
|
+
var LG5 = 0.1818357216161805;
|
|
5915
|
+
var LG6 = 0.15313837699209373;
|
|
5916
|
+
var LG7 = 0.14798198605116586;
|
|
5917
|
+
function nearOne(fraction, scale) {
|
|
5918
|
+
if (fraction === 0) {
|
|
5919
|
+
if (scale === 0) return 0;
|
|
5920
|
+
return scale * LN2_HI + scale * LN2_LO;
|
|
5921
|
+
}
|
|
5922
|
+
const cubic = fraction * fraction * (HALF - THIRD * fraction);
|
|
5923
|
+
if (scale === 0) return fraction - cubic;
|
|
5924
|
+
return scale * LN2_HI - (cubic - scale * LN2_LO - fraction);
|
|
5925
|
+
}
|
|
5926
|
+
function series(fraction, scale, significand) {
|
|
5927
|
+
const ratio = fraction / (TWO + fraction);
|
|
5928
|
+
const square = ratio * ratio;
|
|
5929
|
+
const below = significand - LOWER_SPLIT;
|
|
5930
|
+
const fourth = square * square;
|
|
5931
|
+
const above = UPPER_SPLIT - significand;
|
|
5932
|
+
const even = fourth * (LG2 + fourth * (LG4 + fourth * LG6));
|
|
5933
|
+
const odd = square * (LG1 + fourth * (LG3 + fourth * (LG5 + fourth * LG7)));
|
|
5934
|
+
const poly = odd + even;
|
|
5935
|
+
if ((below | above) > 0) {
|
|
5936
|
+
const halfSquare = HALF * fraction * fraction;
|
|
5937
|
+
if (scale === 0) return fraction - (halfSquare - ratio * (halfSquare + poly));
|
|
5938
|
+
return scale * LN2_HI - (halfSquare - (ratio * (halfSquare + poly) + scale * LN2_LO) - fraction);
|
|
5939
|
+
}
|
|
5940
|
+
if (scale === 0) return fraction - ratio * (fraction - poly);
|
|
5941
|
+
return scale * LN2_HI - (ratio * (fraction - poly) - scale * LN2_LO - fraction);
|
|
5942
|
+
}
|
|
5943
|
+
function log(value) {
|
|
5944
|
+
let high = highWord(value);
|
|
5945
|
+
let scale = 0;
|
|
5946
|
+
let scaled = value;
|
|
5947
|
+
if (high < SMALLEST_NORMAL_HIGH_WORD) {
|
|
5948
|
+
if ((high & MAGNITUDE_MASK | lowWord(value)) === 0) return Number.NEGATIVE_INFINITY;
|
|
5949
|
+
if (high < 0) return Number.NaN;
|
|
5950
|
+
scale -= SUBNORMAL_SCALE_EXPONENT;
|
|
5951
|
+
scaled *= TWO54;
|
|
5952
|
+
high = highWord(scaled);
|
|
5953
|
+
}
|
|
5954
|
+
if (high >= INFINITE_HIGH_WORD) return scaled + scaled;
|
|
5955
|
+
scale += (high >> EXPONENT_SHIFT) - EXPONENT_BIAS;
|
|
5956
|
+
const significand = high & SIGNIFICAND_MASK;
|
|
5957
|
+
const halving = significand + ROUNDING_OFFSET & HALVING_BIT;
|
|
5958
|
+
scaled = withHighWord(scaled, significand | halving ^ ONE_HIGH_WORD);
|
|
5959
|
+
scale += halving >> EXPONENT_SHIFT;
|
|
5960
|
+
const fraction = scaled - 1;
|
|
5961
|
+
if ((SIGNIFICAND_MASK & 2 + significand) < NEAR_ONE_LIMIT) return nearOne(fraction, scale);
|
|
5962
|
+
return series(fraction, scale, significand);
|
|
5963
|
+
}
|
|
5964
|
+
|
|
6219
5965
|
// src/core/scorer.ts
|
|
6220
5966
|
function computeIDF(docFrequency, totalDocs) {
|
|
6221
5967
|
if (totalDocs === 0) return 0;
|
|
6222
|
-
return
|
|
5968
|
+
return log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
6223
5969
|
}
|
|
6224
5970
|
function pairKeepsBoundSound(k1, b) {
|
|
6225
5971
|
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
@@ -6283,6 +6029,43 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
6283
6029
|
return idf * (numerator / denominator);
|
|
6284
6030
|
}
|
|
6285
6031
|
|
|
6032
|
+
// src/core/partition/field-order.ts
|
|
6033
|
+
var fieldRun = { entries: new Uint32Array(FIELD_RUN_ENTRIES_START), length: 0 };
|
|
6034
|
+
var ranksByFieldNames = /* @__PURE__ */ new WeakMap();
|
|
6035
|
+
var ONLY_FIELD_RANKS = new Uint32Array(1);
|
|
6036
|
+
function fieldNameRanks(fieldNames) {
|
|
6037
|
+
if (fieldNames.length < 2) return ONLY_FIELD_RANKS;
|
|
6038
|
+
const held = ranksByFieldNames.get(fieldNames);
|
|
6039
|
+
if (held !== void 0 && held.length === fieldNames.length) return held;
|
|
6040
|
+
const order = Array.from(fieldNames.keys()).sort(
|
|
6041
|
+
(left, right) => compareCodePoints(fieldNames[left], fieldNames[right])
|
|
6042
|
+
);
|
|
6043
|
+
const ranks = new Uint32Array(order.length);
|
|
6044
|
+
for (let rank = 0; rank < order.length; rank++) ranks[order[rank]] = rank;
|
|
6045
|
+
ranksByFieldNames.set(fieldNames, ranks);
|
|
6046
|
+
return ranks;
|
|
6047
|
+
}
|
|
6048
|
+
function orderRunByFieldName(docIds, fieldIndices, ranks, start, count) {
|
|
6049
|
+
let entries = fieldRun.entries;
|
|
6050
|
+
let end = start;
|
|
6051
|
+
for (; end < count && docIds[end] === docIds[start]; end++) {
|
|
6052
|
+
let slot = end - start;
|
|
6053
|
+
if (slot === entries.length) {
|
|
6054
|
+
const grown2 = new Uint32Array(slot * 2);
|
|
6055
|
+
grown2.set(entries);
|
|
6056
|
+
entries = fieldRun.entries = grown2;
|
|
6057
|
+
}
|
|
6058
|
+
const rank = ranks[fieldIndices[end]];
|
|
6059
|
+
while (slot > 0 && ranks[fieldIndices[entries[slot - 1]]] > rank) {
|
|
6060
|
+
entries[slot] = entries[slot - 1];
|
|
6061
|
+
slot--;
|
|
6062
|
+
}
|
|
6063
|
+
entries[slot] = end;
|
|
6064
|
+
}
|
|
6065
|
+
fieldRun.length = end - start;
|
|
6066
|
+
return entries;
|
|
6067
|
+
}
|
|
6068
|
+
|
|
6286
6069
|
// src/core/partition/field-scoring.ts
|
|
6287
6070
|
var NO_FIELD_VALUES = new Float64Array(0);
|
|
6288
6071
|
function differentPairsOf(searchable, k1s, bs) {
|
|
@@ -6310,7 +6093,8 @@ function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
|
6310
6093
|
lengthColumns: new Array(fieldCount).fill(null),
|
|
6311
6094
|
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
6312
6095
|
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
6313
|
-
boundPairs: [k1, b]
|
|
6096
|
+
boundPairs: [k1, b],
|
|
6097
|
+
nameRanks: fieldNameRanks(fieldNames)
|
|
6314
6098
|
};
|
|
6315
6099
|
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
6316
6100
|
const fieldName = fieldNames[fieldIndex];
|
|
@@ -6327,8 +6111,8 @@ function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
|
6327
6111
|
if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
|
|
6328
6112
|
return scoring;
|
|
6329
6113
|
}
|
|
6330
|
-
function fieldLengthOf(
|
|
6331
|
-
const column = fieldIndex <
|
|
6114
|
+
function fieldLengthOf(columns2, fieldIndex, internalId, averageLength) {
|
|
6115
|
+
const column = fieldIndex < columns2.length ? columns2[fieldIndex] : null;
|
|
6332
6116
|
if (column === null || internalId >= column.length) return averageLength;
|
|
6333
6117
|
const stored = column[internalId];
|
|
6334
6118
|
return stored > 0 ? stored : averageLength;
|
|
@@ -6413,20 +6197,26 @@ function termGroupOf(groups, token) {
|
|
|
6413
6197
|
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
6414
6198
|
if (group !== void 0) {
|
|
6415
6199
|
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
6416
|
-
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true);
|
|
6200
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true).sort(byToken);
|
|
6201
|
+
}
|
|
6202
|
+
if (!exact) {
|
|
6203
|
+
const matches = index.fuzzyLookup(token, tolerance, prefixLength);
|
|
6204
|
+
return matches.length > 1 ? matches.sort(byToken) : matches;
|
|
6417
6205
|
}
|
|
6418
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
6419
6206
|
const postingList = index.lookup(token);
|
|
6420
6207
|
return postingList ? [{ token, postingList }] : [];
|
|
6421
6208
|
}
|
|
6209
|
+
function byToken(left, right) {
|
|
6210
|
+
return compareCodePoints(left.token, right.token);
|
|
6211
|
+
}
|
|
6422
6212
|
function resolveGroupMatches(context, group) {
|
|
6423
|
-
const
|
|
6213
|
+
const frequencyOf2 = (token) => context.globalFrequency(token) ?? context.index.documentFrequency(token);
|
|
6424
6214
|
const lookup = (token) => context.index.lookup(token);
|
|
6425
6215
|
const synonyms = group.synonyms;
|
|
6426
|
-
let groupDocumentFrequency = synonyms?.documentFrequency(
|
|
6216
|
+
let groupDocumentFrequency = synonyms?.documentFrequency(frequencyOf2) ?? 0;
|
|
6427
6217
|
const found = [];
|
|
6428
6218
|
for (const member of group.members) {
|
|
6429
|
-
const documents =
|
|
6219
|
+
const documents = frequencyOf2(member.token);
|
|
6430
6220
|
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
6431
6221
|
const postingList = synonyms !== void 0 && member.token === group.token ? synonyms.postings(lookup, member.token) : lookup(member.token);
|
|
6432
6222
|
if (postingList === void 0) continue;
|
|
@@ -6451,10 +6241,20 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
6451
6241
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
6452
6242
|
match.postingList
|
|
6453
6243
|
);
|
|
6454
|
-
|
|
6455
|
-
|
|
6244
|
+
let runStart = 0;
|
|
6245
|
+
let runEnd = 0;
|
|
6246
|
+
for (let at = 0; at < count; at++) {
|
|
6247
|
+
const internalId = docIds[at];
|
|
6456
6248
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
6457
6249
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
6250
|
+
let pi = at;
|
|
6251
|
+
if (at < runEnd) {
|
|
6252
|
+
pi = fieldRun.entries[at - runStart];
|
|
6253
|
+
} else if (at + 2 < count && docIds[at + 2] === internalId) {
|
|
6254
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, fields.nameRanks, at, count)[0];
|
|
6255
|
+
runStart = at;
|
|
6256
|
+
runEnd = at + fieldRun.length;
|
|
6257
|
+
}
|
|
6458
6258
|
const fieldIndex = fieldNameIndices[pi];
|
|
6459
6259
|
if (fields.searchable[fieldIndex] === 0) continue;
|
|
6460
6260
|
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
@@ -6732,7 +6532,7 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
6732
6532
|
for (const [fieldPath, index] of source.geoIndexes) {
|
|
6733
6533
|
let targetIndex = target.geoIndexes.get(fieldPath);
|
|
6734
6534
|
if (targetIndex === void 0) {
|
|
6735
|
-
targetIndex =
|
|
6535
|
+
targetIndex = createLoadedGeoIndex();
|
|
6736
6536
|
target.geoIndexes.set(fieldPath, targetIndex);
|
|
6737
6537
|
}
|
|
6738
6538
|
for (const entry of index.serialize()) {
|
|
@@ -6763,8 +6563,8 @@ function mergeSegmentState(target, source) {
|
|
|
6763
6563
|
// src/core/partition/block-bounds.ts
|
|
6764
6564
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
6765
6565
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
6766
|
-
function fieldLengthBound(
|
|
6767
|
-
const column = fieldIndex <
|
|
6566
|
+
function fieldLengthBound(columns2, fieldIndex, internalId) {
|
|
6567
|
+
const column = fieldIndex < columns2.length ? columns2[fieldIndex] : null;
|
|
6768
6568
|
if (column === null || internalId >= column.length) return ABSENT_FIELD_LENGTH_BOUND;
|
|
6769
6569
|
const stored = column[internalId];
|
|
6770
6570
|
return stored > 0 ? stored : ABSENT_FIELD_LENGTH_BOUND;
|
|
@@ -6772,7 +6572,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
|
|
|
6772
6572
|
function entriesCovered(bounds) {
|
|
6773
6573
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
6774
6574
|
}
|
|
6775
|
-
function scanPostingBlocks(list,
|
|
6575
|
+
function scanPostingBlocks(list, columns2, previous, keepBlocks) {
|
|
6776
6576
|
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
6777
6577
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
6778
6578
|
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
@@ -6806,7 +6606,7 @@ function scanPostingBlocks(list, columns, previous, keepBlocks) {
|
|
|
6806
6606
|
const position = entry + run;
|
|
6807
6607
|
const termFrequency = termFrequencies[position];
|
|
6808
6608
|
if (termFrequency > blockMaxTf) blockMaxTf = termFrequency;
|
|
6809
|
-
const fieldLength = fieldLengthBound(
|
|
6609
|
+
const fieldLength = fieldLengthBound(columns2, fieldNameIndices[position], docId);
|
|
6810
6610
|
if (fieldLength < blockMinLen) blockMinLen = fieldLength;
|
|
6811
6611
|
}
|
|
6812
6612
|
run++;
|
|
@@ -6840,19 +6640,19 @@ function scanPostingBlocks(list, columns, previous, keepBlocks) {
|
|
|
6840
6640
|
documentCount
|
|
6841
6641
|
};
|
|
6842
6642
|
}
|
|
6843
|
-
function blockBoundsFor(list,
|
|
6643
|
+
function blockBoundsFor(list, columns2) {
|
|
6844
6644
|
const existing = boundsByList.get(list);
|
|
6845
6645
|
if (existing !== void 0 && existing.structureRevision === list.structureRevision) {
|
|
6846
6646
|
const covered = entriesCovered(existing);
|
|
6847
6647
|
if (covered === list.length) return existing;
|
|
6848
6648
|
if (list.ordered && covered < list.length) {
|
|
6849
|
-
const extended = scanPostingBlocks(list,
|
|
6649
|
+
const extended = scanPostingBlocks(list, columns2, existing, Math.max(0, existing.blockCount - 1));
|
|
6850
6650
|
boundsByList.set(list, extended);
|
|
6851
6651
|
return extended;
|
|
6852
6652
|
}
|
|
6853
6653
|
}
|
|
6854
6654
|
const stored = list.storedBlockBounds?.() ?? null;
|
|
6855
|
-
const built = stored === null ? scanPostingBlocks(list,
|
|
6655
|
+
const built = stored === null ? scanPostingBlocks(list, columns2, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
6856
6656
|
boundsByList.set(list, built);
|
|
6857
6657
|
return built;
|
|
6858
6658
|
}
|
|
@@ -6913,9 +6713,9 @@ function beginScoring(buffer, capacity) {
|
|
|
6913
6713
|
}
|
|
6914
6714
|
function appendTouched(buffer, internalId) {
|
|
6915
6715
|
if (buffer.touchedCount === buffer.touched.length) {
|
|
6916
|
-
const
|
|
6917
|
-
|
|
6918
|
-
buffer.touched =
|
|
6716
|
+
const grown2 = new Int32Array(buffer.touched.length * 2);
|
|
6717
|
+
grown2.set(buffer.touched);
|
|
6718
|
+
buffer.touched = grown2;
|
|
6919
6719
|
}
|
|
6920
6720
|
buffer.touched[buffer.touchedCount] = internalId;
|
|
6921
6721
|
buffer.touchedCount++;
|
|
@@ -7005,13 +6805,13 @@ function topKFromBuffer(buffer, k, resolver, components) {
|
|
|
7005
6805
|
const result = new Array(heap.length);
|
|
7006
6806
|
for (let index = 0; index < heap.length; index++) {
|
|
7007
6807
|
const entry = heap[index];
|
|
7008
|
-
const
|
|
6808
|
+
const record2 = components?.get(entry.internalId);
|
|
7009
6809
|
result[index] = {
|
|
7010
6810
|
docId: entry.externalId,
|
|
7011
6811
|
score: entry.score,
|
|
7012
|
-
termFrequencies:
|
|
7013
|
-
fieldLengths:
|
|
7014
|
-
idf:
|
|
6812
|
+
termFrequencies: record2?.termFrequencies ?? EMPTY_COMPONENTS,
|
|
6813
|
+
fieldLengths: record2?.fieldLengths ?? EMPTY_COMPONENTS,
|
|
6814
|
+
idf: record2?.idf ?? EMPTY_COMPONENTS
|
|
7015
6815
|
};
|
|
7016
6816
|
}
|
|
7017
6817
|
return result;
|
|
@@ -7081,20 +6881,20 @@ function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
|
7081
6881
|
return best;
|
|
7082
6882
|
}
|
|
7083
6883
|
function samePairEntryScore(scan, entry, internalId) {
|
|
7084
|
-
const { fields, columns } = scan;
|
|
7085
|
-
const fieldIndex =
|
|
6884
|
+
const { fields, columns: columns2 } = scan;
|
|
6885
|
+
const fieldIndex = columns2.fieldNameIndices[entry];
|
|
7086
6886
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
7087
6887
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
7088
|
-
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6888
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns2.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
|
|
7089
6889
|
return bm25 * fields.boosts[fieldIndex];
|
|
7090
6890
|
}
|
|
7091
6891
|
function fieldPairEntryScore(scan, entry, internalId) {
|
|
7092
|
-
const { fields, columns } = scan;
|
|
7093
|
-
const fieldIndex =
|
|
6892
|
+
const { fields, columns: columns2 } = scan;
|
|
6893
|
+
const fieldIndex = columns2.fieldNameIndices[entry];
|
|
7094
6894
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
7095
6895
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
7096
6896
|
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7097
|
-
|
|
6897
|
+
columns2.termFrequencies[entry],
|
|
7098
6898
|
scan.idf,
|
|
7099
6899
|
fieldLength,
|
|
7100
6900
|
averageLength,
|
|
@@ -7159,15 +6959,30 @@ function firstEntryOf(docIds, count, internalId) {
|
|
|
7159
6959
|
}
|
|
7160
6960
|
return low;
|
|
7161
6961
|
}
|
|
6962
|
+
function fieldOrderedRunScore(scan, start, internalId, total, entryScore) {
|
|
6963
|
+
const { docIds, fieldNameIndices, count } = scan.columns;
|
|
6964
|
+
const ordered = orderRunByFieldName(docIds, fieldNameIndices, scan.fields.nameRanks, start, count);
|
|
6965
|
+
let score = total;
|
|
6966
|
+
for (let at = 0; at < fieldRun.length; at++) {
|
|
6967
|
+
const row = ordered[at];
|
|
6968
|
+
if (scan.fields.searchable[fieldNameIndices[row]] !== 1) continue;
|
|
6969
|
+
score += entryScore(scan, row, internalId);
|
|
6970
|
+
}
|
|
6971
|
+
return score;
|
|
6972
|
+
}
|
|
7162
6973
|
function samePairQueryOrderScore(scans, internalId) {
|
|
7163
6974
|
let score = 0;
|
|
7164
6975
|
for (const scan of scans) {
|
|
7165
6976
|
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
7166
6977
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7167
|
-
|
|
7168
|
-
|
|
7169
|
-
score
|
|
6978
|
+
const start = firstEntryOf(docIds, count, internalId);
|
|
6979
|
+
if (start + 1 < count && docIds[start + 1] === internalId) {
|
|
6980
|
+
score = fieldOrderedRunScore(scan, start, internalId, score, samePairEntryScore);
|
|
6981
|
+
continue;
|
|
7170
6982
|
}
|
|
6983
|
+
if (start === count || docIds[start] !== internalId) continue;
|
|
6984
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[start]] !== 1) continue;
|
|
6985
|
+
score += samePairEntryScore(scan, start, internalId);
|
|
7171
6986
|
}
|
|
7172
6987
|
return score;
|
|
7173
6988
|
}
|
|
@@ -7176,10 +6991,14 @@ function fieldPairQueryOrderScore(scans, internalId) {
|
|
|
7176
6991
|
for (const scan of scans) {
|
|
7177
6992
|
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
7178
6993
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7179
|
-
|
|
7180
|
-
|
|
7181
|
-
score
|
|
6994
|
+
const start = firstEntryOf(docIds, count, internalId);
|
|
6995
|
+
if (start + 1 < count && docIds[start + 1] === internalId) {
|
|
6996
|
+
score = fieldOrderedRunScore(scan, start, internalId, score, fieldPairEntryScore);
|
|
6997
|
+
continue;
|
|
7182
6998
|
}
|
|
6999
|
+
if (start === count || docIds[start] !== internalId) continue;
|
|
7000
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[start]] !== 1) continue;
|
|
7001
|
+
score += fieldPairEntryScore(scan, start, internalId);
|
|
7183
7002
|
}
|
|
7184
7003
|
return score;
|
|
7185
7004
|
}
|
|
@@ -7317,7 +7136,8 @@ function singleTermTopK(request) {
|
|
|
7317
7136
|
lengthColumns: fieldLengthColumns,
|
|
7318
7137
|
k1s,
|
|
7319
7138
|
bs,
|
|
7320
|
-
boundPairs
|
|
7139
|
+
boundPairs,
|
|
7140
|
+
nameRanks
|
|
7321
7141
|
} = request.fields;
|
|
7322
7142
|
const onePair = boundPairs.length === 2;
|
|
7323
7143
|
const onlyK1 = boundPairs[0];
|
|
@@ -7357,12 +7177,14 @@ function singleTermTopK(request) {
|
|
|
7357
7177
|
}
|
|
7358
7178
|
let score = 0;
|
|
7359
7179
|
let scored2 = false;
|
|
7360
|
-
|
|
7361
|
-
|
|
7180
|
+
const ordered = entry + 2 < end && docIds[entry + 2] === internalId ? orderRunByFieldName(docIds, fieldNameIndices, nameRanks, entry, end) : null;
|
|
7181
|
+
for (let at = 0; entry < end && docIds[entry] === internalId; at++, entry++) {
|
|
7182
|
+
const row = ordered === null ? entry : ordered[at];
|
|
7183
|
+
const fieldIndex = fieldNameIndices[row];
|
|
7362
7184
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
7363
7185
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
7364
7186
|
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7365
|
-
termFrequencies[
|
|
7187
|
+
termFrequencies[row],
|
|
7366
7188
|
idf,
|
|
7367
7189
|
fieldLength,
|
|
7368
7190
|
fieldAvgLengths[fieldIndex],
|
|
@@ -7371,7 +7193,6 @@ function singleTermTopK(request) {
|
|
|
7371
7193
|
)) * fieldBoosts[fieldIndex];
|
|
7372
7194
|
scored2 = true;
|
|
7373
7195
|
}
|
|
7374
|
-
entry++;
|
|
7375
7196
|
}
|
|
7376
7197
|
if (!scored2) continue;
|
|
7377
7198
|
totalMatched++;
|
|
@@ -7411,6 +7232,10 @@ function singleTermTopK(request) {
|
|
|
7411
7232
|
}
|
|
7412
7233
|
|
|
7413
7234
|
// src/core/partition/search.ts
|
|
7235
|
+
var nativeKeywordSearch = null;
|
|
7236
|
+
function installNativeKeywordSearch(search) {
|
|
7237
|
+
nativeKeywordSearch = search;
|
|
7238
|
+
}
|
|
7414
7239
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
7415
7240
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
7416
7241
|
}
|
|
@@ -7437,15 +7262,11 @@ function searchFulltext(state, params) {
|
|
|
7437
7262
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
7438
7263
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
7439
7264
|
const scoresAreZero = totalDocs === 0;
|
|
7440
|
-
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
7441
|
-
const scoreBuffer = state.scoreBuffer;
|
|
7442
|
-
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
7443
|
-
const components = collectComponents ? /* @__PURE__ */ new Map() : null;
|
|
7444
7265
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
7445
7266
|
const fieldNames = state.fieldNameTable.names;
|
|
7446
7267
|
const resolver = state.docStore.resolver();
|
|
7447
7268
|
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
7448
|
-
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
7269
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs, nameRanks } = fieldScoring;
|
|
7449
7270
|
const onePair = boundPairs.length === 2;
|
|
7450
7271
|
const onlyK1 = boundPairs[0];
|
|
7451
7272
|
const onlyB = boundPairs[1];
|
|
@@ -7461,7 +7282,13 @@ function searchFulltext(state, params) {
|
|
|
7461
7282
|
if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
|
|
7462
7283
|
return { scored: [], totalMatched: 0 };
|
|
7463
7284
|
}
|
|
7285
|
+
const inTheCore = nativeKeywordSearch?.(state, params, phraseMatches);
|
|
7286
|
+
if (inTheCore) return inTheCore;
|
|
7464
7287
|
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
7288
|
+
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
7289
|
+
const scoreBuffer = state.scoreBuffer;
|
|
7290
|
+
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
7291
|
+
const components = collectComponents ? /* @__PURE__ */ new Map() : null;
|
|
7465
7292
|
function documentFrequency(token, list) {
|
|
7466
7293
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
7467
7294
|
}
|
|
@@ -7470,25 +7297,35 @@ function searchFulltext(state, params) {
|
|
|
7470
7297
|
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
7471
7298
|
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
7472
7299
|
}
|
|
7473
|
-
for (let
|
|
7474
|
-
const
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
7478
|
-
|
|
7479
|
-
|
|
7480
|
-
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7481
|
-
frequency,
|
|
7482
|
-
idf,
|
|
7483
|
-
fieldLength,
|
|
7484
|
-
avgLen,
|
|
7485
|
-
onePair ? onlyK1 : k1s[fieldIndex],
|
|
7486
|
-
onePair ? onlyB : bs[fieldIndex]
|
|
7300
|
+
for (let start = 0; start < rows.docIds.length; start += fieldRun.length) {
|
|
7301
|
+
const entries = orderRunByFieldName(
|
|
7302
|
+
rows.docIds.values,
|
|
7303
|
+
rows.fieldIndices.values,
|
|
7304
|
+
nameRanks,
|
|
7305
|
+
start,
|
|
7306
|
+
rows.docIds.length
|
|
7487
7307
|
);
|
|
7488
|
-
|
|
7489
|
-
|
|
7490
|
-
|
|
7491
|
-
|
|
7308
|
+
for (let at = 0; at < fieldRun.length; at++) scorePhraseRow(rows, matched, entries[at], idf);
|
|
7309
|
+
}
|
|
7310
|
+
}
|
|
7311
|
+
function scorePhraseRow(rows, matched, row, idf) {
|
|
7312
|
+
const internalId = rows.docIds.values[row];
|
|
7313
|
+
if (!bitsetHas(matched, internalId)) return;
|
|
7314
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
7315
|
+
const frequency = rows.frequencies.values[row];
|
|
7316
|
+
const avgLen = averageLengths[fieldIndex];
|
|
7317
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
7318
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7319
|
+
frequency,
|
|
7320
|
+
idf,
|
|
7321
|
+
fieldLength,
|
|
7322
|
+
avgLen,
|
|
7323
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
7324
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
7325
|
+
);
|
|
7326
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
7327
|
+
if (components !== null) {
|
|
7328
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
7492
7329
|
}
|
|
7493
7330
|
}
|
|
7494
7331
|
const groupContext = {
|
|
@@ -7511,13 +7348,23 @@ function searchFulltext(state, params) {
|
|
|
7511
7348
|
const docFreq = documentFrequency(token, postingList);
|
|
7512
7349
|
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
7513
7350
|
}
|
|
7514
|
-
function scorePostings(match,
|
|
7351
|
+
function scorePostings(match, only) {
|
|
7515
7352
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
7516
7353
|
match.postingList
|
|
7517
7354
|
);
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
|
|
7355
|
+
let runStart = 0;
|
|
7356
|
+
let runEnd = 0;
|
|
7357
|
+
if (only === null && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
7358
|
+
for (let at = 0; at < count; at++) {
|
|
7359
|
+
const internalId = docIds[at];
|
|
7360
|
+
let pi = at;
|
|
7361
|
+
if (at < runEnd) {
|
|
7362
|
+
pi = fieldRun.entries[at - runStart];
|
|
7363
|
+
} else if (at + 1 < count && docIds[at + 1] === internalId) {
|
|
7364
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, nameRanks, at, count)[0];
|
|
7365
|
+
runStart = at;
|
|
7366
|
+
runEnd = at + fieldRun.length;
|
|
7367
|
+
}
|
|
7521
7368
|
const fieldIndex = fieldNameIndices[pi];
|
|
7522
7369
|
if (searchable[fieldIndex] === 0) continue;
|
|
7523
7370
|
const avgLen = averageLengths[fieldIndex];
|
|
@@ -7534,11 +7381,19 @@ function searchFulltext(state, params) {
|
|
|
7534
7381
|
}
|
|
7535
7382
|
return;
|
|
7536
7383
|
}
|
|
7537
|
-
for (let
|
|
7538
|
-
const internalId = docIds[
|
|
7384
|
+
for (let at = 0; at < count; at++) {
|
|
7385
|
+
const internalId = docIds[at];
|
|
7539
7386
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
7540
7387
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
7541
|
-
if (
|
|
7388
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
7389
|
+
let pi = at;
|
|
7390
|
+
if (at < runEnd) {
|
|
7391
|
+
pi = fieldRun.entries[at - runStart];
|
|
7392
|
+
} else if (at + 1 < count && docIds[at + 1] === internalId) {
|
|
7393
|
+
pi = orderRunByFieldName(docIds, fieldNameIndices, nameRanks, at, count)[0];
|
|
7394
|
+
runStart = at;
|
|
7395
|
+
runEnd = at + fieldRun.length;
|
|
7396
|
+
}
|
|
7542
7397
|
const fieldIndex = fieldNameIndices[pi];
|
|
7543
7398
|
if (searchable[fieldIndex] === 0) continue;
|
|
7544
7399
|
const termFrequency = termFrequencies[pi];
|
|
@@ -7566,10 +7421,10 @@ function searchFulltext(state, params) {
|
|
|
7566
7421
|
}
|
|
7567
7422
|
}
|
|
7568
7423
|
}
|
|
7569
|
-
function scoreGroup(group,
|
|
7424
|
+
function scoreGroup(group, only) {
|
|
7570
7425
|
if (group === null) return;
|
|
7571
7426
|
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
7572
|
-
if (
|
|
7427
|
+
if (only !== null && !bitsetHas(only, internalId)) continue;
|
|
7573
7428
|
mergeGroupContribution(internalId, contribution);
|
|
7574
7429
|
}
|
|
7575
7430
|
}
|
|
@@ -7610,19 +7465,27 @@ function searchFulltext(state, params) {
|
|
|
7610
7465
|
}
|
|
7611
7466
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
7612
7467
|
}
|
|
7613
|
-
|
|
7614
|
-
for (
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7468
|
+
let anchor = resolved[0];
|
|
7469
|
+
for (const term of resolved) if (term.totalPostings < anchor.totalPostings) anchor = term;
|
|
7470
|
+
for (const match of anchor.matches) scorePostings(match, null);
|
|
7471
|
+
scoreGroup(anchor.group, null);
|
|
7472
|
+
const anchorDocuments = createBitSet(state.docStore.internalIdCapacity());
|
|
7473
|
+
for (let index = 0; index < scoreBuffer.touchedCount; index++) {
|
|
7474
|
+
bitsetSet(anchorDocuments, scoreBuffer.touched[index]);
|
|
7475
|
+
}
|
|
7476
|
+
beginScoring(scoreBuffer, state.docStore.internalIdCapacity());
|
|
7477
|
+
components?.clear();
|
|
7478
|
+
for (const term of resolved) {
|
|
7479
|
+
for (const match of term.matches) scorePostings(match, anchorDocuments);
|
|
7480
|
+
scoreGroup(term.group, anchorDocuments);
|
|
7618
7481
|
}
|
|
7619
7482
|
} else {
|
|
7620
7483
|
for (const qt of queryTokens) {
|
|
7621
7484
|
const group = termGroupOf(termGroups, qt.token);
|
|
7622
7485
|
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7623
|
-
scorePostings(resolveMatch(m.token, m.postingList),
|
|
7486
|
+
scorePostings(resolveMatch(m.token, m.postingList), null);
|
|
7624
7487
|
}
|
|
7625
|
-
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group),
|
|
7488
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), null);
|
|
7626
7489
|
}
|
|
7627
7490
|
}
|
|
7628
7491
|
if (phraseMatches !== null) {
|
|
@@ -7748,7 +7611,7 @@ function lookupReadOnce(state, token) {
|
|
|
7748
7611
|
}
|
|
7749
7612
|
function encodeFieldLengths(state, remap, documentCount) {
|
|
7750
7613
|
const names = [];
|
|
7751
|
-
const
|
|
7614
|
+
const columns2 = [];
|
|
7752
7615
|
const totals = {};
|
|
7753
7616
|
for (const fieldName of Object.keys(state.stats.totalFieldLengths)) {
|
|
7754
7617
|
const column = state.docStore.fieldLengthColumn(fieldName);
|
|
@@ -7762,10 +7625,10 @@ function encodeFieldLengths(state, remap, documentCount) {
|
|
|
7762
7625
|
total += column[internalId];
|
|
7763
7626
|
}
|
|
7764
7627
|
names.push(fieldName);
|
|
7765
|
-
|
|
7628
|
+
columns2.push(packed);
|
|
7766
7629
|
totals[fieldName] = total;
|
|
7767
7630
|
}
|
|
7768
|
-
return { names, columns, totals };
|
|
7631
|
+
return { names, columns: columns2, totals };
|
|
7769
7632
|
}
|
|
7770
7633
|
function encodeSegmentState(state) {
|
|
7771
7634
|
const { remap, docIds } = buildOrdinalRemap(state);
|
|
@@ -7908,7 +7771,7 @@ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
|
7908
7771
|
for (const entry of payload.geo) {
|
|
7909
7772
|
let index = target.geoIndexes.get(entry.fieldPath);
|
|
7910
7773
|
if (index === void 0) {
|
|
7911
|
-
index =
|
|
7774
|
+
index = createLoadedGeoIndex();
|
|
7912
7775
|
target.geoIndexes.set(entry.fieldPath, index);
|
|
7913
7776
|
}
|
|
7914
7777
|
for (let i = 0; i < entry.docIds.length; i++) {
|
|
@@ -8142,7 +8005,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8142
8005
|
state.enumIndexes.set(path, idx);
|
|
8143
8006
|
}
|
|
8144
8007
|
for (const [path, entries] of Object.entries(data.fieldIndexes.geopoint)) {
|
|
8145
|
-
const geoIdx =
|
|
8008
|
+
const geoIdx = createLoadedGeoIndex();
|
|
8146
8009
|
const converted = entries.map((e) => ({
|
|
8147
8010
|
lat: e.lat,
|
|
8148
8011
|
lon: e.lon,
|
|
@@ -9474,22 +9337,22 @@ function createWriteLogRecorder() {
|
|
|
9474
9337
|
const current = [null, null];
|
|
9475
9338
|
return {
|
|
9476
9339
|
open(owner) {
|
|
9477
|
-
const
|
|
9478
|
-
current[owner] =
|
|
9340
|
+
const log2 = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
9341
|
+
current[owner] = log2;
|
|
9479
9342
|
return {
|
|
9480
9343
|
get partitionsReplaced() {
|
|
9481
|
-
return
|
|
9344
|
+
return log2.replaced;
|
|
9482
9345
|
},
|
|
9483
9346
|
get pendingDocuments() {
|
|
9484
|
-
return
|
|
9347
|
+
return log2.docIds.size;
|
|
9485
9348
|
},
|
|
9486
9349
|
takeWrittenDocIds() {
|
|
9487
|
-
const taken =
|
|
9488
|
-
|
|
9350
|
+
const taken = log2.docIds;
|
|
9351
|
+
log2.docIds = /* @__PURE__ */ new Set();
|
|
9489
9352
|
return taken;
|
|
9490
9353
|
},
|
|
9491
9354
|
close() {
|
|
9492
|
-
if (current[owner] ===
|
|
9355
|
+
if (current[owner] === log2) current[owner] = null;
|
|
9493
9356
|
}
|
|
9494
9357
|
};
|
|
9495
9358
|
},
|
|
@@ -9498,17 +9361,17 @@ function createWriteLogRecorder() {
|
|
|
9498
9361
|
current[1]?.docIds.add(docId);
|
|
9499
9362
|
},
|
|
9500
9363
|
segmentWritten(segment) {
|
|
9501
|
-
for (const
|
|
9502
|
-
if (
|
|
9364
|
+
for (const log2 of current) {
|
|
9365
|
+
if (log2 === null) continue;
|
|
9503
9366
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
9504
9367
|
const docId = segment.docStore.getExternalId(internalId);
|
|
9505
|
-
if (docId !== void 0)
|
|
9368
|
+
if (docId !== void 0) log2.docIds.add(docId);
|
|
9506
9369
|
}
|
|
9507
9370
|
}
|
|
9508
9371
|
},
|
|
9509
9372
|
partitionsReplaced() {
|
|
9510
|
-
for (const
|
|
9511
|
-
if (
|
|
9373
|
+
for (const log2 of current) {
|
|
9374
|
+
if (log2 !== null) log2.replaced = true;
|
|
9512
9375
|
}
|
|
9513
9376
|
}
|
|
9514
9377
|
};
|
|
@@ -10945,6 +10808,22 @@ async function runQueryAction(entry, action) {
|
|
|
10945
10808
|
return { count: result.totalMatched };
|
|
10946
10809
|
}
|
|
10947
10810
|
|
|
10811
|
+
// src/workers/vector-code.ts
|
|
10812
|
+
var vectorCode = codeLoader(() => import('./module-TTEVPRCW.mjs'), "vector");
|
|
10813
|
+
var loadedVectorCode = vectorCode.loaded;
|
|
10814
|
+
var requireLoadedVectorCode = vectorCode.require;
|
|
10815
|
+
var loadVectorCode = vectorCode.load;
|
|
10816
|
+
var loadVectorCodeFor = vectorCode.loadFor;
|
|
10817
|
+
|
|
10818
|
+
// src/workers/schema-code.ts
|
|
10819
|
+
function loadSchemaCodeFor(schema) {
|
|
10820
|
+
const vectorCode2 = loadVectorCodeFor(schema);
|
|
10821
|
+
const geoCode2 = loadGeoCodeFor(schema);
|
|
10822
|
+
if (vectorCode2 === null) return geoCode2;
|
|
10823
|
+
if (geoCode2 === null) return vectorCode2;
|
|
10824
|
+
return Promise.all([vectorCode2, geoCode2]);
|
|
10825
|
+
}
|
|
10826
|
+
|
|
10948
10827
|
// src/core/partition/frozen/doc-table.ts
|
|
10949
10828
|
function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
10950
10829
|
const capacity = idTable.count;
|
|
@@ -11780,17 +11659,17 @@ function createFrozenInvertedReader(table, views, counts) {
|
|
|
11780
11659
|
// src/core/partition/frozen/live-counts.ts
|
|
11781
11660
|
var UINT32_BYTES = 4;
|
|
11782
11661
|
var FLOAT64_BYTES = 8;
|
|
11783
|
-
function removalIndexBytes(
|
|
11784
|
-
const documentWords = 2 * (
|
|
11785
|
-
return (
|
|
11662
|
+
function removalIndexBytes(columns2, terms) {
|
|
11663
|
+
const documentWords = 2 * (columns2.documentCount + 1);
|
|
11664
|
+
return (columns2.postingDocIds.length + documentWords + 2 * terms) * UINT32_BYTES + terms * FLOAT64_BYTES;
|
|
11786
11665
|
}
|
|
11787
11666
|
function processHasRoomFor(bytes) {
|
|
11788
11667
|
const memory = readResidentMemory();
|
|
11789
11668
|
if (memory === null) return true;
|
|
11790
11669
|
return memory.residentBytes + bytes <= FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION * memory.limitBytes;
|
|
11791
11670
|
}
|
|
11792
|
-
function buildRemovalIndex(
|
|
11793
|
-
const { documentCount, postingDocIds } =
|
|
11671
|
+
function buildRemovalIndex(columns2, table) {
|
|
11672
|
+
const { documentCount, postingDocIds } = columns2;
|
|
11794
11673
|
const starts = new Uint32Array(documentCount + 1);
|
|
11795
11674
|
for (let row = 0; row < postingDocIds.length; row++) starts[postingDocIds[row] + 1]++;
|
|
11796
11675
|
for (let ordinal = 0; ordinal < documentCount; ordinal++) starts[ordinal + 1] += starts[ordinal];
|
|
@@ -11809,10 +11688,10 @@ function buildRemovalIndex(columns, table) {
|
|
|
11809
11688
|
occurrences: new Float64Array(table.size)
|
|
11810
11689
|
};
|
|
11811
11690
|
}
|
|
11812
|
-
function tryBuildRemovalIndex(
|
|
11813
|
-
if (!processHasRoomFor(removalIndexBytes(
|
|
11691
|
+
function tryBuildRemovalIndex(columns2, table) {
|
|
11692
|
+
if (!processHasRoomFor(removalIndexBytes(columns2, table.size))) return null;
|
|
11814
11693
|
try {
|
|
11815
|
-
return buildRemovalIndex(
|
|
11694
|
+
return buildRemovalIndex(columns2, table);
|
|
11816
11695
|
} catch (error) {
|
|
11817
11696
|
if (error instanceof RangeError) return null;
|
|
11818
11697
|
throw error;
|
|
@@ -11828,8 +11707,8 @@ function slotOfRow(postingOffsets, slots, row) {
|
|
|
11828
11707
|
}
|
|
11829
11708
|
return low;
|
|
11830
11709
|
}
|
|
11831
|
-
function createFrozenLiveCounts(
|
|
11832
|
-
const { postingOffsets, postingDocIds, postingFrequencies } =
|
|
11710
|
+
function createFrozenLiveCounts(columns2, table, tombstones, storedFrequencies) {
|
|
11711
|
+
const { postingOffsets, postingDocIds, postingFrequencies } = columns2;
|
|
11833
11712
|
let index = null;
|
|
11834
11713
|
let lengths = null;
|
|
11835
11714
|
let frequencies = null;
|
|
@@ -11883,10 +11762,10 @@ function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
|
|
|
11883
11762
|
}
|
|
11884
11763
|
return {
|
|
11885
11764
|
totalDocuments() {
|
|
11886
|
-
return
|
|
11765
|
+
return columns2.documentCount - tombstones.size;
|
|
11887
11766
|
},
|
|
11888
11767
|
totalFieldLengths() {
|
|
11889
|
-
return lengths ??
|
|
11768
|
+
return lengths ?? columns2.totalFieldLengths;
|
|
11890
11769
|
},
|
|
11891
11770
|
documentFrequency,
|
|
11892
11771
|
removedOccurrences(sortedIndex) {
|
|
@@ -11907,15 +11786,15 @@ function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
|
|
|
11907
11786
|
return frequencies;
|
|
11908
11787
|
},
|
|
11909
11788
|
subtractRemoved(ordinal) {
|
|
11910
|
-
lengths ??= { ...
|
|
11911
|
-
for (let field = 0; field <
|
|
11912
|
-
lengths[
|
|
11789
|
+
lengths ??= { ...columns2.totalFieldLengths };
|
|
11790
|
+
for (let field = 0; field < columns2.fieldLengthNames.length; field++) {
|
|
11791
|
+
lengths[columns2.fieldLengthNames[field]] -= columns2.fieldLengthColumns[field][ordinal];
|
|
11913
11792
|
}
|
|
11914
11793
|
if (index !== null) {
|
|
11915
11794
|
countRemoval(index, ordinal);
|
|
11916
11795
|
return;
|
|
11917
11796
|
}
|
|
11918
|
-
const built = tryBuildRemovalIndex(
|
|
11797
|
+
const built = tryBuildRemovalIndex(columns2, table);
|
|
11919
11798
|
if (built === null) return;
|
|
11920
11799
|
for (const removed of tombstones.values()) countRemoval(built, removed);
|
|
11921
11800
|
index = built;
|
|
@@ -12079,7 +11958,7 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12079
11958
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
12080
11959
|
}
|
|
12081
11960
|
}
|
|
12082
|
-
function
|
|
11961
|
+
function loaded2() {
|
|
12083
11962
|
if (registry === null) {
|
|
12084
11963
|
const built = createSurfaceRegistry();
|
|
12085
11964
|
const forms = load();
|
|
@@ -12098,10 +11977,10 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12098
11977
|
return {
|
|
12099
11978
|
heldBytes: () => table?.heldBytes() ?? 0,
|
|
12100
11979
|
reader: {
|
|
12101
|
-
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) :
|
|
12102
|
-
stemChangedTotalFor: (token) =>
|
|
12103
|
-
size: () =>
|
|
12104
|
-
serialize: () =>
|
|
11980
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded2().candidatesForPrefix(prefix),
|
|
11981
|
+
stemChangedTotalFor: (token) => loaded2().stemChangedTotalFor(token),
|
|
11982
|
+
size: () => loaded2().size(),
|
|
11983
|
+
serialize: () => loaded2().serialize()
|
|
12105
11984
|
},
|
|
12106
11985
|
tableStaysOnDisk() {
|
|
12107
11986
|
return registry === null && tableOnDisk !== void 0;
|
|
@@ -12113,20 +11992,20 @@ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
|
12113
11992
|
waitingRemovals.push(ordinal);
|
|
12114
11993
|
},
|
|
12115
11994
|
holdsAny() {
|
|
12116
|
-
return
|
|
11995
|
+
return loaded2().size() > 0;
|
|
12117
11996
|
},
|
|
12118
11997
|
holds(spelling) {
|
|
12119
|
-
return
|
|
11998
|
+
return loaded2().holds(spelling);
|
|
12120
11999
|
},
|
|
12121
12000
|
subtractSpellings(ordinal, spellings) {
|
|
12122
12001
|
uncounted ??= new Uint8Array(documents.count);
|
|
12123
12002
|
uncounted[ordinal] = 1;
|
|
12124
12003
|
if (spellings.length === 0) return;
|
|
12125
|
-
const target =
|
|
12004
|
+
const target = loaded2();
|
|
12126
12005
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
12127
12006
|
},
|
|
12128
12007
|
addKeptForms(target, mergedOrdinals, analysis) {
|
|
12129
|
-
const source =
|
|
12008
|
+
const source = loaded2();
|
|
12130
12009
|
if (analysis === null) {
|
|
12131
12010
|
addForms(target, source.serialize());
|
|
12132
12011
|
return;
|
|
@@ -12528,7 +12407,7 @@ function storedFrequenciesOf(source) {
|
|
|
12528
12407
|
function buildGeoReaders(entries) {
|
|
12529
12408
|
const readers = /* @__PURE__ */ new Map();
|
|
12530
12409
|
for (const entry of entries) {
|
|
12531
|
-
const index =
|
|
12410
|
+
const index = createLoadedGeoIndex();
|
|
12532
12411
|
for (let i = 0; i < entry.docIds.length; i++) {
|
|
12533
12412
|
index.insert(entry.docIds[i], entry.latitudes[i], entry.longitudes[i]);
|
|
12534
12413
|
}
|
|
@@ -12584,6 +12463,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
12584
12463
|
sharedSnapshot,
|
|
12585
12464
|
paged: pagedParts?.access ?? null,
|
|
12586
12465
|
arrays: source,
|
|
12466
|
+
tombstones,
|
|
12587
12467
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
12588
12468
|
docStore,
|
|
12589
12469
|
stats: buildStatsView(counts),
|
|
@@ -13051,7 +12931,7 @@ function fieldIndexRemapOf(segment, fieldNames) {
|
|
|
13051
12931
|
}
|
|
13052
12932
|
function mergeFieldLengths(inputs, documentCount) {
|
|
13053
12933
|
const names = [];
|
|
13054
|
-
const
|
|
12934
|
+
const columns2 = [];
|
|
13055
12935
|
const totals = {};
|
|
13056
12936
|
for (const input of inputs) {
|
|
13057
12937
|
const arrays = input.segment.arrays;
|
|
@@ -13062,11 +12942,11 @@ function mergeFieldLengths(inputs, documentCount) {
|
|
|
13062
12942
|
if (at < 0) {
|
|
13063
12943
|
at = names.length;
|
|
13064
12944
|
names.push(name);
|
|
13065
|
-
|
|
12945
|
+
columns2.push(new Uint32Array(documentCount));
|
|
13066
12946
|
totals[name] = 0;
|
|
13067
12947
|
}
|
|
13068
12948
|
const source = arrays.fieldLengthColumns[f];
|
|
13069
|
-
const column =
|
|
12949
|
+
const column = columns2[at];
|
|
13070
12950
|
for (let ordinal = 0; ordinal < source.length; ordinal++) {
|
|
13071
12951
|
const target = remap[ordinal];
|
|
13072
12952
|
if (target < 0) continue;
|
|
@@ -13075,7 +12955,7 @@ function mergeFieldLengths(inputs, documentCount) {
|
|
|
13075
12955
|
}
|
|
13076
12956
|
}
|
|
13077
12957
|
}
|
|
13078
|
-
return { fieldLengthNames: names, fieldLengthColumns:
|
|
12958
|
+
return { fieldLengthNames: names, fieldLengthColumns: columns2, totalFieldLengths: totals };
|
|
13079
12959
|
}
|
|
13080
12960
|
function mergeSurfaceForms2(inputs, analysis) {
|
|
13081
12961
|
const registry = createSurfaceRegistry();
|
|
@@ -13367,7 +13247,7 @@ function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
|
13367
13247
|
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
13368
13248
|
} else if (fieldType === "geopoint") {
|
|
13369
13249
|
const point = value;
|
|
13370
|
-
indexOf(indexes.geoIndexes, fieldPath,
|
|
13250
|
+
indexOf(indexes.geoIndexes, fieldPath, createLoadedGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
13371
13251
|
}
|
|
13372
13252
|
}
|
|
13373
13253
|
function refuseRepeatedDocIds(documents) {
|
|
@@ -13518,7 +13398,7 @@ async function attachSegments(entry, action) {
|
|
|
13518
13398
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
13519
13399
|
const surfaceForms = segment.survivingSurfaceForms;
|
|
13520
13400
|
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
13521
|
-
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-
|
|
13401
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-6RP4ZCBW.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
13522
13402
|
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
13523
13403
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
13524
13404
|
}
|
|
@@ -13573,7 +13453,7 @@ function runSegmentAction(entry, action) {
|
|
|
13573
13453
|
case "attachSegments":
|
|
13574
13454
|
return attachSegments(entry, action);
|
|
13575
13455
|
case "adoptDiskSegments":
|
|
13576
|
-
return import('./module-
|
|
13456
|
+
return import('./module-6RP4ZCBW.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
13577
13457
|
case "compactSegments": {
|
|
13578
13458
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
13579
13459
|
return mergeFrozenSegments(
|
|
@@ -13607,47 +13487,6 @@ function runSegmentAction(entry, action) {
|
|
|
13607
13487
|
}
|
|
13608
13488
|
}
|
|
13609
13489
|
|
|
13610
|
-
// src/workers/vector-code.ts
|
|
13611
|
-
var VECTOR_TYPE_NAME = "vector";
|
|
13612
|
-
var loaded2 = null;
|
|
13613
|
-
var loading2 = null;
|
|
13614
|
-
function loadedVectorCode() {
|
|
13615
|
-
return loaded2;
|
|
13616
|
-
}
|
|
13617
|
-
function requireLoadedVectorCode() {
|
|
13618
|
-
if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
|
|
13619
|
-
return loaded2;
|
|
13620
|
-
}
|
|
13621
|
-
function loadVectorCode() {
|
|
13622
|
-
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
13623
|
-
if (loading2 === null) {
|
|
13624
|
-
loading2 = import('./module-H5MVPSP2.mjs').then(
|
|
13625
|
-
(code) => {
|
|
13626
|
-
loaded2 = code;
|
|
13627
|
-
return code;
|
|
13628
|
-
},
|
|
13629
|
-
(err) => {
|
|
13630
|
-
loading2 = null;
|
|
13631
|
-
throw err;
|
|
13632
|
-
}
|
|
13633
|
-
);
|
|
13634
|
-
}
|
|
13635
|
-
return loading2;
|
|
13636
|
-
}
|
|
13637
|
-
function mayDeclareVectorField(schema, depth) {
|
|
13638
|
-
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
13639
|
-
for (const type of Object.values(schema)) {
|
|
13640
|
-
if (typeof type === "string" ? type.includes(VECTOR_TYPE_NAME) : mayDeclareVectorField(type, depth + 1)) {
|
|
13641
|
-
return true;
|
|
13642
|
-
}
|
|
13643
|
-
}
|
|
13644
|
-
return false;
|
|
13645
|
-
}
|
|
13646
|
-
function loadVectorCodeFor(schema) {
|
|
13647
|
-
if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
|
|
13648
|
-
return loadVectorCode();
|
|
13649
|
-
}
|
|
13650
|
-
|
|
13651
13490
|
// src/workers/vector-copies.ts
|
|
13652
13491
|
function createHeldVectorCopies() {
|
|
13653
13492
|
return { searchers: /* @__PURE__ */ new Map(), fields: /* @__PURE__ */ new Map(), builds: /* @__PURE__ */ new Map() };
|
|
@@ -13853,8 +13692,8 @@ function createDirectExecutor(options) {
|
|
|
13853
13692
|
);
|
|
13854
13693
|
}
|
|
13855
13694
|
case "createIndex": {
|
|
13856
|
-
const
|
|
13857
|
-
if (
|
|
13695
|
+
const loading2 = loadSchemaCodeFor(action.config.schema);
|
|
13696
|
+
if (loading2 !== null) await loading2;
|
|
13858
13697
|
const language = getLanguage(action.config.language ?? "english");
|
|
13859
13698
|
const manager = addIndex(
|
|
13860
13699
|
action.indexName,
|
|
@@ -13905,16 +13744,19 @@ function createDirectExecutor(options) {
|
|
|
13905
13744
|
case "insert": {
|
|
13906
13745
|
const entry = requireIndex(action.indexName);
|
|
13907
13746
|
entry.manager.insert(action.docId, action.document, action.skipClone ? { skipClone: true } : void 0);
|
|
13747
|
+
options?.onDocumentWrite?.(action);
|
|
13908
13748
|
return void 0;
|
|
13909
13749
|
}
|
|
13910
13750
|
case "remove": {
|
|
13911
13751
|
const entry = requireIndex(action.indexName);
|
|
13912
13752
|
entry.manager.remove(action.docId, action.removedSpellings);
|
|
13753
|
+
options?.onDocumentWrite?.(action);
|
|
13913
13754
|
return void 0;
|
|
13914
13755
|
}
|
|
13915
13756
|
case "update": {
|
|
13916
13757
|
const entry = requireIndex(action.indexName);
|
|
13917
13758
|
entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
|
|
13759
|
+
options?.onDocumentWrite?.(action);
|
|
13918
13760
|
return void 0;
|
|
13919
13761
|
}
|
|
13920
13762
|
case "query":
|
|
@@ -13961,6 +13803,7 @@ function createDirectExecutor(options) {
|
|
|
13961
13803
|
entry.manager.resetVectorIndexes(vectorIndexesFor(action.indexName, entry.config));
|
|
13962
13804
|
entry.vectorIndexes = entry.manager.getVectorIndexes();
|
|
13963
13805
|
entry.vectorCopies = createHeldVectorCopies();
|
|
13806
|
+
options?.onDocumentWrite?.(action);
|
|
13964
13807
|
return void 0;
|
|
13965
13808
|
}
|
|
13966
13809
|
case "serialize": {
|
|
@@ -14037,7 +13880,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
14037
13880
|
// src/partitioning/rewrite-spill-of.ts
|
|
14038
13881
|
async function rewriteSpillOf(storage, manager, target) {
|
|
14039
13882
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
14040
|
-
const { openRewriteSpill } = await import('./module-
|
|
13883
|
+
const { openRewriteSpill } = await import('./module-6RP4ZCBW.mjs');
|
|
14041
13884
|
return openRewriteSpill(storage, manager, target);
|
|
14042
13885
|
}
|
|
14043
13886
|
|
|
@@ -14127,16 +13970,16 @@ async function rebuildTextIndexes(manager, hooks, target) {
|
|
|
14127
13970
|
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
14128
13971
|
const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
|
|
14129
13972
|
while (hooks.isCurrent()) {
|
|
14130
|
-
const
|
|
13973
|
+
const log2 = manager.openWriteLog();
|
|
14131
13974
|
let swapped = false;
|
|
14132
13975
|
try {
|
|
14133
|
-
swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
|
|
13976
|
+
swapped = await rebuildOnce({ manager, log: log2, hooks, language, partialWordAnalysis, options, spill });
|
|
14134
13977
|
if (swapped) {
|
|
14135
13978
|
hooks.onRewriteSpill?.(spill);
|
|
14136
13979
|
return true;
|
|
14137
13980
|
}
|
|
14138
13981
|
} finally {
|
|
14139
|
-
|
|
13982
|
+
log2.close();
|
|
14140
13983
|
if (!swapped) await spill?.abandon();
|
|
14141
13984
|
}
|
|
14142
13985
|
}
|
|
@@ -14640,12 +14483,12 @@ function recoveryFailure(metadata, err) {
|
|
|
14640
14483
|
});
|
|
14641
14484
|
}
|
|
14642
14485
|
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
14643
|
-
const
|
|
14644
|
-
if (
|
|
14486
|
+
const loadingSchemaCode = loadData ? loadSchemaCodeFor(metadata.schema) : null;
|
|
14487
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
14645
14488
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
14646
14489
|
if (existing !== void 0) {
|
|
14647
14490
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
14648
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
14491
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-QUVDNABS.mjs');
|
|
14649
14492
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
14650
14493
|
}
|
|
14651
14494
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -15884,16 +15727,16 @@ function documentWeight(action) {
|
|
|
15884
15727
|
}
|
|
15885
15728
|
return 1;
|
|
15886
15729
|
}
|
|
15730
|
+
var NO_VECTOR_FIELDS = /* @__PURE__ */ new Set();
|
|
15887
15731
|
function vectorFieldPathsOf(state, indexName) {
|
|
15888
|
-
|
|
15889
|
-
if (entry === void 0) return /* @__PURE__ */ new Set();
|
|
15890
|
-
return new Set(extractVectorFieldsFromSchema(entry.config.schema).keys());
|
|
15732
|
+
return state.indexRegistry.get(indexName)?.vectorFieldPaths ?? NO_VECTOR_FIELDS;
|
|
15891
15733
|
}
|
|
15892
15734
|
function withoutVectors(document, fieldPaths) {
|
|
15735
|
+
if (fieldPaths.size === 0) return document;
|
|
15893
15736
|
return prepareDocumentVectors(document, fieldPaths).partitionDoc;
|
|
15894
15737
|
}
|
|
15895
15738
|
function detachCallerDocuments(state, action) {
|
|
15896
|
-
if (
|
|
15739
|
+
if (action.type !== "insert" && action.type !== "update" && action.type !== "mergeSegments") return action;
|
|
15897
15740
|
const fieldPaths = vectorFieldPathsOf(state, action.indexName);
|
|
15898
15741
|
if (action.type === "insert") {
|
|
15899
15742
|
const stripped = withoutVectors(action.document, fieldPaths);
|
|
@@ -15904,17 +15747,14 @@ function detachCallerDocuments(state, action) {
|
|
|
15904
15747
|
const stripped = withoutVectors(action.document, fieldPaths);
|
|
15905
15748
|
return { ...action, document: stripped !== action.document ? stripped : structuredClone(stripped) };
|
|
15906
15749
|
}
|
|
15907
|
-
|
|
15908
|
-
|
|
15909
|
-
|
|
15910
|
-
|
|
15911
|
-
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15915
|
-
};
|
|
15916
|
-
}
|
|
15917
|
-
return action;
|
|
15750
|
+
return {
|
|
15751
|
+
...action,
|
|
15752
|
+
segments: action.segments.map((segment) => {
|
|
15753
|
+
const documents = segment.documents.map((document) => withoutVectors(document, fieldPaths));
|
|
15754
|
+
const cloned = action.skipClone === true || fieldPaths.size > 0 ? documents : structuredClone(documents);
|
|
15755
|
+
return { ...segment, documents: cloned };
|
|
15756
|
+
})
|
|
15757
|
+
};
|
|
15918
15758
|
}
|
|
15919
15759
|
function partitionLedger(state, indexName, partitionId) {
|
|
15920
15760
|
let byPartition = state.segmentLedger.get(indexName);
|
|
@@ -15930,6 +15770,10 @@ function partitionLedger(state, indexName, partitionId) {
|
|
|
15930
15770
|
return entries;
|
|
15931
15771
|
}
|
|
15932
15772
|
function recordSegmentBroadcast(state, action) {
|
|
15773
|
+
if (action.type === "clear") {
|
|
15774
|
+
state.segmentLedger.delete(action.indexName);
|
|
15775
|
+
return;
|
|
15776
|
+
}
|
|
15933
15777
|
if (action.type === "attachSegments") {
|
|
15934
15778
|
for (const segment of action.segments) {
|
|
15935
15779
|
partitionLedger(state, action.indexName, segment.partitionId).push({
|
|
@@ -16013,30 +15857,36 @@ async function replicateToWorkers(state, action) {
|
|
|
16013
15857
|
await dispatchToWorkers(state, action);
|
|
16014
15858
|
return;
|
|
16015
15859
|
}
|
|
16016
|
-
|
|
16017
|
-
const
|
|
16018
|
-
|
|
16019
|
-
if (loading3 !== void 0) {
|
|
16020
|
-
queueForCopies(state, detached);
|
|
16021
|
-
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
|
|
16022
|
-
return;
|
|
16023
|
-
}
|
|
16024
|
-
const queue = state.replicationQueues.get(action.indexName);
|
|
16025
|
-
if (queue !== void 0 && queue.pendingDocuments + documentWeight(detached) > MAX_PENDING_REPLICATION_DOCUMENTS) {
|
|
16026
|
-
await new Promise((resume) => queue.drainWaiters.push(resume));
|
|
16027
|
-
}
|
|
16028
|
-
queueForCopies(state, detached);
|
|
15860
|
+
recordMainWrite(state, action);
|
|
15861
|
+
const waiting = replicationRoomWait(state, action.indexName);
|
|
15862
|
+
if (waiting !== null) await waiting;
|
|
16029
15863
|
}
|
|
16030
15864
|
function queueForCopies(state, action) {
|
|
16031
15865
|
if (!("indexName" in action)) return;
|
|
16032
|
-
const
|
|
16033
|
-
if (
|
|
16034
|
-
|
|
15866
|
+
const loading2 = state.copyLoadBuffers.get(action.indexName);
|
|
15867
|
+
if (loading2 !== void 0) {
|
|
15868
|
+
loading2.push(action);
|
|
16035
15869
|
return;
|
|
16036
15870
|
}
|
|
16037
15871
|
if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
|
|
16038
15872
|
enqueueReplication(state, action.indexName, action);
|
|
16039
15873
|
}
|
|
15874
|
+
function recordMainWrite(state, action) {
|
|
15875
|
+
if (!("indexName" in action) || !copiesWillReceive(state, action.indexName)) return;
|
|
15876
|
+
queueForCopies(state, detachCallerDocuments(state, action));
|
|
15877
|
+
}
|
|
15878
|
+
function replicationRoomWait(state, indexName) {
|
|
15879
|
+
const loading2 = state.copyLoadBuffers.get(indexName);
|
|
15880
|
+
if (loading2 !== void 0) {
|
|
15881
|
+
const lastYield = state.copyLoadBufferYieldMarks.get(loading2) ?? 0;
|
|
15882
|
+
if (loading2.length - lastYield < LOAD_BUFFER_YIELD_INTERVAL) return null;
|
|
15883
|
+
state.copyLoadBufferYieldMarks.set(loading2, loading2.length);
|
|
15884
|
+
return yieldToEventLoop2();
|
|
15885
|
+
}
|
|
15886
|
+
const queue = state.replicationQueues.get(indexName);
|
|
15887
|
+
if (queue === void 0 || queue.pendingDocuments < MAX_PENDING_REPLICATION_DOCUMENTS) return null;
|
|
15888
|
+
return new Promise((resume) => queue.drainWaiters.push(resume));
|
|
15889
|
+
}
|
|
16040
15890
|
function settled(queue) {
|
|
16041
15891
|
return Promise.all([...queue.inFlight, queue.tail]).then(() => void 0);
|
|
16042
15892
|
}
|
|
@@ -16113,6 +15963,7 @@ function freezeLiveTail(state, indexName, manager, partitionId, floor) {
|
|
|
16113
15963
|
}
|
|
16114
15964
|
}
|
|
16115
15965
|
function flushGrownTails(state, indexName) {
|
|
15966
|
+
if (!state.scaledOutIndexes.has(indexName)) return;
|
|
16116
15967
|
const manager = state.executor.getManager(indexName);
|
|
16117
15968
|
if (manager === void 0) return;
|
|
16118
15969
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
@@ -16135,25 +15986,10 @@ function mainCompositeOf(manager, partitionId) {
|
|
|
16135
15986
|
const partition = manager.getPartition(partitionId);
|
|
16136
15987
|
return isCompositePartition(partition) ? partition : null;
|
|
16137
15988
|
}
|
|
16138
|
-
function holdUnbroadcastSegments(state, indexName, segmentIds) {
|
|
16139
|
-
let held = state.unbroadcastSegments.get(indexName);
|
|
16140
|
-
if (held === void 0) {
|
|
16141
|
-
held = /* @__PURE__ */ new Set();
|
|
16142
|
-
state.unbroadcastSegments.set(indexName, held);
|
|
16143
|
-
}
|
|
16144
|
-
for (const segmentId of segmentIds) held.add(segmentId);
|
|
16145
|
-
}
|
|
16146
|
-
function releaseUnbroadcastSegments(state, indexName, segmentIds) {
|
|
16147
|
-
const held = state.unbroadcastSegments.get(indexName);
|
|
16148
|
-
if (held === void 0) return;
|
|
16149
|
-
for (const segmentId of segmentIds) held.delete(segmentId);
|
|
16150
|
-
if (held.size === 0) state.unbroadcastSegments.delete(indexName);
|
|
16151
|
-
}
|
|
16152
15989
|
function candidateSegmentIds(state, indexName, manager, partitionId, policy) {
|
|
16153
|
-
const unbroadcast = state.unbroadcastSegments.get(indexName);
|
|
16154
15990
|
const composite = mainCompositeOf(manager, partitionId);
|
|
16155
15991
|
if (composite !== null) {
|
|
16156
|
-
const sizes = composite.frozenSegmentSizes().
|
|
15992
|
+
const sizes = composite.frozenSegmentSizes().map((size) => ({ segmentId: size.segmentId, documentCount: size.liveDocumentCount }));
|
|
16157
15993
|
const picks = pickSegmentIds(sizes, policy);
|
|
16158
15994
|
if (picks !== null) return picks;
|
|
16159
15995
|
}
|
|
@@ -16187,6 +16023,17 @@ async function compactOnWorker(state, indexName, partitionId, segmentIds) {
|
|
|
16187
16023
|
lease.release();
|
|
16188
16024
|
}
|
|
16189
16025
|
}
|
|
16026
|
+
function ledgerHoldsAll(state, indexName, partitionId, segmentIds) {
|
|
16027
|
+
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
16028
|
+
if (entries === void 0) return false;
|
|
16029
|
+
const held = new Set(entries.map((entry) => entry.segmentId));
|
|
16030
|
+
return segmentIds.every((segmentId) => held.has(segmentId));
|
|
16031
|
+
}
|
|
16032
|
+
function segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain) {
|
|
16033
|
+
if (state.executor.getManager(indexName) !== manager) return false;
|
|
16034
|
+
if (mergedOnMain === null) return ledgerHoldsAll(state, indexName, partitionId, segmentIds);
|
|
16035
|
+
return mainCompositeOf(manager, partitionId) === mergedOnMain && mainHoldsAll(mergedOnMain, segmentIds);
|
|
16036
|
+
}
|
|
16190
16037
|
function forgetLedgerSegments(state, indexName, partitionId, segmentIds) {
|
|
16191
16038
|
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
16192
16039
|
if (entries === void 0) return;
|
|
@@ -16212,8 +16059,10 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
16212
16059
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
16213
16060
|
return false;
|
|
16214
16061
|
}
|
|
16215
|
-
const
|
|
16216
|
-
|
|
16062
|
+
const mergedOnMain = mainCanMerge ? composite : null;
|
|
16063
|
+
if (!segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain)) return false;
|
|
16064
|
+
const removedSpellings = mergedOnMain?.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
|
|
16065
|
+
recordMainWrite(state, {
|
|
16217
16066
|
type: "swapSegments",
|
|
16218
16067
|
indexName,
|
|
16219
16068
|
partitionId,
|
|
@@ -16255,19 +16104,34 @@ function maybeCompactSegments(state, indexName, policy = "ingest") {
|
|
|
16255
16104
|
});
|
|
16256
16105
|
state.compactionsInFlight.set(indexName, run);
|
|
16257
16106
|
}
|
|
16258
|
-
function
|
|
16259
|
-
cancelIdleMerge(state, indexName);
|
|
16107
|
+
function armIdleMerge(state, indexName, dueAt, delayMs) {
|
|
16260
16108
|
const timer = setTimeout(() => {
|
|
16109
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
16110
|
+
if (pending === void 0 || pending.timer !== timer) return;
|
|
16111
|
+
const remainingMs = pending.dueAt - Date.now();
|
|
16112
|
+
if (remainingMs > 0) {
|
|
16113
|
+
armIdleMerge(state, indexName, pending.dueAt, remainingMs);
|
|
16114
|
+
return;
|
|
16115
|
+
}
|
|
16261
16116
|
state.idleMergeTimers.delete(indexName);
|
|
16262
16117
|
maybeCompactSegments(state, indexName, "idle");
|
|
16263
|
-
},
|
|
16118
|
+
}, delayMs);
|
|
16264
16119
|
if (typeof timer.unref === "function") timer.unref();
|
|
16265
|
-
state.idleMergeTimers.set(indexName, timer);
|
|
16120
|
+
state.idleMergeTimers.set(indexName, { timer, dueAt });
|
|
16121
|
+
}
|
|
16122
|
+
function scheduleIdleMerge(state, indexName) {
|
|
16123
|
+
const dueAt = Date.now() + IDLE_MERGE_DELAY_MS;
|
|
16124
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
16125
|
+
if (pending !== void 0) {
|
|
16126
|
+
pending.dueAt = dueAt;
|
|
16127
|
+
return;
|
|
16128
|
+
}
|
|
16129
|
+
armIdleMerge(state, indexName, dueAt, IDLE_MERGE_DELAY_MS);
|
|
16266
16130
|
}
|
|
16267
16131
|
function cancelIdleMerge(state, indexName) {
|
|
16268
|
-
const
|
|
16269
|
-
if (
|
|
16270
|
-
clearTimeout(timer);
|
|
16132
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
16133
|
+
if (pending === void 0) return;
|
|
16134
|
+
clearTimeout(pending.timer);
|
|
16271
16135
|
state.idleMergeTimers.delete(indexName);
|
|
16272
16136
|
}
|
|
16273
16137
|
async function awaitCompactions(state) {
|
|
@@ -17019,13 +16883,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17019
16883
|
desyncedIndexes: /* @__PURE__ */ new Set(),
|
|
17020
16884
|
copyReleaseHolds: /* @__PURE__ */ new Map(),
|
|
17021
16885
|
copyLoadBuffers: /* @__PURE__ */ new Map(),
|
|
16886
|
+
copyLoadBufferYieldMarks: /* @__PURE__ */ new WeakMap(),
|
|
17022
16887
|
copyTransitions: /* @__PURE__ */ new Map(),
|
|
17023
16888
|
droppedCopies: /* @__PURE__ */ new Map(),
|
|
17024
16889
|
lastAccessAt: /* @__PURE__ */ new Map(),
|
|
17025
16890
|
copyReloadCounts: /* @__PURE__ */ new Map(),
|
|
17026
16891
|
replicationQueues: /* @__PURE__ */ new Map(),
|
|
17027
16892
|
segmentLedger: /* @__PURE__ */ new Map(),
|
|
17028
|
-
unbroadcastSegments: /* @__PURE__ */ new Map(),
|
|
17029
16893
|
compactionsInFlight: /* @__PURE__ */ new Map(),
|
|
17030
16894
|
idleMergeTimers: /* @__PURE__ */ new Map(),
|
|
17031
16895
|
sharedVectorFields: /* @__PURE__ */ new Map(),
|
|
@@ -17180,7 +17044,6 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17180
17044
|
state.lastAccessAt.delete(indexName);
|
|
17181
17045
|
state.replicationQueues.delete(indexName);
|
|
17182
17046
|
state.segmentLedger.delete(indexName);
|
|
17183
|
-
state.unbroadcastSegments.delete(indexName);
|
|
17184
17047
|
}
|
|
17185
17048
|
async function replicate(action) {
|
|
17186
17049
|
await replicateToWorkers(state, action);
|
|
@@ -17191,10 +17054,31 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17191
17054
|
scheduleIdleMerge(state, action.indexName);
|
|
17192
17055
|
}
|
|
17193
17056
|
}
|
|
17057
|
+
function afterMainWrites(indexName, growth) {
|
|
17058
|
+
if (growth === "segments") maybeCompactSegments(state, indexName);
|
|
17059
|
+
if (growth === "liveTail") flushGrownTails(state, indexName);
|
|
17060
|
+
if (state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) {
|
|
17061
|
+
scheduleIdleMerge(state, indexName);
|
|
17062
|
+
}
|
|
17063
|
+
}
|
|
17064
|
+
function copiesFollow(indexName) {
|
|
17065
|
+
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName) || state.replicationQueues.has(indexName);
|
|
17066
|
+
}
|
|
17067
|
+
function settleMainWrites(indexName, growth = "liveTail") {
|
|
17068
|
+
if (growth !== "segments" && !copiesFollow(indexName)) return null;
|
|
17069
|
+
const waiting = replicationRoomWait(state, indexName);
|
|
17070
|
+
if (waiting === null) {
|
|
17071
|
+
afterMainWrites(indexName, growth);
|
|
17072
|
+
return null;
|
|
17073
|
+
}
|
|
17074
|
+
return waiting.then(() => afterMainWrites(indexName, growth));
|
|
17075
|
+
}
|
|
17194
17076
|
return {
|
|
17195
17077
|
scaleOutReadyIndexes: () => scaleOutReadyIndexes(state),
|
|
17196
17078
|
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
17197
17079
|
replicateToWorkers: replicate,
|
|
17080
|
+
recordMainWrite: (action) => recordMainWrite(state, action),
|
|
17081
|
+
settleMainWrites,
|
|
17198
17082
|
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
17199
17083
|
queueDiskAdoption: (action) => queueForCopies(state, action),
|
|
17200
17084
|
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
@@ -17204,8 +17088,6 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17204
17088
|
isIndexBusy: (indexName) => isIndexBusy(state, indexName),
|
|
17205
17089
|
buildSegments: (requests) => buildSegments(state, requests),
|
|
17206
17090
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
17207
|
-
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
17208
|
-
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
17209
17091
|
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25),
|
|
17210
17092
|
hasWorkerPool: () => state.workerPool !== null,
|
|
17211
17093
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
@@ -17218,21 +17100,21 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
17218
17100
|
stopRequestThreads: () => stopRequestThreads(state),
|
|
17219
17101
|
desyncIndex: (indexName) => {
|
|
17220
17102
|
state.desyncedIndexes.add(indexName);
|
|
17221
|
-
const
|
|
17103
|
+
const loading2 = state.copyTransitions.get(indexName);
|
|
17222
17104
|
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
17223
17105
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
17224
|
-
return wasScaledOut ||
|
|
17106
|
+
return wasScaledOut || loading2 !== void 0 && loading2.kind !== "drop";
|
|
17225
17107
|
},
|
|
17226
17108
|
async releaseIndexCopies(indexName) {
|
|
17227
17109
|
const hold = state.copyReleaseHolds.get(indexName);
|
|
17228
17110
|
if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
|
|
17229
17111
|
else hold.holders += 1;
|
|
17230
17112
|
state.desyncedIndexes.add(indexName);
|
|
17231
|
-
const { releaseIndexCopies } = await import('./release-
|
|
17113
|
+
const { releaseIndexCopies } = await import('./release-2SIHZWDO.mjs');
|
|
17232
17114
|
return releaseIndexCopies(state, indexName, copyReleaseSteps);
|
|
17233
17115
|
},
|
|
17234
17116
|
async endCopyRelease(indexName, reloads) {
|
|
17235
|
-
const { endCopyRelease } = await import('./release-
|
|
17117
|
+
const { endCopyRelease } = await import('./release-2SIHZWDO.mjs');
|
|
17236
17118
|
await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
|
|
17237
17119
|
},
|
|
17238
17120
|
resyncIndex,
|
|
@@ -17281,12 +17163,13 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17281
17163
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
17282
17164
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
17283
17165
|
}
|
|
17284
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
17166
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-RN64S2TL.mjs');
|
|
17285
17167
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
17286
17168
|
const executor = createDirectExecutor({
|
|
17287
17169
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
17288
17170
|
vectorStorage: filesystemDurability ? "disk" : "memory",
|
|
17289
|
-
onVectorGraphRepaired: (indexName) => void durability?.manager.checkpointAfterInFlight(indexName)
|
|
17171
|
+
onVectorGraphRepaired: (indexName) => void durability?.manager.checkpointAfterInFlight(indexName),
|
|
17172
|
+
onDocumentWrite: (action) => orchestrator.recordMainWrite(action)
|
|
17290
17173
|
});
|
|
17291
17174
|
const pluginRegistry = createPluginRegistry();
|
|
17292
17175
|
if (config?.plugins) {
|
|
@@ -17425,7 +17308,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17425
17308
|
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
17426
17309
|
{ eventHandlers, mutationCtx },
|
|
17427
17310
|
indexName,
|
|
17428
|
-
|
|
17311
|
+
durabilityWiring?.reloadIndexAndCopies(orchestrator, durability, indexName) ?? Promise.resolve()
|
|
17429
17312
|
),
|
|
17430
17313
|
onError(error) {
|
|
17431
17314
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
@@ -17467,7 +17350,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
17467
17350
|
requireManager,
|
|
17468
17351
|
resumeChange: async (indexName) => {
|
|
17469
17352
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
17470
|
-
await (await import('./analysis-change-entry-
|
|
17353
|
+
await (await import('./analysis-change-entry-OC4XASBF.mjs')).resumeChangeOf(engineCore, indexName);
|
|
17471
17354
|
},
|
|
17472
17355
|
onClose: hooks?.onIndexClose,
|
|
17473
17356
|
onOpen: (indexName) => {
|
|
@@ -17555,7 +17438,7 @@ async function startEngineCore(core, config) {
|
|
|
17555
17438
|
checkHeapAfterRecovery(core);
|
|
17556
17439
|
}
|
|
17557
17440
|
if (core.invalidation) await core.invalidation.start();
|
|
17558
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
17441
|
+
if (core.durability) await (await import('./analysis-change-entry-OC4XASBF.mjs')).resumeEveryChange(core);
|
|
17559
17442
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
17560
17443
|
} catch (err) {
|
|
17561
17444
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -17602,6 +17485,874 @@ async function shutdownEngine(core) {
|
|
|
17602
17485
|
indexRegistry.clear();
|
|
17603
17486
|
}
|
|
17604
17487
|
|
|
17488
|
+
// src/core/keyword-score/layout.ts
|
|
17489
|
+
var TERM_SLOT = 0;
|
|
17490
|
+
var GROUP_SLOT = 1;
|
|
17491
|
+
var PHRASE_SLOT = 2;
|
|
17492
|
+
var SLOT_STRIDE = 4;
|
|
17493
|
+
var SLOT_KIND = 0;
|
|
17494
|
+
var SLOT_TERM = 1;
|
|
17495
|
+
var SLOT_LIST_END = 2;
|
|
17496
|
+
var SLOT_VALUE_END = 3;
|
|
17497
|
+
var LIST_STRIDE = 3;
|
|
17498
|
+
var LIST_FROM_ROWS = 1;
|
|
17499
|
+
var LIST_ORDERED = 2;
|
|
17500
|
+
var LIST_VALUE_STRIDE = 2;
|
|
17501
|
+
var LIST_DOCUMENT_FREQUENCY = 0;
|
|
17502
|
+
var LIST_FACTOR = 0;
|
|
17503
|
+
var LIST_PARTIAL_FACTOR = 1;
|
|
17504
|
+
var FIELD_BOOSTS = 0;
|
|
17505
|
+
var FIELD_AVERAGE_LENGTHS = 1;
|
|
17506
|
+
var FIELD_K1 = 2;
|
|
17507
|
+
var FIELD_B = 3;
|
|
17508
|
+
var FIELD_NUMBER_COLUMNS = 4;
|
|
17509
|
+
var SETTING_SLOTS = 0;
|
|
17510
|
+
var SETTING_TERMS = 1;
|
|
17511
|
+
var SETTING_CAPACITY = 2;
|
|
17512
|
+
var SETTING_TOTAL_DOCUMENTS = 3;
|
|
17513
|
+
var SETTING_TERM_MATCH = 4;
|
|
17514
|
+
var SETTING_ANCHOR = 5;
|
|
17515
|
+
var SETTING_LIMIT = 6;
|
|
17516
|
+
var SETTING_MIN_SCORE = 7;
|
|
17517
|
+
var SETTING_COLLECT = 8;
|
|
17518
|
+
var SETTING_COUNT = 9;
|
|
17519
|
+
var NO_LIMIT = -1;
|
|
17520
|
+
var OUTCOME_COUNT = 0;
|
|
17521
|
+
var OUTCOME_TOTAL_MATCHED = 1;
|
|
17522
|
+
var OUTCOME_LENGTH = 3;
|
|
17523
|
+
|
|
17524
|
+
// src/core/keyword-score/constants.ts
|
|
17525
|
+
var KEYWORD_FIRST_SLOTS = 16;
|
|
17526
|
+
var KEYWORD_FIRST_LISTS = 64;
|
|
17527
|
+
var KEYWORD_FIRST_ROWS = 1024;
|
|
17528
|
+
var KEYWORD_INSERTION_SORT_ROWS = 16;
|
|
17529
|
+
|
|
17530
|
+
// src/core/keyword-score/rows.ts
|
|
17531
|
+
var UNKNOWN_FIELD_RANK = 65536;
|
|
17532
|
+
function rankOf(keys, row) {
|
|
17533
|
+
const field = keys.fields[row];
|
|
17534
|
+
return field < keys.nameRanks.length ? keys.nameRanks[field] : UNKNOWN_FIELD_RANK;
|
|
17535
|
+
}
|
|
17536
|
+
function rowBefore(keys, left, right) {
|
|
17537
|
+
const leftDoc = keys.docIds[left];
|
|
17538
|
+
const rightDoc = keys.docIds[right];
|
|
17539
|
+
if (leftDoc !== rightDoc) return leftDoc < rightDoc;
|
|
17540
|
+
return rankOf(keys, left) < rankOf(keys, right);
|
|
17541
|
+
}
|
|
17542
|
+
function insertionSort(keys, order, from, count) {
|
|
17543
|
+
for (let at = from + 1; at < from + count; at++) {
|
|
17544
|
+
const row = order[at];
|
|
17545
|
+
let to = at;
|
|
17546
|
+
while (to > from && rowBefore(keys, row, order[to - 1])) {
|
|
17547
|
+
order[to] = order[to - 1];
|
|
17548
|
+
to--;
|
|
17549
|
+
}
|
|
17550
|
+
order[to] = row;
|
|
17551
|
+
}
|
|
17552
|
+
}
|
|
17553
|
+
function mergeRuns(keys, source, target, from, middle, end) {
|
|
17554
|
+
let left = from;
|
|
17555
|
+
let right = middle;
|
|
17556
|
+
let out = from;
|
|
17557
|
+
while (left < middle && right < end) {
|
|
17558
|
+
if (rowBefore(keys, source[right], source[left])) target[out++] = source[right++];
|
|
17559
|
+
else target[out++] = source[left++];
|
|
17560
|
+
}
|
|
17561
|
+
while (left < middle) target[out++] = source[left++];
|
|
17562
|
+
while (right < end) target[out++] = source[right++];
|
|
17563
|
+
}
|
|
17564
|
+
function sortRows(keys, order, spare, count) {
|
|
17565
|
+
if (count <= KEYWORD_INSERTION_SORT_ROWS) {
|
|
17566
|
+
insertionSort(keys, order, 0, count);
|
|
17567
|
+
return order;
|
|
17568
|
+
}
|
|
17569
|
+
for (let from = 0; from < count; from += KEYWORD_INSERTION_SORT_ROWS) {
|
|
17570
|
+
insertionSort(keys, order, from, Math.min(KEYWORD_INSERTION_SORT_ROWS, count - from));
|
|
17571
|
+
}
|
|
17572
|
+
let source = order;
|
|
17573
|
+
let target = spare;
|
|
17574
|
+
for (let width = KEYWORD_INSERTION_SORT_ROWS; width < count; width *= 2) {
|
|
17575
|
+
for (let from = 0; from < count; from += 2 * width) {
|
|
17576
|
+
const middle = Math.min(from + width, count);
|
|
17577
|
+
const end = Math.min(from + 2 * width, count);
|
|
17578
|
+
mergeRuns(keys, source, target, from, middle, end);
|
|
17579
|
+
}
|
|
17580
|
+
const held = source;
|
|
17581
|
+
source = target;
|
|
17582
|
+
target = held;
|
|
17583
|
+
}
|
|
17584
|
+
return source;
|
|
17585
|
+
}
|
|
17586
|
+
|
|
17587
|
+
// src/core/partition/keyword-components.ts
|
|
17588
|
+
function holds(bits, ordinal) {
|
|
17589
|
+
const word = ordinal >>> 5;
|
|
17590
|
+
return word < bits.length && (bits[word] >>> (ordinal & 31) & 1) === 1;
|
|
17591
|
+
}
|
|
17592
|
+
function scorable(request, ordinal) {
|
|
17593
|
+
if (!holds(request.live, ordinal) || holds(request.removed, ordinal)) return false;
|
|
17594
|
+
return request.filter === null || holds(request.filter, ordinal);
|
|
17595
|
+
}
|
|
17596
|
+
function rowsOfList(request, list, wanted) {
|
|
17597
|
+
const flags = request.lists[list * LIST_STRIDE];
|
|
17598
|
+
const start = request.lists[list * LIST_STRIDE + 1];
|
|
17599
|
+
const length = request.lists[list * LIST_STRIDE + 2];
|
|
17600
|
+
const fromRows = (flags & LIST_FROM_ROWS) !== 0;
|
|
17601
|
+
const docIds = fromRows ? request.rowDocIds : request.pool.docIds;
|
|
17602
|
+
const keys = {
|
|
17603
|
+
docIds,
|
|
17604
|
+
fields: fromRows ? request.rowFields : request.pool.fields,
|
|
17605
|
+
nameRanks: request.fields.nameRanks
|
|
17606
|
+
};
|
|
17607
|
+
const gathered = /* @__PURE__ */ new Map();
|
|
17608
|
+
for (let row = start; row < start + length; row++) {
|
|
17609
|
+
const ordinal = docIds[row];
|
|
17610
|
+
if (!wanted.has(ordinal)) continue;
|
|
17611
|
+
const rows = gathered.get(ordinal);
|
|
17612
|
+
if (rows === void 0) gathered.set(ordinal, [row]);
|
|
17613
|
+
else rows.push(row);
|
|
17614
|
+
}
|
|
17615
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
17616
|
+
for (const [ordinal, rows] of gathered) {
|
|
17617
|
+
const order = Uint32Array.from(rows);
|
|
17618
|
+
byDocument.set(ordinal, sortRows(keys, order, new Uint32Array(order.length), order.length));
|
|
17619
|
+
}
|
|
17620
|
+
return { keys, frequencies: fromRows ? request.rowFrequencies : request.pool.frequencies, byDocument };
|
|
17621
|
+
}
|
|
17622
|
+
function componentsOf(found, ordinal) {
|
|
17623
|
+
let held = found.get(ordinal);
|
|
17624
|
+
if (held === void 0) {
|
|
17625
|
+
held = { termFrequencies: {}, fieldLengths: {}, idf: {} };
|
|
17626
|
+
found.set(ordinal, held);
|
|
17627
|
+
}
|
|
17628
|
+
return held;
|
|
17629
|
+
}
|
|
17630
|
+
function lengthIn(request, field, ordinal) {
|
|
17631
|
+
const fieldCount = request.fields.searchable.length;
|
|
17632
|
+
const average = request.fields.numbers[FIELD_AVERAGE_LENGTHS * fieldCount + field];
|
|
17633
|
+
const column = request.fields.lengths[field];
|
|
17634
|
+
const stored = column !== null && ordinal < column.length ? column[ordinal] : 0;
|
|
17635
|
+
return stored > 0 ? stored : average;
|
|
17636
|
+
}
|
|
17637
|
+
function rowScore(request, frequency, idf, field, ordinal) {
|
|
17638
|
+
const fieldCount = request.fields.searchable.length;
|
|
17639
|
+
const numbers = request.fields.numbers;
|
|
17640
|
+
const bm25 = computeBM25WithIDF(
|
|
17641
|
+
frequency,
|
|
17642
|
+
idf,
|
|
17643
|
+
lengthIn(request, field, ordinal),
|
|
17644
|
+
numbers[FIELD_AVERAGE_LENGTHS * fieldCount + field],
|
|
17645
|
+
numbers[FIELD_K1 * fieldCount + field],
|
|
17646
|
+
numbers[FIELD_B * fieldCount + field]
|
|
17647
|
+
);
|
|
17648
|
+
return bm25 * numbers[FIELD_BOOSTS * fieldCount + field];
|
|
17649
|
+
}
|
|
17650
|
+
function searchableField(request, field) {
|
|
17651
|
+
return field < request.fields.searchable.length && request.fields.searchable[field] !== 0;
|
|
17652
|
+
}
|
|
17653
|
+
function record(request, found, names, source, rows, ordinal, label, idf) {
|
|
17654
|
+
for (let at = 0; at < rows.length; at++) {
|
|
17655
|
+
const row = rows[at];
|
|
17656
|
+
const field = source.keys.fields[row];
|
|
17657
|
+
if (!searchableField(request, field)) continue;
|
|
17658
|
+
const components = componentsOf(found, ordinal);
|
|
17659
|
+
const name = names[field];
|
|
17660
|
+
components.termFrequencies[`${name}:${label}`] = source.frequencies[row];
|
|
17661
|
+
components.fieldLengths[name] = lengthIn(request, field, ordinal);
|
|
17662
|
+
components.idf[label] = idf;
|
|
17663
|
+
}
|
|
17664
|
+
}
|
|
17665
|
+
function groupWinners(request, wanted, listFrom, listEnd, idf) {
|
|
17666
|
+
const winners = /* @__PURE__ */ new Map();
|
|
17667
|
+
for (let list = listFrom; list < listEnd; list++) {
|
|
17668
|
+
const factor = request.listValues[list * LIST_VALUE_STRIDE + LIST_FACTOR];
|
|
17669
|
+
const partialFactor = request.listValues[list * LIST_VALUE_STRIDE + LIST_PARTIAL_FACTOR];
|
|
17670
|
+
const source = rowsOfList(request, list, wanted);
|
|
17671
|
+
for (const [ordinal, rows] of source.byDocument) {
|
|
17672
|
+
if (!scorable(request, ordinal)) continue;
|
|
17673
|
+
let sum = 0;
|
|
17674
|
+
const kept = [];
|
|
17675
|
+
for (const row of rows) {
|
|
17676
|
+
const field = source.keys.fields[row];
|
|
17677
|
+
if (!searchableField(request, field)) continue;
|
|
17678
|
+
const rowFactor = request.fields.partialMarks[field] === 1 ? partialFactor : factor;
|
|
17679
|
+
if (!(rowFactor > 0)) continue;
|
|
17680
|
+
sum += rowScore(request, source.frequencies[row], idf, field, ordinal) * rowFactor;
|
|
17681
|
+
kept.push(row);
|
|
17682
|
+
}
|
|
17683
|
+
if (kept.length === 0) continue;
|
|
17684
|
+
const best = winners.get(ordinal);
|
|
17685
|
+
if (best === void 0 || sum > best.sum) winners.set(ordinal, { sum, rows: kept, list, source });
|
|
17686
|
+
}
|
|
17687
|
+
}
|
|
17688
|
+
return winners;
|
|
17689
|
+
}
|
|
17690
|
+
function keywordComponents(request, labels, names, ordinals, count) {
|
|
17691
|
+
const wanted = /* @__PURE__ */ new Set();
|
|
17692
|
+
for (let at = 0; at < count; at++) wanted.add(ordinals[at]);
|
|
17693
|
+
const found = /* @__PURE__ */ new Map();
|
|
17694
|
+
const { slots, settings, listValues, slotValues } = request;
|
|
17695
|
+
const totalDocuments = settings[SETTING_TOTAL_DOCUMENTS];
|
|
17696
|
+
for (let slot = 0; slot < settings[SETTING_SLOTS]; slot++) {
|
|
17697
|
+
const base = slot * SLOT_STRIDE;
|
|
17698
|
+
const listFrom = slot === 0 ? 0 : slots[base - SLOT_STRIDE + SLOT_LIST_END];
|
|
17699
|
+
const listEnd = slots[base + SLOT_LIST_END];
|
|
17700
|
+
const valueFrom = slot === 0 ? 0 : slots[base - SLOT_STRIDE + SLOT_VALUE_END];
|
|
17701
|
+
const valueEnd = slots[base + SLOT_VALUE_END];
|
|
17702
|
+
const kind = slots[base + SLOT_KIND];
|
|
17703
|
+
if (kind === GROUP_SLOT) {
|
|
17704
|
+
const idf = computeIDF(slotValues[valueFrom], totalDocuments);
|
|
17705
|
+
for (const [ordinal, winner] of groupWinners(request, wanted, listFrom, listEnd, idf)) {
|
|
17706
|
+
record(request, found, names, winner.source, winner.rows, ordinal, labels[winner.list], idf);
|
|
17707
|
+
}
|
|
17708
|
+
continue;
|
|
17709
|
+
}
|
|
17710
|
+
let phraseIdf = 0;
|
|
17711
|
+
if (kind === PHRASE_SLOT) {
|
|
17712
|
+
for (let value = valueFrom; value < valueEnd; value++) phraseIdf += computeIDF(slotValues[value], totalDocuments);
|
|
17713
|
+
}
|
|
17714
|
+
for (let list = listFrom; list < listEnd; list++) {
|
|
17715
|
+
const idf = kind === PHRASE_SLOT ? phraseIdf : computeIDF(listValues[list * LIST_VALUE_STRIDE + LIST_DOCUMENT_FREQUENCY], totalDocuments);
|
|
17716
|
+
const source = rowsOfList(request, list, wanted);
|
|
17717
|
+
for (const [ordinal, rows] of source.byDocument) {
|
|
17718
|
+
if (scorable(request, ordinal)) record(request, found, names, source, rows, ordinal, labels[list], idf);
|
|
17719
|
+
}
|
|
17720
|
+
}
|
|
17721
|
+
}
|
|
17722
|
+
return found;
|
|
17723
|
+
}
|
|
17724
|
+
|
|
17725
|
+
// src/core/partition/keyword-fields.ts
|
|
17726
|
+
var columns = {
|
|
17727
|
+
searchable: new Uint8Array(0),
|
|
17728
|
+
partialMarks: new Uint8Array(0),
|
|
17729
|
+
nameRanks: new Uint16Array(0),
|
|
17730
|
+
numbers: new Float64Array(0),
|
|
17731
|
+
lengths: [],
|
|
17732
|
+
rankedNames: null
|
|
17733
|
+
};
|
|
17734
|
+
function sizedFor(fieldCount) {
|
|
17735
|
+
if (columns.searchable.length === fieldCount) return;
|
|
17736
|
+
columns.searchable = new Uint8Array(fieldCount);
|
|
17737
|
+
columns.partialMarks = new Uint8Array(fieldCount);
|
|
17738
|
+
columns.nameRanks = new Uint16Array(fieldCount);
|
|
17739
|
+
columns.numbers = new Float64Array(fieldCount * FIELD_NUMBER_COLUMNS);
|
|
17740
|
+
columns.lengths = new Array(fieldCount).fill(null);
|
|
17741
|
+
columns.rankedNames = null;
|
|
17742
|
+
}
|
|
17743
|
+
function rankNames(names) {
|
|
17744
|
+
if (columns.rankedNames === names) return;
|
|
17745
|
+
const order = names.map((_, index) => index).sort((left, right) => compareCodePoints(names[left], names[right]));
|
|
17746
|
+
for (let rank = 0; rank < order.length; rank++) columns.nameRanks[order[rank]] = rank;
|
|
17747
|
+
columns.rankedNames = names;
|
|
17748
|
+
}
|
|
17749
|
+
function loadKeywordFields(state, fields, boost, averageFieldLengths, bm25, partialFields) {
|
|
17750
|
+
const names = state.fieldNameTable.names;
|
|
17751
|
+
const fieldCount = names.length;
|
|
17752
|
+
sizedFor(fieldCount);
|
|
17753
|
+
rankNames(names);
|
|
17754
|
+
const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
|
|
17755
|
+
const b = bm25?.b ?? DEFAULT_BM25_B;
|
|
17756
|
+
const perField = bm25?.fields;
|
|
17757
|
+
const { searchable, partialMarks, numbers, lengths } = columns;
|
|
17758
|
+
for (let field = 0; field < fieldCount; field++) {
|
|
17759
|
+
const name = names[field];
|
|
17760
|
+
searchable[field] = fields === void 0 || fields.includes(name) ? 1 : 0;
|
|
17761
|
+
partialMarks[field] = partialFields?.includes(name) ? 1 : 0;
|
|
17762
|
+
numbers[FIELD_BOOSTS * fieldCount + field] = boost?.[name] ?? 1;
|
|
17763
|
+
numbers[FIELD_AVERAGE_LENGTHS * fieldCount + field] = averageFieldLengths[name] ?? 1;
|
|
17764
|
+
const pair = perField !== void 0 && Object.hasOwn(perField, name) ? perField[name] : void 0;
|
|
17765
|
+
numbers[FIELD_K1 * fieldCount + field] = pair?.k1 ?? k1;
|
|
17766
|
+
numbers[FIELD_B * fieldCount + field] = pair?.b ?? b;
|
|
17767
|
+
lengths[field] = state.docStore.fieldLengthColumn(name);
|
|
17768
|
+
}
|
|
17769
|
+
return columns;
|
|
17770
|
+
}
|
|
17771
|
+
|
|
17772
|
+
// src/core/partition/keyword-slots.ts
|
|
17773
|
+
var planned = { labels: [] };
|
|
17774
|
+
function frequencyOf(state, params, token, list) {
|
|
17775
|
+
const stats = params.globalStats;
|
|
17776
|
+
if (stats !== void 0 && Object.hasOwn(stats.docFrequencies, token)) return stats.docFrequencies[token];
|
|
17777
|
+
if (list !== void 0) return list.docIdSet.size;
|
|
17778
|
+
return state.invertedIdx.documentFrequency(token);
|
|
17779
|
+
}
|
|
17780
|
+
function planGroup(state, params, builder2, group, term) {
|
|
17781
|
+
const lookup = (token) => state.invertedIdx.lookup(token);
|
|
17782
|
+
const synonyms = group.synonyms;
|
|
17783
|
+
let documentFrequency = synonyms?.documentFrequency((token) => frequencyOf(state, params, token, void 0)) ?? 0;
|
|
17784
|
+
const members = [];
|
|
17785
|
+
for (const member of group.members) {
|
|
17786
|
+
documentFrequency = Math.max(documentFrequency, frequencyOf(state, params, member.token, void 0));
|
|
17787
|
+
const list = synonyms !== void 0 && member.token === group.token ? synonyms.postings(lookup, member.token) : lookup(member.token);
|
|
17788
|
+
if (list !== void 0)
|
|
17789
|
+
members.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, list });
|
|
17790
|
+
}
|
|
17791
|
+
builder2.groupSlot(term, documentFrequency);
|
|
17792
|
+
for (const member of members) {
|
|
17793
|
+
builder2.memberList(member.list, member.factor, member.partialFactor);
|
|
17794
|
+
planned.labels.push(member.token);
|
|
17795
|
+
}
|
|
17796
|
+
}
|
|
17797
|
+
function planPhrases(state, params, builder2, phraseMatches) {
|
|
17798
|
+
for (const rows of phraseMatches.rows) {
|
|
17799
|
+
const frequencies = rows.phrase.tokens.map((token, at) => frequencyOf(state, params, token, rows.lists[at]));
|
|
17800
|
+
builder2.phraseSlot(frequencies, {
|
|
17801
|
+
docIds: rows.docIds.values,
|
|
17802
|
+
fields: rows.fieldIndices.values,
|
|
17803
|
+
frequencies: rows.frequencies.values,
|
|
17804
|
+
count: rows.docIds.length
|
|
17805
|
+
});
|
|
17806
|
+
planned.labels.push(rows.phrase.key);
|
|
17807
|
+
}
|
|
17808
|
+
}
|
|
17809
|
+
function planKeywordSlots(state, params, builder2, phraseMatches) {
|
|
17810
|
+
planned.labels.length = 0;
|
|
17811
|
+
const { queryTokens, termGroups } = params;
|
|
17812
|
+
for (let term = 0; term < queryTokens.length; term++) {
|
|
17813
|
+
const token = queryTokens[term].token;
|
|
17814
|
+
const group = termGroupOf(termGroups, token);
|
|
17815
|
+
const matches = typoMatchesFor(
|
|
17816
|
+
state.invertedIdx,
|
|
17817
|
+
token,
|
|
17818
|
+
group,
|
|
17819
|
+
params.exact ?? false,
|
|
17820
|
+
params.tolerance ?? 0,
|
|
17821
|
+
params.prefixLength ?? 2
|
|
17822
|
+
);
|
|
17823
|
+
builder2.termSlot(term);
|
|
17824
|
+
for (const match of matches) {
|
|
17825
|
+
builder2.termList(match.postingList, frequencyOf(state, params, match.token, match.postingList));
|
|
17826
|
+
planned.labels.push(match.token);
|
|
17827
|
+
}
|
|
17828
|
+
if (group !== void 0) planGroup(state, params, builder2, group, term);
|
|
17829
|
+
}
|
|
17830
|
+
if (phraseMatches !== null) planPhrases(state, params, builder2, phraseMatches);
|
|
17831
|
+
return planned;
|
|
17832
|
+
}
|
|
17833
|
+
|
|
17834
|
+
// src/core/keyword-score/request.ts
|
|
17835
|
+
var EMPTY_POOL = {
|
|
17836
|
+
docIds: new Uint32Array(0),
|
|
17837
|
+
frequencies: new Uint16Array(0),
|
|
17838
|
+
fields: new Uint16Array(0)
|
|
17839
|
+
};
|
|
17840
|
+
function grown(values, needed) {
|
|
17841
|
+
if (values.length >= needed) return values;
|
|
17842
|
+
const sized = new values.constructor(Math.max(needed, values.length * 2));
|
|
17843
|
+
sized.set(values);
|
|
17844
|
+
return sized;
|
|
17845
|
+
}
|
|
17846
|
+
function sharesPool(column, pooled, start) {
|
|
17847
|
+
if (!(column instanceof Uint32Array || column instanceof Uint16Array)) return false;
|
|
17848
|
+
if (column.buffer !== pooled.buffer || column.BYTES_PER_ELEMENT !== pooled.BYTES_PER_ELEMENT) return false;
|
|
17849
|
+
return column.byteOffset - pooled.byteOffset === start * pooled.BYTES_PER_ELEMENT;
|
|
17850
|
+
}
|
|
17851
|
+
var KeywordRequestBuilder = class {
|
|
17852
|
+
pool = EMPTY_POOL;
|
|
17853
|
+
slots = new Uint32Array(KEYWORD_FIRST_SLOTS * SLOT_STRIDE);
|
|
17854
|
+
slotCount = 0;
|
|
17855
|
+
lists = new Uint32Array(KEYWORD_FIRST_LISTS * LIST_STRIDE);
|
|
17856
|
+
listCount = 0;
|
|
17857
|
+
listValues = new Float64Array(KEYWORD_FIRST_LISTS * LIST_VALUE_STRIDE);
|
|
17858
|
+
slotValues = new Float64Array(KEYWORD_FIRST_SLOTS);
|
|
17859
|
+
valueCount = 0;
|
|
17860
|
+
rowDocIds = new Uint32Array(KEYWORD_FIRST_ROWS);
|
|
17861
|
+
rowFrequencies = new Float64Array(KEYWORD_FIRST_ROWS);
|
|
17862
|
+
rowFields = new Uint16Array(KEYWORD_FIRST_ROWS);
|
|
17863
|
+
rowCount = 0;
|
|
17864
|
+
settings = new Float64Array(SETTING_COUNT);
|
|
17865
|
+
begin(pool) {
|
|
17866
|
+
this.pool = pool ?? EMPTY_POOL;
|
|
17867
|
+
this.slotCount = 0;
|
|
17868
|
+
this.listCount = 0;
|
|
17869
|
+
this.valueCount = 0;
|
|
17870
|
+
this.rowCount = 0;
|
|
17871
|
+
}
|
|
17872
|
+
release() {
|
|
17873
|
+
this.pool = EMPTY_POOL;
|
|
17874
|
+
}
|
|
17875
|
+
get slotsWritten() {
|
|
17876
|
+
return this.slotCount;
|
|
17877
|
+
}
|
|
17878
|
+
termSlot(term) {
|
|
17879
|
+
this.openSlot(TERM_SLOT, term);
|
|
17880
|
+
}
|
|
17881
|
+
groupSlot(term, documentFrequency) {
|
|
17882
|
+
this.openSlot(GROUP_SLOT, term);
|
|
17883
|
+
this.pushSlotValue(documentFrequency);
|
|
17884
|
+
}
|
|
17885
|
+
phraseSlot(documentFrequencies, rows) {
|
|
17886
|
+
this.openSlot(PHRASE_SLOT, 0);
|
|
17887
|
+
for (const frequency of documentFrequencies) this.pushSlotValue(frequency);
|
|
17888
|
+
const start = this.reserveRows(rows.count);
|
|
17889
|
+
let ordered = true;
|
|
17890
|
+
for (let row = 0; row < rows.count; row++) {
|
|
17891
|
+
const at = start + row;
|
|
17892
|
+
this.rowDocIds[at] = rows.docIds[row];
|
|
17893
|
+
this.rowFields[at] = rows.fields[row];
|
|
17894
|
+
this.rowFrequencies[at] = rows.frequencies[row];
|
|
17895
|
+
if (row > 0 && this.rowDocIds[at] < this.rowDocIds[at - 1]) ordered = false;
|
|
17896
|
+
}
|
|
17897
|
+
this.pushList(LIST_FROM_ROWS | (ordered ? LIST_ORDERED : 0), start, rows.count, 0, 0);
|
|
17898
|
+
}
|
|
17899
|
+
termList(list, documentFrequency) {
|
|
17900
|
+
this.addList(list, documentFrequency, 0);
|
|
17901
|
+
}
|
|
17902
|
+
memberList(list, factor, partialFactor) {
|
|
17903
|
+
this.addList(list, factor, partialFactor);
|
|
17904
|
+
}
|
|
17905
|
+
request(scope, selection) {
|
|
17906
|
+
const settings = this.settings;
|
|
17907
|
+
settings[SETTING_SLOTS] = this.slotCount;
|
|
17908
|
+
settings[SETTING_TERMS] = selection.terms;
|
|
17909
|
+
settings[SETTING_CAPACITY] = scope.capacity;
|
|
17910
|
+
settings[SETTING_TOTAL_DOCUMENTS] = selection.totalDocuments;
|
|
17911
|
+
settings[SETTING_TERM_MATCH] = selection.termMatch;
|
|
17912
|
+
settings[SETTING_ANCHOR] = selection.anchor;
|
|
17913
|
+
settings[SETTING_LIMIT] = selection.limit;
|
|
17914
|
+
settings[SETTING_MIN_SCORE] = selection.minScore;
|
|
17915
|
+
settings[SETTING_COLLECT] = selection.collect ? 1 : 0;
|
|
17916
|
+
return {
|
|
17917
|
+
pool: this.pool,
|
|
17918
|
+
rowDocIds: this.rowDocIds,
|
|
17919
|
+
rowFrequencies: this.rowFrequencies,
|
|
17920
|
+
rowFields: this.rowFields,
|
|
17921
|
+
slots: this.slots,
|
|
17922
|
+
lists: this.lists,
|
|
17923
|
+
listValues: this.listValues,
|
|
17924
|
+
slotValues: this.slotValues,
|
|
17925
|
+
fields: scope.fields,
|
|
17926
|
+
filter: scope.filter,
|
|
17927
|
+
live: scope.live,
|
|
17928
|
+
removed: scope.removed,
|
|
17929
|
+
settings
|
|
17930
|
+
};
|
|
17931
|
+
}
|
|
17932
|
+
openSlot(kind, term) {
|
|
17933
|
+
this.slots = grown(this.slots, (this.slotCount + 1) * SLOT_STRIDE);
|
|
17934
|
+
const base = this.slotCount * SLOT_STRIDE;
|
|
17935
|
+
this.slots[base + SLOT_KIND] = kind;
|
|
17936
|
+
this.slots[base + SLOT_TERM] = term;
|
|
17937
|
+
this.slots[base + SLOT_LIST_END] = this.listCount;
|
|
17938
|
+
this.slots[base + SLOT_VALUE_END] = this.valueCount;
|
|
17939
|
+
this.slotCount++;
|
|
17940
|
+
}
|
|
17941
|
+
pushSlotValue(value) {
|
|
17942
|
+
this.slotValues = grown(this.slotValues, this.valueCount + 1);
|
|
17943
|
+
this.slotValues[this.valueCount++] = value;
|
|
17944
|
+
this.slots[(this.slotCount - 1) * SLOT_STRIDE + SLOT_VALUE_END] = this.valueCount;
|
|
17945
|
+
}
|
|
17946
|
+
pushList(flags, start, length, first, second) {
|
|
17947
|
+
this.lists = grown(this.lists, (this.listCount + 1) * LIST_STRIDE);
|
|
17948
|
+
this.listValues = grown(this.listValues, (this.listCount + 1) * LIST_VALUE_STRIDE);
|
|
17949
|
+
const base = this.listCount * LIST_STRIDE;
|
|
17950
|
+
this.lists[base] = flags;
|
|
17951
|
+
this.lists[base + 1] = start;
|
|
17952
|
+
this.lists[base + 2] = length;
|
|
17953
|
+
this.listValues[this.listCount * LIST_VALUE_STRIDE] = first;
|
|
17954
|
+
this.listValues[this.listCount * LIST_VALUE_STRIDE + 1] = second;
|
|
17955
|
+
this.listCount++;
|
|
17956
|
+
this.slots[(this.slotCount - 1) * SLOT_STRIDE + SLOT_LIST_END] = this.listCount;
|
|
17957
|
+
}
|
|
17958
|
+
reserveRows(count) {
|
|
17959
|
+
const start = this.rowCount;
|
|
17960
|
+
const needed = start + count;
|
|
17961
|
+
this.rowDocIds = grown(this.rowDocIds, needed);
|
|
17962
|
+
this.rowFrequencies = grown(this.rowFrequencies, needed);
|
|
17963
|
+
this.rowFields = grown(this.rowFields, needed);
|
|
17964
|
+
this.rowCount = needed;
|
|
17965
|
+
return start;
|
|
17966
|
+
}
|
|
17967
|
+
addList(list, first, second) {
|
|
17968
|
+
const docIds = list.docIds;
|
|
17969
|
+
const frequencies = list.termFrequencies;
|
|
17970
|
+
const fields = list.fieldNameIndices;
|
|
17971
|
+
const length = Math.min(list.length, docIds.length, frequencies.length, fields.length);
|
|
17972
|
+
const orderedFlag = list.ordered ? LIST_ORDERED : 0;
|
|
17973
|
+
const pool = this.pool;
|
|
17974
|
+
if (docIds instanceof Uint32Array && docIds.buffer === pool.docIds.buffer) {
|
|
17975
|
+
const start2 = (docIds.byteOffset - pool.docIds.byteOffset) / Uint32Array.BYTES_PER_ELEMENT;
|
|
17976
|
+
if (Number.isInteger(start2) && start2 >= 0 && start2 + length <= pool.docIds.length && start2 + length <= pool.frequencies.length && start2 + length <= pool.fields.length && sharesPool(frequencies, pool.frequencies, start2) && sharesPool(fields, pool.fields, start2)) {
|
|
17977
|
+
this.pushList(orderedFlag, start2, length, first, second);
|
|
17978
|
+
return;
|
|
17979
|
+
}
|
|
17980
|
+
}
|
|
17981
|
+
const start = this.reserveRows(length);
|
|
17982
|
+
for (let row = 0; row < length; row++) {
|
|
17983
|
+
this.rowDocIds[start + row] = docIds[row];
|
|
17984
|
+
this.rowFrequencies[start + row] = frequencies[row];
|
|
17985
|
+
this.rowFields[start + row] = fields[row];
|
|
17986
|
+
}
|
|
17987
|
+
this.pushList(LIST_FROM_ROWS | orderedFlag, start, length, first, second);
|
|
17988
|
+
}
|
|
17989
|
+
};
|
|
17990
|
+
|
|
17991
|
+
// src/core/keyword-score/segment-storage.ts
|
|
17992
|
+
var bitsByTombstones = /* @__PURE__ */ new WeakMap();
|
|
17993
|
+
function isFrozenSegment(state) {
|
|
17994
|
+
return "tombstones" in state && "arrays" in state && "paged" in state;
|
|
17995
|
+
}
|
|
17996
|
+
function tombstoneBits(tombstones) {
|
|
17997
|
+
let held = bitsByTombstones.get(tombstones);
|
|
17998
|
+
if (held === void 0) {
|
|
17999
|
+
held = { bits: new Uint32Array(0), read: 0 };
|
|
18000
|
+
bitsByTombstones.set(tombstones, held);
|
|
18001
|
+
}
|
|
18002
|
+
for (; held.read < tombstones.size; held.read++) {
|
|
18003
|
+
const ordinal = tombstones.at(held.read);
|
|
18004
|
+
const word = ordinal >>> 5;
|
|
18005
|
+
if (word >= held.bits.length) {
|
|
18006
|
+
const grown2 = new Uint32Array(Math.max(word + 1, held.bits.length * 2));
|
|
18007
|
+
grown2.set(held.bits);
|
|
18008
|
+
held.bits = grown2;
|
|
18009
|
+
}
|
|
18010
|
+
held.bits[word] |= 1 << (ordinal & 31);
|
|
18011
|
+
}
|
|
18012
|
+
return held.bits;
|
|
18013
|
+
}
|
|
18014
|
+
function segmentStorageOf(state) {
|
|
18015
|
+
if (isFrozenSegment(state)) {
|
|
18016
|
+
const { arrays } = state;
|
|
18017
|
+
const pool2 = state.paged === null ? { docIds: arrays.postingDocIds, frequencies: arrays.postingFrequencies, fields: arrays.postingFieldIndices } : null;
|
|
18018
|
+
return { pool: pool2, removed: tombstoneBits(state.tombstones) };
|
|
18019
|
+
}
|
|
18020
|
+
const postings = state.invertedIdx.livePostings;
|
|
18021
|
+
if (postings === void 0) return null;
|
|
18022
|
+
const { pool } = postings;
|
|
18023
|
+
return {
|
|
18024
|
+
pool: { docIds: pool.docIds, frequencies: pool.frequencies, fields: pool.fieldIndices },
|
|
18025
|
+
removed: postings.retired
|
|
18026
|
+
};
|
|
18027
|
+
}
|
|
18028
|
+
|
|
18029
|
+
// src/core/keyword-score/native.ts
|
|
18030
|
+
var TERM_MATCH_LEFT_TO_THE_CALLER = 0;
|
|
18031
|
+
var NO_MINIMUM_SCORE = 0;
|
|
18032
|
+
var FIRST_TERM_ANCHOR = 0;
|
|
18033
|
+
var builder = new KeywordRequestBuilder();
|
|
18034
|
+
var output = {
|
|
18035
|
+
ordinals: new Int32Array(0),
|
|
18036
|
+
scores: new Float64Array(0),
|
|
18037
|
+
matched: new Uint32Array(0),
|
|
18038
|
+
outcome: new Float64Array(OUTCOME_LENGTH)
|
|
18039
|
+
};
|
|
18040
|
+
var everyOrdinal = new Uint32Array(0);
|
|
18041
|
+
function outputFor(capacity) {
|
|
18042
|
+
if (output.ordinals.length < capacity) {
|
|
18043
|
+
output.ordinals = new Int32Array(capacity);
|
|
18044
|
+
output.scores = new Float64Array(capacity);
|
|
18045
|
+
output.matched = new Uint32Array(capacity + 31 >>> 5);
|
|
18046
|
+
}
|
|
18047
|
+
return output;
|
|
18048
|
+
}
|
|
18049
|
+
function everyOrdinalBelow(capacity) {
|
|
18050
|
+
const words = capacity + 31 >>> 5;
|
|
18051
|
+
if (everyOrdinal.length < words) everyOrdinal = new Uint32Array(Math.max(words, everyOrdinal.length * 2)).fill(-1);
|
|
18052
|
+
return everyOrdinal;
|
|
18053
|
+
}
|
|
18054
|
+
function scoredByTheCore(core, request, written) {
|
|
18055
|
+
const { pool, fields } = request;
|
|
18056
|
+
const status = core.keywordScore(
|
|
18057
|
+
pool.docIds,
|
|
18058
|
+
pool.frequencies,
|
|
18059
|
+
pool.fields,
|
|
18060
|
+
request.rowDocIds,
|
|
18061
|
+
request.rowFrequencies,
|
|
18062
|
+
request.rowFields,
|
|
18063
|
+
request.slots,
|
|
18064
|
+
request.lists,
|
|
18065
|
+
request.listValues,
|
|
18066
|
+
request.slotValues,
|
|
18067
|
+
fields.searchable,
|
|
18068
|
+
fields.partialMarks,
|
|
18069
|
+
fields.nameRanks,
|
|
18070
|
+
fields.numbers,
|
|
18071
|
+
fields.lengths,
|
|
18072
|
+
request.filter,
|
|
18073
|
+
request.live,
|
|
18074
|
+
request.removed,
|
|
18075
|
+
request.settings,
|
|
18076
|
+
written.ordinals,
|
|
18077
|
+
written.scores,
|
|
18078
|
+
written.matched,
|
|
18079
|
+
written.outcome
|
|
18080
|
+
);
|
|
18081
|
+
if (status === NATIVE_OK) return true;
|
|
18082
|
+
noteFallback("keywordScore");
|
|
18083
|
+
return false;
|
|
18084
|
+
}
|
|
18085
|
+
function byScoreThenId(left, right) {
|
|
18086
|
+
if (left.score !== right.score) return left.score > right.score ? -1 : 1;
|
|
18087
|
+
return compareCodePoints(left.docId, right.docId);
|
|
18088
|
+
}
|
|
18089
|
+
function pageOf(state, written, count, maxResults, components) {
|
|
18090
|
+
const resolver = state.docStore.resolver();
|
|
18091
|
+
const page = [];
|
|
18092
|
+
for (let at = 0; at < count; at++) {
|
|
18093
|
+
const ordinal = written.ordinals[at];
|
|
18094
|
+
const docId = resolver.toExternal(ordinal);
|
|
18095
|
+
if (docId === void 0) continue;
|
|
18096
|
+
const record2 = components?.get(ordinal);
|
|
18097
|
+
page.push({
|
|
18098
|
+
docId,
|
|
18099
|
+
score: written.scores[at],
|
|
18100
|
+
termFrequencies: record2?.termFrequencies ?? EMPTY_COMPONENTS,
|
|
18101
|
+
fieldLengths: record2?.fieldLengths ?? EMPTY_COMPONENTS,
|
|
18102
|
+
idf: record2?.idf ?? EMPTY_COMPONENTS
|
|
18103
|
+
});
|
|
18104
|
+
}
|
|
18105
|
+
page.sort(byScoreThenId);
|
|
18106
|
+
if (maxResults !== void 0 && page.length > maxResults) page.length = Math.max(0, Math.floor(maxResults));
|
|
18107
|
+
return page;
|
|
18108
|
+
}
|
|
18109
|
+
function matchedIdsOf(state, matched, capacity) {
|
|
18110
|
+
const resolver = state.docStore.resolver();
|
|
18111
|
+
const ids = [];
|
|
18112
|
+
const words = Math.min(matched.length, capacity + 31 >>> 5);
|
|
18113
|
+
for (let word = 0; word < words; word++) {
|
|
18114
|
+
let bits = matched[word];
|
|
18115
|
+
while (bits !== 0) {
|
|
18116
|
+
const low = bits & -bits;
|
|
18117
|
+
const ordinal = word * 32 + (31 - Math.clz32(low));
|
|
18118
|
+
const docId = resolver.toExternal(ordinal);
|
|
18119
|
+
if (docId !== void 0) ids.push(docId);
|
|
18120
|
+
bits ^= low;
|
|
18121
|
+
}
|
|
18122
|
+
}
|
|
18123
|
+
return ids;
|
|
18124
|
+
}
|
|
18125
|
+
function plannedRequest(state, params, phraseMatches, storage, capacity) {
|
|
18126
|
+
const { queryTokens, globalStats, maxResults } = params;
|
|
18127
|
+
const fields = loadKeywordFields(
|
|
18128
|
+
state,
|
|
18129
|
+
params.fields,
|
|
18130
|
+
params.boost,
|
|
18131
|
+
globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths,
|
|
18132
|
+
params.bm25Params,
|
|
18133
|
+
params.partialFields
|
|
18134
|
+
);
|
|
18135
|
+
builder.begin(storage.pool);
|
|
18136
|
+
const planned2 = planKeywordSlots(state, params, builder, phraseMatches);
|
|
18137
|
+
const request = builder.request(
|
|
18138
|
+
{
|
|
18139
|
+
fields,
|
|
18140
|
+
filter: phraseMatches?.matched ?? params.filterBitset ?? null,
|
|
18141
|
+
live: everyOrdinalBelow(capacity),
|
|
18142
|
+
removed: storage.removed,
|
|
18143
|
+
capacity
|
|
18144
|
+
},
|
|
18145
|
+
{
|
|
18146
|
+
terms: queryTokens.length,
|
|
18147
|
+
totalDocuments: globalStats?.totalDocuments ?? state.stats.totalDocuments,
|
|
18148
|
+
termMatch: TERM_MATCH_LEFT_TO_THE_CALLER,
|
|
18149
|
+
anchor: FIRST_TERM_ANCHOR,
|
|
18150
|
+
limit: maxResults === void 0 ? NO_LIMIT : Math.max(0, Math.floor(maxResults)),
|
|
18151
|
+
minScore: NO_MINIMUM_SCORE,
|
|
18152
|
+
collect: params.collectMatchedSet !== void 0
|
|
18153
|
+
}
|
|
18154
|
+
);
|
|
18155
|
+
return { request, labels: planned2.labels };
|
|
18156
|
+
}
|
|
18157
|
+
function searchInTheCore(state, params, phraseMatches) {
|
|
18158
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
18159
|
+
const core = loadNativeCore();
|
|
18160
|
+
if (core === null) return null;
|
|
18161
|
+
const storage = segmentStorageOf(state);
|
|
18162
|
+
if (storage === null) return null;
|
|
18163
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
18164
|
+
const written = outputFor(capacity);
|
|
18165
|
+
const components = componentsScoredByTheCore(core, state, params, phraseMatches, storage, capacity, written);
|
|
18166
|
+
if (components === void 0) return null;
|
|
18167
|
+
const totalMatched = written.outcome[OUTCOME_TOTAL_MATCHED];
|
|
18168
|
+
const scored = pageOf(state, written, written.outcome[OUTCOME_COUNT], params.maxResults, components);
|
|
18169
|
+
const result = params.collectMatchedSet === void 0 ? { scored, totalMatched } : params.collectMatchedSet === "ordinals" ? { scored, totalMatched, matchedOrdinalBitset: written.matched.slice(0, capacity + 31 >>> 5 || 1) } : { scored, totalMatched, matchedIds: matchedIdsOf(state, written.matched, capacity) };
|
|
18170
|
+
return result;
|
|
18171
|
+
}
|
|
18172
|
+
function componentsScoredByTheCore(core, state, params, phraseMatches, storage, capacity, written) {
|
|
18173
|
+
try {
|
|
18174
|
+
const { request, labels } = plannedRequest(state, params, phraseMatches, storage, capacity);
|
|
18175
|
+
if (!scoredByTheCore(core, request, written)) return void 0;
|
|
18176
|
+
if (params.collectComponents === false) return null;
|
|
18177
|
+
const count = written.outcome[OUTCOME_COUNT];
|
|
18178
|
+
return keywordComponents(request, labels, state.fieldNameTable.names, written.ordinals, count);
|
|
18179
|
+
} finally {
|
|
18180
|
+
builder.release();
|
|
18181
|
+
}
|
|
18182
|
+
}
|
|
18183
|
+
|
|
18184
|
+
// src/core/tokenizer/native-analysis.ts
|
|
18185
|
+
var STEM_FLAG = 1;
|
|
18186
|
+
var REMOVE_STOP_WORDS_FLAG = 2;
|
|
18187
|
+
var REMOVE_DIACRITICS_FLAG = 4;
|
|
18188
|
+
var COLLECT_SURFACES_FLAG = 8;
|
|
18189
|
+
var SPELLINGS_FLAG = 16;
|
|
18190
|
+
var TOKEN_RECORD_WORDS = 8;
|
|
18191
|
+
var NO_SURFACE = 4294967295;
|
|
18192
|
+
var COPY_PREFIX = " ";
|
|
18193
|
+
var coreOfNames = null;
|
|
18194
|
+
var indexByName = /* @__PURE__ */ new Map();
|
|
18195
|
+
var languagesOfCore = /* @__PURE__ */ new WeakMap();
|
|
18196
|
+
var heldStopWords = /* @__PURE__ */ new WeakMap();
|
|
18197
|
+
var sharedRecords = new Uint32Array(1 + NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS * TOKEN_RECORD_WORDS);
|
|
18198
|
+
var recordsInUse = false;
|
|
18199
|
+
var termsBySlot = null;
|
|
18200
|
+
function activeCore() {
|
|
18201
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
18202
|
+
const core = loadNativeCore();
|
|
18203
|
+
if (core === null) return null;
|
|
18204
|
+
if (core !== coreOfNames) {
|
|
18205
|
+
coreOfNames = core;
|
|
18206
|
+
indexByName = /* @__PURE__ */ new Map();
|
|
18207
|
+
languagesOfCore = /* @__PURE__ */ new WeakMap();
|
|
18208
|
+
heldStopWords = /* @__PURE__ */ new WeakMap();
|
|
18209
|
+
termsBySlot = null;
|
|
18210
|
+
const count = core.languageCount();
|
|
18211
|
+
for (let index = 0; index < count; index++) indexByName.set(core.languageName(index), index);
|
|
18212
|
+
}
|
|
18213
|
+
return core;
|
|
18214
|
+
}
|
|
18215
|
+
function stringsOf(words) {
|
|
18216
|
+
const strings = [];
|
|
18217
|
+
for (const word of words) if (typeof word === "string") strings.push(word);
|
|
18218
|
+
return strings;
|
|
18219
|
+
}
|
|
18220
|
+
function holdsTheCoreList(core, index, words) {
|
|
18221
|
+
const coreWords = core.languageStopWords(index);
|
|
18222
|
+
if (coreWords.length !== words.size) return false;
|
|
18223
|
+
for (const word of coreWords) if (!words.has(word)) return false;
|
|
18224
|
+
return true;
|
|
18225
|
+
}
|
|
18226
|
+
function coreLanguageOf(core, module) {
|
|
18227
|
+
const known = languagesOfCore.get(module);
|
|
18228
|
+
if (known !== void 0) return known;
|
|
18229
|
+
const index = indexByName.get(module.name);
|
|
18230
|
+
let language = null;
|
|
18231
|
+
if (index !== void 0 && core.languageRevision(index) === module.revision) {
|
|
18232
|
+
const moduleStopWords = holdsTheCoreList(core, index, module.stopWords) ? null : core.createStopWordSet(stringsOf(module.stopWords));
|
|
18233
|
+
language = { index, moduleStopWords };
|
|
18234
|
+
}
|
|
18235
|
+
languagesOfCore.set(module, language);
|
|
18236
|
+
return language;
|
|
18237
|
+
}
|
|
18238
|
+
function holdsTheSameWords(held, words) {
|
|
18239
|
+
let strings = 0;
|
|
18240
|
+
for (const word of words) {
|
|
18241
|
+
if (typeof word !== "string") continue;
|
|
18242
|
+
if (!held.has(word)) return false;
|
|
18243
|
+
strings++;
|
|
18244
|
+
}
|
|
18245
|
+
return strings === held.size;
|
|
18246
|
+
}
|
|
18247
|
+
function heldSetFor(core, owner, words) {
|
|
18248
|
+
const held = heldStopWords.get(owner);
|
|
18249
|
+
if (held !== void 0 && holdsTheSameWords(held.words, words)) return held.handle;
|
|
18250
|
+
const strings = stringsOf(words);
|
|
18251
|
+
const handle = core.createStopWordSet(strings);
|
|
18252
|
+
heldStopWords.set(owner, { handle, words: new Set(strings) });
|
|
18253
|
+
return handle;
|
|
18254
|
+
}
|
|
18255
|
+
function stopWordsFor(core, language, module, override) {
|
|
18256
|
+
if (!override) return language.moduleStopWords;
|
|
18257
|
+
const words = override instanceof Set ? override : override(module.stopWords);
|
|
18258
|
+
if (words === module.stopWords) return language.moduleStopWords;
|
|
18259
|
+
return heldSetFor(core, override, words);
|
|
18260
|
+
}
|
|
18261
|
+
function analyseWith(core, index, flags, text, stopWords) {
|
|
18262
|
+
let records = recordsInUse ? new Uint32Array(sharedRecords.length) : sharedRecords;
|
|
18263
|
+
let result = core.analyse(index, flags, text, stopWords, records);
|
|
18264
|
+
if (result === null) {
|
|
18265
|
+
records = new Uint32Array(1 + records[0] * TOKEN_RECORD_WORDS);
|
|
18266
|
+
if (!recordsInUse) sharedRecords = records;
|
|
18267
|
+
result = core.analyse(index, flags, text, stopWords, records);
|
|
18268
|
+
}
|
|
18269
|
+
if (typeof result !== "string") {
|
|
18270
|
+
noteFallback("analyse");
|
|
18271
|
+
return null;
|
|
18272
|
+
}
|
|
18273
|
+
return { arena: result, records };
|
|
18274
|
+
}
|
|
18275
|
+
function holdsUnits(held, arena, start, length) {
|
|
18276
|
+
if (held.length !== length) return false;
|
|
18277
|
+
for (let unit = 0; unit < length; unit++) {
|
|
18278
|
+
if (held.charCodeAt(unit) !== arena.charCodeAt(start + unit)) return false;
|
|
18279
|
+
}
|
|
18280
|
+
return true;
|
|
18281
|
+
}
|
|
18282
|
+
function termOf(arena, start, length, slot) {
|
|
18283
|
+
if (slot >= NATIVE_ANALYSIS_HELD_TERMS) return arena.substring(start, start + length);
|
|
18284
|
+
if (termsBySlot === null) termsBySlot = new Array(NATIVE_ANALYSIS_HELD_TERMS);
|
|
18285
|
+
const held = termsBySlot[slot];
|
|
18286
|
+
if (held !== void 0 && holdsUnits(held, arena, start, length)) return held;
|
|
18287
|
+
const term = (COPY_PREFIX + arena.substring(start, start + length)).slice(COPY_PREFIX.length);
|
|
18288
|
+
termsBySlot[slot] = term;
|
|
18289
|
+
return term;
|
|
18290
|
+
}
|
|
18291
|
+
function visitRecords(analysed, visit) {
|
|
18292
|
+
const { arena, records } = analysed;
|
|
18293
|
+
const count = records[0];
|
|
18294
|
+
const ownsShared = records === sharedRecords;
|
|
18295
|
+
if (ownsShared) recordsInUse = true;
|
|
18296
|
+
try {
|
|
18297
|
+
for (let token = 0, at = 1; token < count; token++, at += TOKEN_RECORD_WORDS) {
|
|
18298
|
+
const surfaceStart = records[at + 3];
|
|
18299
|
+
const originalStart = records[at + 5];
|
|
18300
|
+
visit(
|
|
18301
|
+
termOf(arena, records[at + 1], records[at + 2], records[at + 7]),
|
|
18302
|
+
records[at],
|
|
18303
|
+
surfaceStart === NO_SURFACE ? void 0 : arena.substring(surfaceStart, surfaceStart + records[at + 4]),
|
|
18304
|
+
arena.substring(originalStart, originalStart + records[at + 6])
|
|
18305
|
+
);
|
|
18306
|
+
}
|
|
18307
|
+
} finally {
|
|
18308
|
+
if (ownsShared) recordsInUse = false;
|
|
18309
|
+
}
|
|
18310
|
+
}
|
|
18311
|
+
function visitTokens2(text, language, options, visit) {
|
|
18312
|
+
const core = activeCore();
|
|
18313
|
+
if (core === null) return false;
|
|
18314
|
+
const coreLanguage = coreLanguageOf(core, language);
|
|
18315
|
+
if (coreLanguage === null) return false;
|
|
18316
|
+
const removeStopWords = options?.removeStopWords ?? true;
|
|
18317
|
+
const flags = (options?.stem ?? true ? STEM_FLAG : 0) | (removeStopWords ? REMOVE_STOP_WORDS_FLAG : 0) | (options?.removeDiacritics === true ? REMOVE_DIACRITICS_FLAG : 0) | (options?.collectSurfaces === true ? COLLECT_SURFACES_FLAG : 0);
|
|
18318
|
+
const stopWords = removeStopWords ? stopWordsFor(core, coreLanguage, language, options?.stopWordOverride) : null;
|
|
18319
|
+
const analysed = analyseWith(core, coreLanguage.index, flags, text, stopWords);
|
|
18320
|
+
if (analysed === null) return false;
|
|
18321
|
+
visitRecords(analysed, visit);
|
|
18322
|
+
return true;
|
|
18323
|
+
}
|
|
18324
|
+
function visitSpellings(text, language, visit) {
|
|
18325
|
+
const core = activeCore();
|
|
18326
|
+
if (core === null) return false;
|
|
18327
|
+
const coreLanguage = coreLanguageOf(core, language);
|
|
18328
|
+
if (coreLanguage === null) return false;
|
|
18329
|
+
const analysed = analyseWith(core, coreLanguage.index, SPELLINGS_FLAG, text, null);
|
|
18330
|
+
if (analysed === null) return false;
|
|
18331
|
+
visitRecords(analysed, (spelling) => visit(spelling));
|
|
18332
|
+
return true;
|
|
18333
|
+
}
|
|
18334
|
+
function transform(raw, language, stem, removeDiacritics) {
|
|
18335
|
+
const core = activeCore();
|
|
18336
|
+
if (core === null) return null;
|
|
18337
|
+
const coreLanguage = coreLanguageOf(core, language);
|
|
18338
|
+
if (coreLanguage === null) return null;
|
|
18339
|
+
const result = core.transform(
|
|
18340
|
+
coreLanguage.index,
|
|
18341
|
+
(stem ? STEM_FLAG : 0) | (removeDiacritics ? REMOVE_DIACRITICS_FLAG : 0),
|
|
18342
|
+
raw
|
|
18343
|
+
);
|
|
18344
|
+
if (typeof result === "string") return result;
|
|
18345
|
+
noteFallback("analyse");
|
|
18346
|
+
return null;
|
|
18347
|
+
}
|
|
18348
|
+
var nativeCoreAnalysis = { visitTokens: visitTokens2, visitSpellings, transform };
|
|
18349
|
+
|
|
18350
|
+
// src/engine/search-backend-settings.react-native.ts
|
|
18351
|
+
function requireSearchBackendSettings() {
|
|
18352
|
+
installNativeAnalysis(nativeCoreAnalysis);
|
|
18353
|
+
installNativeKeywordSearch(searchInTheCore);
|
|
18354
|
+
}
|
|
18355
|
+
|
|
17605
18356
|
// src/engine/startup-checks.ts
|
|
17606
18357
|
var announcedMissingWorkerEntry = false;
|
|
17607
18358
|
function withoutWorkers(config, reason) {
|
|
@@ -17614,6 +18365,7 @@ function withoutWorkers(config, reason) {
|
|
|
17614
18365
|
return { ...config, workers: { ...config?.workers, enabled: false } };
|
|
17615
18366
|
}
|
|
17616
18367
|
async function resolveRunnableConfig(config) {
|
|
18368
|
+
requireSearchBackendSettings();
|
|
17617
18369
|
if (!(config?.workers?.enabled ?? workersEnabledByDefault())) return config;
|
|
17618
18370
|
const runtime = detectRuntime();
|
|
17619
18371
|
if (!runtime.supportsWorkerThreads && !runtime.supportsWebWorkers) return config;
|
|
@@ -17858,8 +18610,8 @@ async function announceIndexDropped(core, name, config) {
|
|
|
17858
18610
|
}
|
|
17859
18611
|
async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
|
|
17860
18612
|
const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
|
|
17861
|
-
const
|
|
17862
|
-
if (
|
|
18613
|
+
const loadingSchemaCode = loadSchemaCodeFor(requestedConfig.schema);
|
|
18614
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
17863
18615
|
core.guardShutdown();
|
|
17864
18616
|
validateIndexName(name);
|
|
17865
18617
|
if (core.indexRegistry.has(name)) {
|
|
@@ -17970,7 +18722,7 @@ async function dropEngineIndex(core, name) {
|
|
|
17970
18722
|
const entry = core.requireIndex(name);
|
|
17971
18723
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
17972
18724
|
const abandonChange = async () => {
|
|
17973
|
-
await (await import('./analysis-change-entry-
|
|
18725
|
+
await (await import('./analysis-change-entry-OC4XASBF.mjs')).abandonChangeOf(core, name);
|
|
17974
18726
|
};
|
|
17975
18727
|
if (changeRecorded()) await abandonChange();
|
|
17976
18728
|
await core.indexState.drop(name, async () => {
|
|
@@ -18526,14 +19278,8 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
18526
19278
|
} catch (err) {
|
|
18527
19279
|
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
18528
19280
|
}
|
|
18529
|
-
|
|
18530
|
-
|
|
18531
|
-
indexName,
|
|
18532
|
-
docId: resolvedDocId,
|
|
18533
|
-
document: partitionDoc,
|
|
18534
|
-
requestId: `replicate-insert-${resolvedDocId}`,
|
|
18535
|
-
skipClone: options?.skipClone
|
|
18536
|
-
});
|
|
19281
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
19282
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
18537
19283
|
for (const fieldPath of extractedVectors.keys()) {
|
|
18538
19284
|
const vecIndex = insertVecIndexes.get(fieldPath);
|
|
18539
19285
|
if (vecIndex) {
|
|
@@ -18744,6 +19490,8 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
|
|
|
18744
19490
|
}
|
|
18745
19491
|
succeeded.push(doc.docId);
|
|
18746
19492
|
}
|
|
19493
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
19494
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
18747
19495
|
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
18748
19496
|
if (applied < admitted.length) {
|
|
18749
19497
|
ctx.checkHeapPressure(indexName);
|
|
@@ -18788,75 +19536,34 @@ function buildSegmentRequests(indexName, docIds, documents, partitionCount, skip
|
|
|
18788
19536
|
}
|
|
18789
19537
|
return { requests, memberIndexes };
|
|
18790
19538
|
}
|
|
18791
|
-
function
|
|
18792
|
-
|
|
18793
|
-
|
|
18794
|
-
const snapshot = segment.snapshot ?? (segment.payload === null ? null : freezeSegmentShared(segment.payload, segment.documents, segment.segmentId));
|
|
18795
|
-
if (snapshot === null) return null;
|
|
18796
|
-
frozen.push({ partitionId: segment.partitionId, snapshot });
|
|
18797
|
-
}
|
|
18798
|
-
return frozen;
|
|
18799
|
-
}
|
|
18800
|
-
function tryFreezeSegmentsForAttach(segments) {
|
|
19539
|
+
function sharedSnapshotOf(segment) {
|
|
19540
|
+
if (segment.snapshot !== null) return segment.snapshot;
|
|
19541
|
+
if (segment.payload === null) return null;
|
|
18801
19542
|
try {
|
|
18802
|
-
return
|
|
19543
|
+
return freezeSegmentShared(segment.payload, segment.documents, segment.segmentId);
|
|
18803
19544
|
} catch (err) {
|
|
18804
19545
|
console.warn("Segment freeze failed, replicating by merge instead:", err);
|
|
18805
19546
|
return null;
|
|
18806
19547
|
}
|
|
18807
19548
|
}
|
|
18808
|
-
function
|
|
18809
|
-
const
|
|
18810
|
-
|
|
18811
|
-
|
|
18812
|
-
|
|
18813
|
-
|
|
18814
|
-
|
|
18815
|
-
|
|
18816
|
-
}
|
|
18817
|
-
function snapshotSegments(segments) {
|
|
18818
|
-
const shared = [];
|
|
18819
|
-
for (const segment of segments) {
|
|
18820
|
-
if (segment.snapshot !== null) shared.push({ partitionId: segment.partitionId, snapshot: segment.snapshot });
|
|
18821
|
-
}
|
|
18822
|
-
return shared;
|
|
18823
|
-
}
|
|
18824
|
-
function attachSegments2(orchestrator, indexName, segments) {
|
|
18825
|
-
return orchestrator.replicateToWorkers({
|
|
18826
|
-
type: "attachSegments",
|
|
18827
|
-
indexName,
|
|
18828
|
-
segments,
|
|
18829
|
-
requestId: `attach-segments-${indexName}-${segments.length}`
|
|
18830
|
-
});
|
|
18831
|
-
}
|
|
18832
|
-
async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
|
|
18833
|
-
const frozen = tryFreezeSegmentsForAttach(segments);
|
|
18834
|
-
if (frozen !== null) {
|
|
18835
|
-
await attachSegments2(orchestrator, indexName, frozen);
|
|
18836
|
-
return;
|
|
19549
|
+
function copyActionOfSegment(indexName, segment, skipClone) {
|
|
19550
|
+
const snapshot = sharedSnapshotOf(segment);
|
|
19551
|
+
if (snapshot !== null) {
|
|
19552
|
+
return {
|
|
19553
|
+
type: "attachSegments",
|
|
19554
|
+
indexName,
|
|
19555
|
+
segments: [{ partitionId: segment.partitionId, snapshot }],
|
|
19556
|
+
requestId: `attach-segments-${indexName}-${segment.segmentId}`
|
|
19557
|
+
};
|
|
18837
19558
|
}
|
|
18838
|
-
|
|
18839
|
-
|
|
18840
|
-
const plain = payloadSegments(segments);
|
|
18841
|
-
if (plain.length === 0) return;
|
|
18842
|
-
await orchestrator.replicateToWorkers({
|
|
19559
|
+
if (segment.payload === null) return null;
|
|
19560
|
+
return {
|
|
18843
19561
|
type: "mergeSegments",
|
|
18844
19562
|
indexName,
|
|
18845
|
-
segments:
|
|
18846
|
-
requestId: `merge-segments-${indexName}-${
|
|
19563
|
+
segments: [{ partitionId: segment.partitionId, payload: segment.payload, documents: segment.documents }],
|
|
19564
|
+
requestId: `merge-segments-${indexName}-${segment.segmentId}`,
|
|
18847
19565
|
skipClone: skipClone === true ? true : void 0
|
|
18848
|
-
}
|
|
18849
|
-
}
|
|
18850
|
-
async function replicateAsSegments(ctx, indexName, docIds, documents, skipClone) {
|
|
18851
|
-
if (docIds.length < MIN_DOCUMENTS_FOR_SEGMENTS) return false;
|
|
18852
|
-
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
18853
|
-
if (workers <= 0) return false;
|
|
18854
|
-
const manager = ctx.requireManager(indexName);
|
|
18855
|
-
const { requests } = buildSegmentRequests(indexName, docIds, documents, manager.partitionCount, skipClone);
|
|
18856
|
-
const built = await ctx.orchestrator.buildSegments(requests);
|
|
18857
|
-
if (built === null || built.length === 0) return false;
|
|
18858
|
-
await broadcastBuiltSegments(ctx.orchestrator, indexName, built, skipClone);
|
|
18859
|
-
return true;
|
|
19566
|
+
};
|
|
18860
19567
|
}
|
|
18861
19568
|
|
|
18862
19569
|
// src/engine/mutations/insert-batch-segments.ts
|
|
@@ -18864,26 +19571,11 @@ function mainStoreDocument(doc, options) {
|
|
|
18864
19571
|
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
18865
19572
|
return structuredClone(doc.partitionDoc);
|
|
18866
19573
|
}
|
|
18867
|
-
async function replicateDocuments(ctx, indexName, docs, options) {
|
|
18868
|
-
for (const doc of docs) {
|
|
18869
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
18870
|
-
type: "insert",
|
|
18871
|
-
indexName,
|
|
18872
|
-
docId: doc.docId,
|
|
18873
|
-
document: doc.partitionDoc,
|
|
18874
|
-
requestId: `replicate-insert-${doc.docId}`,
|
|
18875
|
-
skipClone: options?.skipClone
|
|
18876
|
-
});
|
|
18877
|
-
}
|
|
18878
|
-
}
|
|
18879
19574
|
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
18880
19575
|
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
18881
|
-
const succeededIds = new Set(applied.succeeded);
|
|
18882
|
-
const replicable = admitted.filter((doc) => succeededIds.has(doc.docId) && !applied.buffered.has(doc.docId));
|
|
18883
|
-
await replicateDocuments(ctx, indexName, replicable, options);
|
|
18884
19576
|
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
18885
19577
|
}
|
|
18886
|
-
function attachOnce(manager, attachment) {
|
|
19578
|
+
function attachOnce(ctx, manager, attachment) {
|
|
18887
19579
|
if (attachment.failure !== void 0) throw attachment.failure;
|
|
18888
19580
|
if (attachment.attached) return;
|
|
18889
19581
|
try {
|
|
@@ -18895,11 +19587,12 @@ function attachOnce(manager, attachment) {
|
|
|
18895
19587
|
attachment.failure = error;
|
|
18896
19588
|
throw error;
|
|
18897
19589
|
}
|
|
19590
|
+
if (attachment.copyAction !== null) ctx.orchestrator.recordMainWrite(attachment.copyAction);
|
|
18898
19591
|
}
|
|
18899
19592
|
function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
18900
19593
|
const manager = ctx.requireManager(indexName);
|
|
18901
19594
|
const attachToIndex = async () => {
|
|
18902
|
-
attachOnce(manager, attachment);
|
|
19595
|
+
attachOnce(ctx, manager, attachment);
|
|
18903
19596
|
progress.applied = true;
|
|
18904
19597
|
try {
|
|
18905
19598
|
insertDocumentVectors(
|
|
@@ -18923,13 +19616,14 @@ function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
|
18923
19616
|
};
|
|
18924
19617
|
return indexChangeOutsideExternalWaits(attachToIndex);
|
|
18925
19618
|
}
|
|
18926
|
-
function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
19619
|
+
function chunksOfWholeSegments(indexName, built, frozen, memberIndexes, admitted, options) {
|
|
18927
19620
|
const chunks = [];
|
|
18928
19621
|
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
18929
19622
|
for (let i = 0; i < built.length; i++) {
|
|
18930
19623
|
const attachment = {
|
|
18931
19624
|
partitionId: built[i].partitionId,
|
|
18932
19625
|
segments: [frozen[i]],
|
|
19626
|
+
copyAction: copyActionOfSegment(indexName, built[i], options?.skipClone),
|
|
18933
19627
|
attached: false,
|
|
18934
19628
|
failure: void 0
|
|
18935
19629
|
};
|
|
@@ -18946,12 +19640,12 @@ function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
|
18946
19640
|
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
18947
19641
|
return chunks;
|
|
18948
19642
|
}
|
|
18949
|
-
async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, failed) {
|
|
19643
|
+
async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, options, failed) {
|
|
18950
19644
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
18951
19645
|
const succeededIds = /* @__PURE__ */ new Set();
|
|
18952
19646
|
const errorOf = /* @__PURE__ */ new Map();
|
|
18953
19647
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
18954
|
-
const chunks = chunksOfWholeSegments(built, frozen, memberIndexes, admitted);
|
|
19648
|
+
const chunks = chunksOfWholeSegments(indexName, built, frozen, memberIndexes, admitted, options);
|
|
18955
19649
|
for (let index = 0; index < chunks.length; index++) {
|
|
18956
19650
|
const { docs, attachmentOf } = chunks[index];
|
|
18957
19651
|
const progress = docs.map(() => ({ applied: false }));
|
|
@@ -18986,40 +19680,23 @@ async function recordMergedDocuments(ctx, indexName, built, frozen, memberIndexe
|
|
|
18986
19680
|
}
|
|
18987
19681
|
succeededIds.add(doc.docId);
|
|
18988
19682
|
}
|
|
19683
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName, "segments");
|
|
19684
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
18989
19685
|
if (index + 1 < chunks.length) {
|
|
18990
19686
|
ctx.checkHeapPressure(indexName);
|
|
18991
19687
|
await new Promise((r) => setTimeout(r, 0));
|
|
18992
19688
|
}
|
|
18993
19689
|
}
|
|
18994
19690
|
const succeeded = [];
|
|
18995
|
-
const failedDocIds = /* @__PURE__ */ new Set();
|
|
18996
19691
|
for (const doc of admitted) {
|
|
18997
19692
|
if (succeededIds.has(doc.docId)) {
|
|
18998
19693
|
succeeded.push(doc.docId);
|
|
18999
19694
|
continue;
|
|
19000
19695
|
}
|
|
19001
19696
|
if (!errorOf.has(doc.docId)) continue;
|
|
19002
|
-
failedDocIds.add(doc.docId);
|
|
19003
19697
|
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
19004
19698
|
}
|
|
19005
|
-
return { succeeded, touchedVectorFields
|
|
19006
|
-
}
|
|
19007
|
-
async function broadcastSegments(ctx, indexName, built, memberIndexes, admitted, failedDocIds, options) {
|
|
19008
|
-
const clean = [];
|
|
19009
|
-
const retryDocs = [];
|
|
19010
|
-
for (let i = 0; i < built.length; i++) {
|
|
19011
|
-
const members = memberIndexes[i].map((m) => admitted[m]);
|
|
19012
|
-
if (members.some((doc) => failedDocIds.has(doc.docId))) {
|
|
19013
|
-
retryDocs.push(...members.filter((doc) => !failedDocIds.has(doc.docId)));
|
|
19014
|
-
continue;
|
|
19015
|
-
}
|
|
19016
|
-
clean.push(built[i]);
|
|
19017
|
-
}
|
|
19018
|
-
if (clean.length > 0) {
|
|
19019
|
-
await broadcastBuiltSegments(ctx.orchestrator, indexName, clean, options?.skipClone);
|
|
19020
|
-
}
|
|
19021
|
-
await replicateDocuments(ctx, indexName, retryDocs, options);
|
|
19022
|
-
return clean.map((segment) => segment.segmentId);
|
|
19699
|
+
return { succeeded, touchedVectorFields };
|
|
19023
19700
|
}
|
|
19024
19701
|
function frozenSegmentOf(segment, members, options) {
|
|
19025
19702
|
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
@@ -19073,20 +19750,7 @@ async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
|
19073
19750
|
for (const doc of admitted) {
|
|
19074
19751
|
if (manager.has(doc.docId)) return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
19075
19752
|
}
|
|
19076
|
-
|
|
19077
|
-
ctx.orchestrator.holdUnbroadcastSegments(indexName, segmentIds);
|
|
19078
|
-
const recorded = await recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, failed);
|
|
19079
|
-
const broadcast = await broadcastSegments(
|
|
19080
|
-
ctx,
|
|
19081
|
-
indexName,
|
|
19082
|
-
built,
|
|
19083
|
-
memberIndexes,
|
|
19084
|
-
admitted,
|
|
19085
|
-
recorded.failedDocIds,
|
|
19086
|
-
options
|
|
19087
|
-
);
|
|
19088
|
-
ctx.orchestrator.releaseUnbroadcastSegments(indexName, broadcast);
|
|
19089
|
-
return { succeeded: recorded.succeeded, touchedVectorFields: recorded.touchedVectorFields };
|
|
19753
|
+
return recordMergedDocuments(ctx, indexName, built, frozen, memberIndexes, admitted, options, failed);
|
|
19090
19754
|
}
|
|
19091
19755
|
async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
19092
19756
|
const failed = [];
|
|
@@ -19114,7 +19778,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19114
19778
|
return insertBatchViaSegments(ctx, indexName, documents, options);
|
|
19115
19779
|
}
|
|
19116
19780
|
const succeeded = [];
|
|
19117
|
-
const succeededDocs = [];
|
|
19118
19781
|
const failed = [];
|
|
19119
19782
|
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
19120
19783
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
@@ -19123,7 +19786,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19123
19786
|
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
19124
19787
|
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
19125
19788
|
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
19126
|
-
const bufferedDocIds = /* @__PURE__ */ new Set();
|
|
19127
19789
|
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
19128
19790
|
if (ctx.abortController.signal.aborted) break;
|
|
19129
19791
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
@@ -19151,9 +19813,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19151
19813
|
continue;
|
|
19152
19814
|
}
|
|
19153
19815
|
if (application.status === "buffered") {
|
|
19154
|
-
bufferedDocIds.add(doc.docId);
|
|
19155
19816
|
succeeded.push(doc.docId);
|
|
19156
|
-
succeededDocs.push(doc.partitionDoc);
|
|
19157
19817
|
continue;
|
|
19158
19818
|
}
|
|
19159
19819
|
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
@@ -19167,7 +19827,6 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19167
19827
|
}
|
|
19168
19828
|
}
|
|
19169
19829
|
succeeded.push(doc.docId);
|
|
19170
|
-
succeededDocs.push(doc.partitionDoc);
|
|
19171
19830
|
}
|
|
19172
19831
|
}
|
|
19173
19832
|
const prepared = [];
|
|
@@ -19201,32 +19860,13 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
19201
19860
|
}
|
|
19202
19861
|
}
|
|
19203
19862
|
if (prepared.length > 0) await applyPrepared(prepared);
|
|
19863
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
19864
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19204
19865
|
if (chunkEnd < documents.length) {
|
|
19205
19866
|
ctx.checkHeapPressure(indexName);
|
|
19206
19867
|
await new Promise((r) => setTimeout(r, 0));
|
|
19207
19868
|
}
|
|
19208
19869
|
}
|
|
19209
|
-
const replicableIds = [];
|
|
19210
|
-
const replicableDocs = [];
|
|
19211
|
-
for (let i = 0; i < succeeded.length; i++) {
|
|
19212
|
-
if (bufferedDocIds.has(succeeded[i])) continue;
|
|
19213
|
-
replicableIds.push(succeeded[i]);
|
|
19214
|
-
replicableDocs.push(succeededDocs[i]);
|
|
19215
|
-
}
|
|
19216
|
-
const replicatedAsSegments = copiesBuildSegments && await replicateAsSegments(ctx, indexName, replicableIds, replicableDocs, options?.skipClone);
|
|
19217
|
-
if (!replicatedAsSegments) {
|
|
19218
|
-
for (let i = 0; i < replicableIds.length; i++) {
|
|
19219
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
19220
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
19221
|
-
type: "insert",
|
|
19222
|
-
indexName,
|
|
19223
|
-
docId: replicableIds[i],
|
|
19224
|
-
document: replicableDocs[i],
|
|
19225
|
-
requestId: `replicate-insert-${replicableIds[i]}`,
|
|
19226
|
-
skipClone: options?.skipClone
|
|
19227
|
-
});
|
|
19228
|
-
}
|
|
19229
|
-
}
|
|
19230
19870
|
for (const fieldPath of touchedVectorFields) {
|
|
19231
19871
|
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
19232
19872
|
if (vecIndex) {
|
|
@@ -19282,19 +19922,14 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
19282
19922
|
return indexChangeOutsideExternalWaits(removeFromIndex);
|
|
19283
19923
|
}
|
|
19284
19924
|
async function settleRemoval(ctx, indexName, prepared) {
|
|
19285
|
-
const { docId
|
|
19925
|
+
const { docId } = prepared;
|
|
19286
19926
|
try {
|
|
19287
19927
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
19288
19928
|
} catch (err) {
|
|
19289
19929
|
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
19290
19930
|
}
|
|
19291
|
-
|
|
19292
|
-
|
|
19293
|
-
indexName,
|
|
19294
|
-
docId,
|
|
19295
|
-
requestId: `replicate-remove-${docId}`,
|
|
19296
|
-
removedSpellings
|
|
19297
|
-
});
|
|
19931
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName, "none");
|
|
19932
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19298
19933
|
}
|
|
19299
19934
|
async function removeDocument(ctx, indexName, docId, options) {
|
|
19300
19935
|
ctx.guardShutdown();
|
|
@@ -19392,14 +20027,8 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
19392
20027
|
} catch (err) {
|
|
19393
20028
|
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
19394
20029
|
}
|
|
19395
|
-
|
|
19396
|
-
|
|
19397
|
-
indexName,
|
|
19398
|
-
docId: prepared.docId,
|
|
19399
|
-
document: prepared.partitionDoc,
|
|
19400
|
-
requestId: `replicate-update-${prepared.docId}`,
|
|
19401
|
-
removedSpellings: prepared.removedSpellings
|
|
19402
|
-
});
|
|
20030
|
+
const roomForCopies = ctx.orchestrator.settleMainWrites(indexName);
|
|
20031
|
+
if (roomForCopies !== null) await roomForCopies;
|
|
19403
20032
|
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
19404
20033
|
for (const [fieldPath, vec] of prepared.extractedVectors) {
|
|
19405
20034
|
if (vec === null) continue;
|
|
@@ -19746,7 +20375,6 @@ async function withRenewedSyncEpoch(core, indexName, change) {
|
|
|
19746
20375
|
function clearEngineIndex(core, indexName) {
|
|
19747
20376
|
return withRenewedSyncEpoch(core, indexName, async () => {
|
|
19748
20377
|
await core.executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
19749
|
-
await core.orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
19750
20378
|
core.watermarkNotifier.forget(indexName);
|
|
19751
20379
|
const durable = core.durability?.manager;
|
|
19752
20380
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
@@ -19843,7 +20471,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
19843
20471
|
const { restoresInFlight } = core;
|
|
19844
20472
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
19845
20473
|
const restored = previous.then(async () => {
|
|
19846
|
-
const { runRestore } = await import('./restore-queue-
|
|
20474
|
+
const { runRestore } = await import('./restore-queue-4R6AASE5.mjs');
|
|
19847
20475
|
await runRestore(core, restoreHost, indexName, data, engine());
|
|
19848
20476
|
});
|
|
19849
20477
|
const settled2 = restored.then(
|
|
@@ -19874,7 +20502,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
19874
20502
|
return {
|
|
19875
20503
|
snapshot(indexName, options) {
|
|
19876
20504
|
return withOpenIndex2(indexName, async () => {
|
|
19877
|
-
const { createSnapshot } = await import('./snapshot-
|
|
20505
|
+
const { createSnapshot } = await import('./snapshot-5LGPQXFS.mjs');
|
|
19878
20506
|
return createSnapshot(snapshotDepsOf(indexName), options?.queryHistory === true ? core : null);
|
|
19879
20507
|
});
|
|
19880
20508
|
},
|
|
@@ -19887,8 +20515,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
19887
20515
|
exportDocuments(indexName) {
|
|
19888
20516
|
return withOpenIndex2(indexName, async () => {
|
|
19889
20517
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
19890
|
-
import('./snapshot-
|
|
19891
|
-
import('./write-
|
|
20518
|
+
import('./snapshot-5LGPQXFS.mjs'),
|
|
20519
|
+
import('./write-6BRYIIZ7.mjs')
|
|
19892
20520
|
]);
|
|
19893
20521
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
19894
20522
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -19896,7 +20524,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
19896
20524
|
},
|
|
19897
20525
|
importDocumentExport(indexName, data) {
|
|
19898
20526
|
core.guardShutdown();
|
|
19899
|
-
const imported = import('./import-
|
|
20527
|
+
const imported = import('./import-5UAWBRJP.mjs').then(
|
|
19900
20528
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
19901
20529
|
);
|
|
19902
20530
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -19922,9 +20550,9 @@ function flush(stream) {
|
|
|
19922
20550
|
}
|
|
19923
20551
|
function ensureCapacity(stream, extra) {
|
|
19924
20552
|
if (stream.length + extra <= stream.bytes.length) return;
|
|
19925
|
-
const
|
|
19926
|
-
|
|
19927
|
-
stream.bytes =
|
|
20553
|
+
const grown2 = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
20554
|
+
grown2.set(stream.bytes.subarray(0, stream.length));
|
|
20555
|
+
stream.bytes = grown2;
|
|
19928
20556
|
}
|
|
19929
20557
|
function writeByte(stream, value) {
|
|
19930
20558
|
ensureCapacity(stream, 1);
|
|
@@ -20018,14 +20646,14 @@ function writeValue(stream, value) {
|
|
|
20018
20646
|
}
|
|
20019
20647
|
return;
|
|
20020
20648
|
}
|
|
20021
|
-
const
|
|
20022
|
-
const keys = Object.keys(
|
|
20649
|
+
const record2 = value;
|
|
20650
|
+
const keys = Object.keys(record2).sort(compareCodePoints);
|
|
20023
20651
|
writeByte(stream, MAP_TAG);
|
|
20024
20652
|
writeUint32(stream, keys.length);
|
|
20025
20653
|
for (const key of keys) {
|
|
20026
20654
|
writeByte(stream, STRING_TAG);
|
|
20027
20655
|
writeString(stream, key);
|
|
20028
|
-
writeValue(stream,
|
|
20656
|
+
writeValue(stream, record2[key] ?? null);
|
|
20029
20657
|
}
|
|
20030
20658
|
}
|
|
20031
20659
|
function boundVectorOf(vector) {
|
|
@@ -20047,9 +20675,9 @@ function boundRerankOf(rerank) {
|
|
|
20047
20675
|
}
|
|
20048
20676
|
return present;
|
|
20049
20677
|
}
|
|
20050
|
-
function presentMembersOf(
|
|
20678
|
+
function presentMembersOf(record2) {
|
|
20051
20679
|
const present = {};
|
|
20052
|
-
for (const [key, value] of Object.entries(
|
|
20680
|
+
for (const [key, value] of Object.entries(record2)) {
|
|
20053
20681
|
if (value !== void 0) present[key] = value;
|
|
20054
20682
|
}
|
|
20055
20683
|
return present;
|
|
@@ -21917,7 +22545,7 @@ function createNarsilFromCore(core, config) {
|
|
|
21917
22545
|
const entry = requireIndex(indexName);
|
|
21918
22546
|
await withOpenIndexWrite(indexName, async () => {
|
|
21919
22547
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
21920
|
-
return (await import('./analysis-change-
|
|
22548
|
+
return (await import('./analysis-change-QUVDNABS.mjs')).resumeAnalysisChange(core, indexName);
|
|
21921
22549
|
});
|
|
21922
22550
|
},
|
|
21923
22551
|
async changeAnalysis(indexName, change) {
|
|
@@ -21925,7 +22553,7 @@ function createNarsilFromCore(core, config) {
|
|
|
21925
22553
|
requireIndex(indexName);
|
|
21926
22554
|
const changed = withOpenIndex2(
|
|
21927
22555
|
indexName,
|
|
21928
|
-
async () => (await import('./analysis-change-
|
|
22556
|
+
async () => (await import('./analysis-change-QUVDNABS.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
21929
22557
|
);
|
|
21930
22558
|
await emitWriteEventAfter(core, indexName, changed);
|
|
21931
22559
|
},
|
|
@@ -21960,7 +22588,7 @@ function createNarsilFromCore(core, config) {
|
|
|
21960
22588
|
const rebalanced = withOpenIndex2(
|
|
21961
22589
|
indexName,
|
|
21962
22590
|
() => withRenewedSyncEpoch(core, indexName, async () => {
|
|
21963
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
22591
|
+
const { executeRebalance } = await import('./analysis-change-QUVDNABS.mjs');
|
|
21964
22592
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
21965
22593
|
releaseWrites();
|
|
21966
22594
|
await running;
|
|
@@ -22045,8 +22673,8 @@ function createNarsilFromCore(core, config) {
|
|
|
22045
22673
|
guardShutdown();
|
|
22046
22674
|
requireIndex(indexName);
|
|
22047
22675
|
const manager = executor.getManager(indexName);
|
|
22048
|
-
const
|
|
22049
|
-
return manager === void 0 ||
|
|
22676
|
+
const vectorCode2 = loadedVectorCode();
|
|
22677
|
+
return manager === void 0 || vectorCode2 === null ? [] : vectorCode2.getVectorMaintenanceStatus(manager);
|
|
22050
22678
|
},
|
|
22051
22679
|
on(event, handler) {
|
|
22052
22680
|
const key = event;
|
|
@@ -22084,7 +22712,7 @@ async function createNarsil(config) {
|
|
|
22084
22712
|
await startEngineCore(core, runnableConfig);
|
|
22085
22713
|
const engine = createNarsilFromCore(core, runnableConfig);
|
|
22086
22714
|
if (runnableConfig?.querySuggestions !== void 0 || core.durability !== null) {
|
|
22087
|
-
await (await import('./engine-hooks-
|
|
22715
|
+
await (await import('./engine-hooks-CWA5CKCA.mjs')).startQuerySuggestions(core, engine, runnableConfig);
|
|
22088
22716
|
}
|
|
22089
22717
|
return engine;
|
|
22090
22718
|
}
|
|
@@ -22092,4 +22720,4 @@ async function createNarsil(config) {
|
|
|
22092
22720
|
// src/index.ts
|
|
22093
22721
|
var VERSION = "0.1.0";
|
|
22094
22722
|
|
|
22095
|
-
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, REBALANCE_CHUNK_SIZE, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, dropEngineIndex, emitEngineEvent, emitWriteEventAfter, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineCoreOf, engineVersionParts, extractVectorFromDoc, fieldIndexRemapOf, freezeSegmentShared, generateId, holdSyncEpochChange, holdsADiscardedField, insertDocumentVectors, isCompositePartition,
|
|
22723
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, REBALANCE_CHUNK_SIZE, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, createCompositePartition, createEncodedDocumentSource, createEngineIndex, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, dropEngineIndex, emitEngineEvent, emitWriteEventAfter, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineCoreOf, engineVersionParts, extractVectorFromDoc, fieldIndexRemapOf, freezeSegmentShared, generateId, holdSyncEpochChange, holdsADiscardedField, insertDocumentVectors, isCompositePartition, loadSchemaCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, packChecksummedEnvelope, packEnvelopeWithChecksum, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerPartialWordSearch, registerPatternSearch, registerSynonyms, removeDocumentBatch2 as removeDocumentBatch, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, vectorsEqual, wrapFrozenTokenTable, yieldToEventLoop2 as yieldToEventLoop };
|