@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,1635 @@
|
|
|
1
|
+
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
2
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-65WCV5JO.mjs';
|
|
3
|
+
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-ZXX5GZEX.mjs';
|
|
4
|
+
import { isWordIndexedBase, parseFieldType, fuzzyTermMatches } from './chunk-KYVRZAYP.mjs';
|
|
5
|
+
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
6
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
8
|
+
import { compareCodePoints, codePointCount, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
|
+
import { Encoder, encode } from '@msgpack/msgpack';
|
|
10
|
+
|
|
11
|
+
// src/schema/validator/schema.ts
|
|
12
|
+
function validateSchemaFields(schema, depth, prefix) {
|
|
13
|
+
if (depth > MAX_NESTING_DEPTH) {
|
|
14
|
+
throw new NarsilError(
|
|
15
|
+
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
16
|
+
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
17
|
+
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
let canonical = null;
|
|
21
|
+
const replace = (field, value) => {
|
|
22
|
+
if (canonical === null) canonical = { ...schema };
|
|
23
|
+
canonical[field] = value;
|
|
24
|
+
};
|
|
25
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
26
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
27
|
+
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
28
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
29
|
+
field,
|
|
30
|
+
path
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
34
|
+
throw new NarsilError(
|
|
35
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
36
|
+
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
37
|
+
{ field, path }
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
41
|
+
throw new NarsilError(
|
|
42
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
43
|
+
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
44
|
+
{ field, path }
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
if (isPlainObject(type)) {
|
|
48
|
+
const nested = validateSchemaFields(type, depth + 1, path);
|
|
49
|
+
if (nested !== type) replace(field, nested);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (typeof type !== "string") {
|
|
53
|
+
throw new NarsilError(
|
|
54
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
55
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
56
|
+
{
|
|
57
|
+
field: path,
|
|
58
|
+
type: String(type)
|
|
59
|
+
}
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
const parsed = parseFieldType(type, path);
|
|
63
|
+
if (parsed.type !== type) replace(field, parsed.type);
|
|
64
|
+
}
|
|
65
|
+
return canonical ?? schema;
|
|
66
|
+
}
|
|
67
|
+
function validateSchema(schema) {
|
|
68
|
+
if (!isPlainObject(schema)) {
|
|
69
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
70
|
+
received: typeof schema
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (Object.keys(schema).length === 0) {
|
|
74
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
75
|
+
}
|
|
76
|
+
const validated = validateSchemaFields(schema, 1, "");
|
|
77
|
+
requireTextFieldCount(schemaFieldsOf(validated).filter((field) => isWordIndexedBase(field.base)).length);
|
|
78
|
+
return validated;
|
|
79
|
+
}
|
|
80
|
+
function requireTextFieldCount(count) {
|
|
81
|
+
if (count <= MAX_TEXT_FIELDS) return;
|
|
82
|
+
throw new NarsilError(
|
|
83
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
84
|
+
`The schema declares ${count} string and string[] fields, but a schema holds at most ${MAX_TEXT_FIELDS}`,
|
|
85
|
+
{ textFields: count, maximum: MAX_TEXT_FIELDS }
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
function requireCreatableSchema(schema) {
|
|
89
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
90
|
+
if (field.base !== "vector" || field.dimension <= MAX_VECTOR_DIMENSION) continue;
|
|
91
|
+
throw new NarsilError(
|
|
92
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
93
|
+
`Vector field "${field.path}" has ${field.dimension} dimensions, but a new index takes at most ${MAX_VECTOR_DIMENSION}`,
|
|
94
|
+
{ field: field.path, dimension: field.dimension, maximum: MAX_VECTOR_DIMENSION }
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function requireValidFieldTypes(flatSchema) {
|
|
99
|
+
let textFields = 0;
|
|
100
|
+
for (const [path, type] of Object.entries(flatSchema)) {
|
|
101
|
+
if (typeof type !== "string") {
|
|
102
|
+
throw new NarsilError(
|
|
103
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
104
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
105
|
+
{ field: path, type: String(type) }
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (isWordIndexedBase(parseFieldType(type, path).base)) textFields += 1;
|
|
109
|
+
}
|
|
110
|
+
requireTextFieldCount(textFields);
|
|
111
|
+
}
|
|
112
|
+
function flattenRecursive(schema, prefix, result) {
|
|
113
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
114
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
115
|
+
if (isPlainObject(type)) {
|
|
116
|
+
flattenRecursive(type, path, result);
|
|
117
|
+
} else {
|
|
118
|
+
result[path] = type;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
123
|
+
function flattenSchema(schema) {
|
|
124
|
+
const cached = flattenedSchemas.get(schema);
|
|
125
|
+
if (cached !== void 0) return cached;
|
|
126
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
127
|
+
flattenRecursive(schema, "", result);
|
|
128
|
+
flattenedSchemas.set(schema, Object.freeze(result));
|
|
129
|
+
return result;
|
|
130
|
+
}
|
|
131
|
+
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
132
|
+
function schemaFieldsOf(schema) {
|
|
133
|
+
const cached = parsedSchemas.get(schema);
|
|
134
|
+
if (cached !== void 0) return cached;
|
|
135
|
+
const fields = [];
|
|
136
|
+
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
137
|
+
fields.push({ ...parseFieldType(type, path), path });
|
|
138
|
+
}
|
|
139
|
+
const frozen = Object.freeze(fields);
|
|
140
|
+
parsedSchemas.set(schema, frozen);
|
|
141
|
+
return frozen;
|
|
142
|
+
}
|
|
143
|
+
var partialFieldsBySchema = /* @__PURE__ */ new WeakMap();
|
|
144
|
+
function partialFieldsOf(schema) {
|
|
145
|
+
const cached = partialFieldsBySchema.get(schema);
|
|
146
|
+
if (cached !== void 0) return cached;
|
|
147
|
+
const paths = [];
|
|
148
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
149
|
+
if (field.partial) paths.push(field.path);
|
|
150
|
+
}
|
|
151
|
+
const frozen = Object.freeze(paths.sort(compareCodePoints));
|
|
152
|
+
partialFieldsBySchema.set(schema, frozen);
|
|
153
|
+
return frozen;
|
|
154
|
+
}
|
|
155
|
+
function extractVectorFieldsFromSchema(schema) {
|
|
156
|
+
const result = /* @__PURE__ */ new Map();
|
|
157
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
158
|
+
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
159
|
+
}
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// src/geo/constants.ts
|
|
164
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
165
|
+
var MAX_LATITUDE_DEGREES = 90;
|
|
166
|
+
var MAX_LONGITUDE_DEGREES = 180;
|
|
167
|
+
var MIN_POLYGON_POINTS = 3;
|
|
168
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
169
|
+
var FULL_TURN_DEGREES = 360;
|
|
170
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
171
|
+
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
172
|
+
var MAX_COVERING_CELLS = 16;
|
|
173
|
+
var RADIUS_BOX_MARGIN_RATIO = 1.02;
|
|
174
|
+
var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
|
|
175
|
+
var VINCENTY_MAX_ITERATIONS = 200;
|
|
176
|
+
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
177
|
+
|
|
178
|
+
// src/geo/coordinates.ts
|
|
179
|
+
function isLatitude(value) {
|
|
180
|
+
return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
|
|
181
|
+
}
|
|
182
|
+
function isLongitude(value) {
|
|
183
|
+
return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// src/schema/validator/field-values.ts
|
|
187
|
+
function validateGeopoint(path, value) {
|
|
188
|
+
if (!isPlainObject(value)) {
|
|
189
|
+
throw new NarsilError(
|
|
190
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
191
|
+
`Field "${path}" expected a geopoint object with lat and lon`,
|
|
192
|
+
{
|
|
193
|
+
field: path,
|
|
194
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
195
|
+
}
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
|
|
199
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
|
|
200
|
+
field: path,
|
|
201
|
+
lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
|
|
202
|
+
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
if (!isLatitude(value.lat)) {
|
|
206
|
+
throw new NarsilError(
|
|
207
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
208
|
+
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
209
|
+
{
|
|
210
|
+
field: path,
|
|
211
|
+
lat: value.lat
|
|
212
|
+
}
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
if (!isLongitude(value.lon)) {
|
|
216
|
+
throw new NarsilError(
|
|
217
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
218
|
+
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
219
|
+
{
|
|
220
|
+
field: path,
|
|
221
|
+
lon: value.lon
|
|
222
|
+
}
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function validateVector(path, value, expectedDimension) {
|
|
227
|
+
const isArray = Array.isArray(value);
|
|
228
|
+
const isFloat32 = value instanceof Float32Array;
|
|
229
|
+
if (!isArray && !isFloat32) {
|
|
230
|
+
throw new NarsilError(
|
|
231
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
232
|
+
`Field "${path}" expected a vector (Array or Float32Array)`,
|
|
233
|
+
{ field: path, received: typeof value }
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
const arr = value;
|
|
237
|
+
if (arr.length !== expectedDimension) {
|
|
238
|
+
throw new NarsilError(
|
|
239
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
240
|
+
`Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
|
|
241
|
+
{ field: path, expected: expectedDimension, received: arr.length }
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
for (let i = 0; i < arr.length; i++) {
|
|
245
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
246
|
+
throw new NarsilError(
|
|
247
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
248
|
+
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
249
|
+
{ field: path, index: i, value: arr[i] }
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
function validateTypedArray(path, value, elementType) {
|
|
255
|
+
if (!Array.isArray(value)) {
|
|
256
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
|
|
257
|
+
field: path,
|
|
258
|
+
received: typeof value
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
for (let i = 0; i < value.length; i++) {
|
|
262
|
+
const el = value[i];
|
|
263
|
+
if (typeof el !== elementType) {
|
|
264
|
+
throw new NarsilError(
|
|
265
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
266
|
+
`Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
|
|
267
|
+
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
if (elementType === "number" && !Number.isFinite(el)) {
|
|
271
|
+
throw new NarsilError(
|
|
272
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
273
|
+
`Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
|
|
274
|
+
{ field: path, index: i }
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
function validatePatternValue(path, value, patternValueLimit, index) {
|
|
280
|
+
if (!exceedsCodePoints(value, patternValueLimit)) return;
|
|
281
|
+
const subject = index === void 0 ? `The value of field "${path}"` : `Element ${index} of field "${path}"`;
|
|
282
|
+
throw new NarsilError(
|
|
283
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
284
|
+
`${subject} is longer than ${patternValueLimit} code points, which is the pattern value limit of this index`,
|
|
285
|
+
{ field: path, limit: patternValueLimit, ...index === void 0 ? {} : { index } }
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
function validateFieldValue(path, value, type, patternValueLimit) {
|
|
289
|
+
switch (type.base) {
|
|
290
|
+
case "string":
|
|
291
|
+
case "verbatim":
|
|
292
|
+
if (typeof value !== "string") {
|
|
293
|
+
throw new NarsilError(
|
|
294
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
295
|
+
`Field "${path}" expected string, got ${typeof value}`,
|
|
296
|
+
{
|
|
297
|
+
field: path,
|
|
298
|
+
expected: "string",
|
|
299
|
+
received: typeof value
|
|
300
|
+
}
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
304
|
+
return;
|
|
305
|
+
case "number":
|
|
306
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
307
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
|
|
308
|
+
field: path,
|
|
309
|
+
expected: "number",
|
|
310
|
+
received: typeof value === "number" ? String(value) : typeof value
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
return;
|
|
314
|
+
case "boolean":
|
|
315
|
+
if (typeof value !== "boolean") {
|
|
316
|
+
throw new NarsilError(
|
|
317
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
318
|
+
`Field "${path}" expected boolean, got ${typeof value}`,
|
|
319
|
+
{
|
|
320
|
+
field: path,
|
|
321
|
+
expected: "boolean",
|
|
322
|
+
received: typeof value
|
|
323
|
+
}
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
return;
|
|
327
|
+
case "enum":
|
|
328
|
+
if (typeof value !== "string") {
|
|
329
|
+
throw new NarsilError(
|
|
330
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
331
|
+
`Field "${path}" expected enum (string), got ${typeof value}`,
|
|
332
|
+
{
|
|
333
|
+
field: path,
|
|
334
|
+
expected: "enum",
|
|
335
|
+
received: typeof value
|
|
336
|
+
}
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
return;
|
|
340
|
+
case "geopoint":
|
|
341
|
+
validateGeopoint(path, value);
|
|
342
|
+
return;
|
|
343
|
+
case "string[]":
|
|
344
|
+
case "verbatim[]":
|
|
345
|
+
validateTypedArray(path, value, "string");
|
|
346
|
+
if (type.pattern) {
|
|
347
|
+
const elements = value;
|
|
348
|
+
for (let i = 0; i < elements.length; i++) validatePatternValue(path, elements[i], patternValueLimit, i);
|
|
349
|
+
}
|
|
350
|
+
return;
|
|
351
|
+
case "number[]":
|
|
352
|
+
validateTypedArray(path, value, "number");
|
|
353
|
+
return;
|
|
354
|
+
case "boolean[]":
|
|
355
|
+
validateTypedArray(path, value, "boolean");
|
|
356
|
+
return;
|
|
357
|
+
case "enum[]":
|
|
358
|
+
validateTypedArray(path, value, "string");
|
|
359
|
+
return;
|
|
360
|
+
case "vector":
|
|
361
|
+
validateVector(path, value, type.dimension);
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// src/schema/validator/storable.ts
|
|
367
|
+
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
368
|
+
var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
|
|
369
|
+
function reject(path, reason) {
|
|
370
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
371
|
+
field: path
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
function isPlainObject2(value) {
|
|
375
|
+
const proto = Object.getPrototypeOf(value);
|
|
376
|
+
return proto === Object.prototype || proto === null;
|
|
377
|
+
}
|
|
378
|
+
function walkStorable(value, path, depth, ancestors, skip) {
|
|
379
|
+
if (value === null || value === void 0) return;
|
|
380
|
+
const kind = typeof value;
|
|
381
|
+
if (kind === "string") {
|
|
382
|
+
if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (kind === "boolean" || kind === "number") return;
|
|
386
|
+
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
387
|
+
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
388
|
+
}
|
|
389
|
+
const container = value;
|
|
390
|
+
if (container instanceof Date) {
|
|
391
|
+
if (Number.isNaN(container.getTime())) reject(path, "holds an invalid date, which the engine cannot store");
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
if (container instanceof Uint8Array) return;
|
|
395
|
+
const isArray = Array.isArray(container);
|
|
396
|
+
if (!isArray && !isPlainObject2(container)) {
|
|
397
|
+
reject(
|
|
398
|
+
path,
|
|
399
|
+
"holds a value that does not survive storage; use plain objects, arrays, strings, numbers, booleans, dates, or Uint8Array bytes"
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) {
|
|
403
|
+
reject(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
404
|
+
}
|
|
405
|
+
if (ancestors.has(container)) {
|
|
406
|
+
reject(path, "refers back to one of its own ancestors");
|
|
407
|
+
}
|
|
408
|
+
ancestors.add(container);
|
|
409
|
+
if (isArray) {
|
|
410
|
+
const items = container;
|
|
411
|
+
for (let i = 0; i < items.length; i++) {
|
|
412
|
+
walkStorable(items[i], `${path}[${i}]`, depth + 1, ancestors, skip);
|
|
413
|
+
}
|
|
414
|
+
} else {
|
|
415
|
+
const record = container;
|
|
416
|
+
for (const key of Object.keys(record)) {
|
|
417
|
+
const keyPath = path === "" ? key : `${path}.${key}`;
|
|
418
|
+
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
419
|
+
reject(keyPath, "uses a reserved object key");
|
|
420
|
+
}
|
|
421
|
+
if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
|
|
422
|
+
if (skip.has(keyPath)) continue;
|
|
423
|
+
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
ancestors.delete(container);
|
|
427
|
+
}
|
|
428
|
+
function assertStorableDocument(document, skipFieldPaths) {
|
|
429
|
+
walkStorable(document, "", 0, /* @__PURE__ */ new Set(), skipFieldPaths ?? /* @__PURE__ */ new Set());
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// src/schema/validator/document.ts
|
|
433
|
+
var vectorPathCache = /* @__PURE__ */ new WeakMap();
|
|
434
|
+
function vectorFieldPaths(schema) {
|
|
435
|
+
let paths = vectorPathCache.get(schema);
|
|
436
|
+
if (paths !== void 0) return paths;
|
|
437
|
+
paths = /* @__PURE__ */ new Set();
|
|
438
|
+
const collect = (level, prefix) => {
|
|
439
|
+
for (const [field, type] of Object.entries(level)) {
|
|
440
|
+
const path = prefix === "" ? field : `${prefix}.${field}`;
|
|
441
|
+
if (typeof type === "string" && VECTOR_PATTERN.test(type)) {
|
|
442
|
+
paths?.add(path);
|
|
443
|
+
} else if (isPlainObject(type)) {
|
|
444
|
+
collect(type, path);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
collect(schema, "");
|
|
449
|
+
vectorPathCache.set(schema, paths);
|
|
450
|
+
return paths;
|
|
451
|
+
}
|
|
452
|
+
var parsedLevelCache = /* @__PURE__ */ new WeakMap();
|
|
453
|
+
function parsedLevel(schema, prefix) {
|
|
454
|
+
let entries = parsedLevelCache.get(schema);
|
|
455
|
+
if (entries !== void 0) return entries;
|
|
456
|
+
entries = Object.entries(schema).map(([field, type]) => {
|
|
457
|
+
if (isPlainObject(type)) return { field, nested: type, type: null };
|
|
458
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
459
|
+
return { field, nested: null, type: parseFieldType(type, path) };
|
|
460
|
+
});
|
|
461
|
+
parsedLevelCache.set(schema, entries);
|
|
462
|
+
return entries;
|
|
463
|
+
}
|
|
464
|
+
function validateDocumentFields(doc, schema, prefix, patternValueLimit) {
|
|
465
|
+
for (const { field, nested, type } of parsedLevel(schema, prefix)) {
|
|
466
|
+
const value = doc[field];
|
|
467
|
+
if (value === void 0 || value === null) continue;
|
|
468
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
469
|
+
if (nested !== null) {
|
|
470
|
+
if (!isPlainObject(value)) {
|
|
471
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
472
|
+
field: path,
|
|
473
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
validateDocumentFields(value, nested, path, patternValueLimit);
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
if (type !== null) validateFieldValue(path, value, type, patternValueLimit);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
function validateDocument(document, schema, patternValueLimit = DEFAULT_PATTERN_VALUE_LIMIT) {
|
|
483
|
+
if (!isPlainObject(document)) {
|
|
484
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
485
|
+
received: typeof document
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
requireIdFieldKey(document);
|
|
489
|
+
assertStorableDocument(document, vectorFieldPaths(schema));
|
|
490
|
+
validateDocumentFields(document, schema, "", patternValueLimit);
|
|
491
|
+
}
|
|
492
|
+
function requireIdFieldKey(document) {
|
|
493
|
+
if (!Object.hasOwn(document, "id")) return;
|
|
494
|
+
const id = document.id;
|
|
495
|
+
if (typeof id === "string" && id.length > 0) return;
|
|
496
|
+
throw new NarsilError(
|
|
497
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
498
|
+
`Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
|
|
499
|
+
{ field: "id", received: id === null ? "null" : typeof id }
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
function collectExtraFields(doc, schema, prefix, extras) {
|
|
503
|
+
for (const key of Object.keys(doc)) {
|
|
504
|
+
if (key === "id") continue;
|
|
505
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
506
|
+
if (!Object.hasOwn(schema, key)) {
|
|
507
|
+
extras.push(path);
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
const schemaType = schema[key];
|
|
511
|
+
if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
|
|
512
|
+
collectExtraFields(doc[key], schemaType, path, extras);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
function validateDocumentStrict(document, schema) {
|
|
517
|
+
const extras = [];
|
|
518
|
+
collectExtraFields(document, schema, "", extras);
|
|
519
|
+
if (extras.length > 0) {
|
|
520
|
+
throw new NarsilError(
|
|
521
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
522
|
+
`Document contains fields not defined in schema: ${extras.join(", ")}`,
|
|
523
|
+
{ extraFields: extras }
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
function resolveNestedValue(document, path) {
|
|
528
|
+
if (!path.includes(".")) return document[path];
|
|
529
|
+
const segments = path.split(".");
|
|
530
|
+
let current = document;
|
|
531
|
+
for (const segment of segments) {
|
|
532
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
533
|
+
current = current[segment];
|
|
534
|
+
}
|
|
535
|
+
return current;
|
|
536
|
+
}
|
|
537
|
+
function validateRequiredFields(document, required) {
|
|
538
|
+
if (required.length === 0) return;
|
|
539
|
+
for (const field of required) {
|
|
540
|
+
const value = resolveNestedValue(document, field);
|
|
541
|
+
if (value === void 0 || value === null) {
|
|
542
|
+
throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
|
|
543
|
+
field
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
var INTEGER_ENCODING = { maxDepth: MAX_DOCUMENT_ENCODING_DEPTH };
|
|
549
|
+
var FLOAT_ENCODING = { maxDepth: MAX_DOCUMENT_ENCODING_DEPTH, forceIntegerToFloat: true };
|
|
550
|
+
var integerEncoder = new Encoder(INTEGER_ENCODING);
|
|
551
|
+
var floatEncoder = new Encoder(FLOAT_ENCODING);
|
|
552
|
+
function encodeExactly(document, forceIntegerToFloat) {
|
|
553
|
+
const bytes = (forceIntegerToFloat ? floatEncoder : integerEncoder).encode(document);
|
|
554
|
+
if (bytes.byteLength > DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES) {
|
|
555
|
+
if (forceIntegerToFloat) floatEncoder = new Encoder(FLOAT_ENCODING);
|
|
556
|
+
else integerEncoder = new Encoder(INTEGER_ENCODING);
|
|
557
|
+
}
|
|
558
|
+
return bytes;
|
|
559
|
+
}
|
|
560
|
+
function holdsNegativeZero(value, depth) {
|
|
561
|
+
if (typeof value === "number") return Object.is(value, -0);
|
|
562
|
+
if (value === null || typeof value !== "object" || ArrayBuffer.isView(value) || depth > MAX_DOCUMENT_ENCODING_DEPTH) {
|
|
563
|
+
return false;
|
|
564
|
+
}
|
|
565
|
+
for (const child of Array.isArray(value) ? value : Object.values(value)) {
|
|
566
|
+
if (holdsNegativeZero(child, depth + 1)) return true;
|
|
567
|
+
}
|
|
568
|
+
return false;
|
|
569
|
+
}
|
|
570
|
+
function encodeDocumentReply(payload) {
|
|
571
|
+
return encode(payload, { forceIntegerToFloat: holdsNegativeZero(payload, 1 - DOCUMENT_REPLY_ENVELOPE_DEPTH) });
|
|
572
|
+
}
|
|
573
|
+
function encodeDocument(document) {
|
|
574
|
+
try {
|
|
575
|
+
return encodeExactly(document, holdsNegativeZero(document, 1));
|
|
576
|
+
} catch (error) {
|
|
577
|
+
throw new NarsilError(
|
|
578
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
579
|
+
`The document cannot be encoded as MessagePack: ${error instanceof Error ? error.message : String(error)}`
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// src/schema/validator/document-size.ts
|
|
585
|
+
var LARGEST_HEADER_BYTES = 5;
|
|
586
|
+
var LARGEST_NUMBER_BYTES = 9;
|
|
587
|
+
var LARGEST_TIMESTAMP_BYTES = 15;
|
|
588
|
+
var LARGEST_UTF8_BYTES_PER_UNIT = 3;
|
|
589
|
+
function upperBoundBytes(value, budget, depth) {
|
|
590
|
+
if (value === null || value === void 0 || typeof value === "boolean") return 1;
|
|
591
|
+
if (typeof value === "number") return LARGEST_NUMBER_BYTES;
|
|
592
|
+
if (typeof value === "string") return LARGEST_HEADER_BYTES + value.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
593
|
+
if (value instanceof Date) return LARGEST_TIMESTAMP_BYTES;
|
|
594
|
+
if (ArrayBuffer.isView(value)) return LARGEST_HEADER_BYTES + value.byteLength;
|
|
595
|
+
if (typeof value !== "object" || depth > MAX_DOCUMENT_NESTING_DEPTH) return LARGEST_NUMBER_BYTES;
|
|
596
|
+
let total = LARGEST_HEADER_BYTES;
|
|
597
|
+
if (Array.isArray(value)) {
|
|
598
|
+
for (const item of value) {
|
|
599
|
+
total += upperBoundBytes(item, budget - total, depth + 1);
|
|
600
|
+
if (total > budget) return total;
|
|
601
|
+
}
|
|
602
|
+
return total;
|
|
603
|
+
}
|
|
604
|
+
const record = value;
|
|
605
|
+
for (const key of Object.keys(record)) {
|
|
606
|
+
total += LARGEST_HEADER_BYTES + key.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
607
|
+
total += upperBoundBytes(record[key], budget - total, depth + 1);
|
|
608
|
+
if (total > budget) return total;
|
|
609
|
+
}
|
|
610
|
+
return total;
|
|
611
|
+
}
|
|
612
|
+
function messagePackUpperBoundBytes(value) {
|
|
613
|
+
return upperBoundBytes(value, Number.POSITIVE_INFINITY, 0);
|
|
614
|
+
}
|
|
615
|
+
function encodedBytesOf(document) {
|
|
616
|
+
try {
|
|
617
|
+
return encodeDocument(document).byteLength;
|
|
618
|
+
} catch {
|
|
619
|
+
return null;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
function requireDocumentWithinSizeLimit(document) {
|
|
623
|
+
if (upperBoundBytes(document, MAX_DOCUMENT_BYTES, 0) <= MAX_DOCUMENT_BYTES) return;
|
|
624
|
+
const encodedBytes = encodedBytesOf(document);
|
|
625
|
+
if (encodedBytes === null || encodedBytes <= MAX_DOCUMENT_BYTES) return;
|
|
626
|
+
throw new NarsilError(
|
|
627
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
628
|
+
`The document takes ${encodedBytes} bytes as MessagePack, vector values included, but a document may take at most ${MAX_DOCUMENT_BYTES}`,
|
|
629
|
+
{ bytes: encodedBytes, maximum: MAX_DOCUMENT_BYTES }
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// src/schema/validator/vector-promotion.ts
|
|
634
|
+
var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
|
|
635
|
+
var STORAGE_MODES = /* @__PURE__ */ new Set(["memory", "disk"]);
|
|
636
|
+
function fail(message, details) {
|
|
637
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
638
|
+
}
|
|
639
|
+
function isPositiveInteger(value) {
|
|
640
|
+
return Number.isInteger(value) && value >= 1;
|
|
641
|
+
}
|
|
642
|
+
function isQuantizationMode(value) {
|
|
643
|
+
return typeof value === "string" && QUANTIZATION_MODES.has(value);
|
|
644
|
+
}
|
|
645
|
+
function isStorageMode(value) {
|
|
646
|
+
return typeof value === "string" && STORAGE_MODES.has(value);
|
|
647
|
+
}
|
|
648
|
+
function validateVectorStorage(config, filesystemDurability) {
|
|
649
|
+
if (config?.storage === "disk" && !filesystemDurability) {
|
|
650
|
+
fail("vectorPromotion.storage 'disk' needs filesystem durability, because a checkpoint file holds the vectors", {
|
|
651
|
+
storage: config.storage
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
function validateTextStorage(textStorage, filesystemDurability) {
|
|
656
|
+
if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
|
|
657
|
+
fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
|
|
658
|
+
}
|
|
659
|
+
function raiseEarlierReleaseGraphSetting(config) {
|
|
660
|
+
if (typeof config !== "object" || config === null) return config;
|
|
661
|
+
const hnsw = config.hnswConfig;
|
|
662
|
+
if (typeof hnsw !== "object" || hnsw === null) return config;
|
|
663
|
+
if (hnsw.m !== EARLIER_RELEASE_MIN_M) return config;
|
|
664
|
+
return { ...config, hnswConfig: { ...hnsw, m: MIN_M } };
|
|
665
|
+
}
|
|
666
|
+
function validateVectorPromotion(config) {
|
|
667
|
+
if (config === void 0) return;
|
|
668
|
+
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
669
|
+
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
670
|
+
}
|
|
671
|
+
const quantization = config.quantization;
|
|
672
|
+
if (quantization !== void 0 && !isQuantizationMode(quantization)) {
|
|
673
|
+
fail("vectorPromotion.quantization must be 'osq8', 'osq4', 'osq2', 'osq1', or 'none'", { quantization });
|
|
674
|
+
}
|
|
675
|
+
const storage = config.storage;
|
|
676
|
+
if (storage !== void 0 && !isStorageMode(storage)) {
|
|
677
|
+
fail("vectorPromotion.storage must be 'memory' or 'disk'", { storage });
|
|
678
|
+
}
|
|
679
|
+
const hnsw = config.hnswConfig;
|
|
680
|
+
if (hnsw === void 0) return;
|
|
681
|
+
if (hnsw.m !== void 0 && (!Number.isInteger(hnsw.m) || hnsw.m < MIN_M || hnsw.m > MAX_M)) {
|
|
682
|
+
fail(`vectorPromotion.hnswConfig.m must be a whole number from ${MIN_M} to ${MAX_M}`, { m: hnsw.m });
|
|
683
|
+
}
|
|
684
|
+
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
685
|
+
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
686
|
+
efConstruction: hnsw.efConstruction
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
const metric = hnsw.metric;
|
|
690
|
+
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
691
|
+
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
// src/core/partition/utils.ts
|
|
696
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
697
|
+
const existing = table.indexMap.get(fieldName);
|
|
698
|
+
if (existing !== void 0) return existing;
|
|
699
|
+
const idx = table.names.length;
|
|
700
|
+
table.names.push(fieldName);
|
|
701
|
+
table.indexMap.set(fieldName, idx);
|
|
702
|
+
return idx;
|
|
703
|
+
}
|
|
704
|
+
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
705
|
+
function getNestedValue(obj, path) {
|
|
706
|
+
const parts = path.split(".");
|
|
707
|
+
let current = obj;
|
|
708
|
+
for (let i = 0; i < parts.length; i++) {
|
|
709
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
710
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
711
|
+
current = current[parts[i]];
|
|
712
|
+
}
|
|
713
|
+
return current;
|
|
714
|
+
}
|
|
715
|
+
function stringArraysEqual(a, b) {
|
|
716
|
+
if (a === b) return true;
|
|
717
|
+
if (!a || !b) return false;
|
|
718
|
+
if (a.length !== b.length) return false;
|
|
719
|
+
for (let i = 0; i < a.length; i++) {
|
|
720
|
+
if (a[i] !== b[i]) return false;
|
|
721
|
+
}
|
|
722
|
+
return true;
|
|
723
|
+
}
|
|
724
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
725
|
+
for (const { path, base } of fields) {
|
|
726
|
+
if (!isWordIndexedBase(base)) continue;
|
|
727
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
728
|
+
const newVal = getNestedValue(newDoc, path);
|
|
729
|
+
if (base === "string") {
|
|
730
|
+
if (oldVal !== newVal) return true;
|
|
731
|
+
} else {
|
|
732
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return false;
|
|
736
|
+
}
|
|
737
|
+
function parseVectorDimension(fieldType) {
|
|
738
|
+
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
739
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
740
|
+
}
|
|
741
|
+
function tokenizeOptions(options) {
|
|
742
|
+
return {
|
|
743
|
+
stem: true,
|
|
744
|
+
removeStopWords: true,
|
|
745
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
746
|
+
stopWordOverride: options?.stopWordOverride,
|
|
747
|
+
customTokenizer: options?.customTokenizer
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
function getFlatSchema(state, schema) {
|
|
751
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
752
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
753
|
+
state.lastSchemaRef = schema;
|
|
754
|
+
return state.flatSchemaCache;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
// src/core/partition/suggestions.ts
|
|
758
|
+
function verbatimOccurrences(state, token) {
|
|
759
|
+
const postingList = state.invertedIdx.lookup(token);
|
|
760
|
+
if (!postingList) return 0;
|
|
761
|
+
const derived = postingList.totalTermFrequency - state.surfaceRegistry.stemChangedTotalFor(token);
|
|
762
|
+
return derived > 0 ? derived : 0;
|
|
763
|
+
}
|
|
764
|
+
function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
765
|
+
if (limit <= 0 || surfacePrefix.length === 0) return [];
|
|
766
|
+
const groups = /* @__PURE__ */ new Map();
|
|
767
|
+
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
768
|
+
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
769
|
+
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
770
|
+
let group = groups.get(candidate.token);
|
|
771
|
+
if (!group) {
|
|
772
|
+
group = { documentFrequency: postingList.docIdSet.size, surfaces: [] };
|
|
773
|
+
groups.set(candidate.token, group);
|
|
774
|
+
}
|
|
775
|
+
group.surfaces.push({ surface: candidate.surface, occurrences: candidate.occurrences });
|
|
776
|
+
}
|
|
777
|
+
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedPrefix !== surfacePrefix && stemmedPrefix.length > 0 ? [surfacePrefix, stemmedPrefix] : [surfacePrefix];
|
|
778
|
+
for (const prefix of termPrefixes) {
|
|
779
|
+
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
780
|
+
const occurrences = verbatimOccurrences(state, suggestion.term);
|
|
781
|
+
if (occurrences === 0) continue;
|
|
782
|
+
let group = groups.get(suggestion.term);
|
|
783
|
+
if (!group) {
|
|
784
|
+
group = { documentFrequency: suggestion.documentFrequency, surfaces: [] };
|
|
785
|
+
groups.set(suggestion.term, group);
|
|
786
|
+
} else if (group.surfaces.some((s) => s.surface === suggestion.term)) {
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
group.surfaces.push({ surface: suggestion.term, occurrences });
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
const results = [];
|
|
793
|
+
for (const [token, group] of groups) {
|
|
794
|
+
results.push({ token, documentFrequency: group.documentFrequency, surfaces: group.surfaces });
|
|
795
|
+
}
|
|
796
|
+
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
797
|
+
if (results.length > limit) results.length = limit;
|
|
798
|
+
return results;
|
|
799
|
+
}
|
|
800
|
+
function partitionDocumentFrequency(subs, token) {
|
|
801
|
+
let documents = 0;
|
|
802
|
+
for (const sub of subs) documents += sub.invertedIdx.documentFrequency(token);
|
|
803
|
+
return documents;
|
|
804
|
+
}
|
|
805
|
+
function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
806
|
+
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
807
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
808
|
+
const holdsSurfaceForms = subs.some((sub) => sub.surfaceRegistry.size() > 0);
|
|
809
|
+
const termPrefixes = !holdsSurfaceForms && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
810
|
+
for (const sub of subs) {
|
|
811
|
+
for (const candidate of sub.surfaceRegistry.candidatesForPrefix(surfacePrefix)) candidates.add(candidate.token);
|
|
812
|
+
for (const prefix of termPrefixes) {
|
|
813
|
+
for (const suggestion of sub.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
814
|
+
if (candidates.has(suggestion.term)) continue;
|
|
815
|
+
if (verbatimOccurrences(sub, suggestion.term) > 0) candidates.add(suggestion.term);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
const ranked = [];
|
|
820
|
+
for (const token of candidates) {
|
|
821
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
822
|
+
if (documents > 0) ranked.push({ token, documents });
|
|
823
|
+
}
|
|
824
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
825
|
+
if (ranked.length > maxExpansions) ranked.length = maxExpansions;
|
|
826
|
+
return ranked.map((entry) => entry.token);
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// src/core/partition/partial-words/text.ts
|
|
830
|
+
var encoder = new TextEncoder();
|
|
831
|
+
var LAST_BASIC_PLANE_CODE_POINT = 65535;
|
|
832
|
+
var BASIC_PLANE_SIZE = 65536;
|
|
833
|
+
var UTF8_CONTINUATION_MASK = 192;
|
|
834
|
+
var UTF8_CONTINUATION_BITS = 128;
|
|
835
|
+
function codePointsOf(text) {
|
|
836
|
+
const points = [];
|
|
837
|
+
let index = 0;
|
|
838
|
+
while (index < text.length) {
|
|
839
|
+
const point = text.codePointAt(index) ?? text.charCodeAt(index);
|
|
840
|
+
points.push(point);
|
|
841
|
+
index += point > LAST_BASIC_PLANE_CODE_POINT ? 2 : 1;
|
|
842
|
+
}
|
|
843
|
+
return points;
|
|
844
|
+
}
|
|
845
|
+
function runKeyAt(points, at) {
|
|
846
|
+
return String.fromCodePoint(points[at], points[at + 1], points[at + 2]);
|
|
847
|
+
}
|
|
848
|
+
function packedRunKeyAt(points, at) {
|
|
849
|
+
const first = points[at];
|
|
850
|
+
const second = points[at + 1];
|
|
851
|
+
const third = points[at + 2];
|
|
852
|
+
if ((first | second | third) > LAST_BASIC_PLANE_CODE_POINT) return String.fromCodePoint(first, second, third);
|
|
853
|
+
return first * BASIC_PLANE_SIZE * BASIC_PLANE_SIZE + second * BASIC_PLANE_SIZE + third;
|
|
854
|
+
}
|
|
855
|
+
function partialWordProbe(text) {
|
|
856
|
+
return { text, codePoints: codePointsOf(text), utf8: encoder.encode(text) };
|
|
857
|
+
}
|
|
858
|
+
function utf8Contains(haystack, start, end, needle) {
|
|
859
|
+
const last = end - needle.length;
|
|
860
|
+
for (let from = start; from <= last; from++) {
|
|
861
|
+
let at = 0;
|
|
862
|
+
while (at < needle.length && haystack[from + at] === needle[at]) at++;
|
|
863
|
+
if (at === needle.length) return true;
|
|
864
|
+
}
|
|
865
|
+
return false;
|
|
866
|
+
}
|
|
867
|
+
function utf8CodePointCount(bytes, start, end) {
|
|
868
|
+
let count = 0;
|
|
869
|
+
for (let at = start; at < end; at++) {
|
|
870
|
+
if ((bytes[at] & UTF8_CONTINUATION_MASK) !== UTF8_CONTINUATION_BITS) count++;
|
|
871
|
+
}
|
|
872
|
+
return count;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// src/core/partition/partial-words/expansion.ts
|
|
876
|
+
function excludedFromRequest(request, token) {
|
|
877
|
+
if (token === request.ownToken) return true;
|
|
878
|
+
return request.tolerance > 0 && fuzzyTermMatches(request.ownToken, token, request.tolerance, request.prefixLength);
|
|
879
|
+
}
|
|
880
|
+
function addedWordsFor(subs, readers, request) {
|
|
881
|
+
if (request.share <= 0) return [];
|
|
882
|
+
const probe = partialWordProbe(request.written);
|
|
883
|
+
const shortestSpelling = /* @__PURE__ */ new Map();
|
|
884
|
+
for (const reader of readers) {
|
|
885
|
+
reader.forEachContaining(probe, (token, spellingLength) => {
|
|
886
|
+
const known = shortestSpelling.get(token);
|
|
887
|
+
if (known === void 0 || spellingLength < known) shortestSpelling.set(token, spellingLength);
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
const ranked = [];
|
|
891
|
+
for (const [token, spellingLength] of shortestSpelling) {
|
|
892
|
+
if (excludedFromRequest(request, token)) continue;
|
|
893
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
894
|
+
if (documents > 0) ranked.push({ token, documents, spellingLength });
|
|
895
|
+
}
|
|
896
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
897
|
+
if (ranked.length > request.share) ranked.length = request.share;
|
|
898
|
+
const writtenLength = probe.codePoints.length;
|
|
899
|
+
return ranked.map((word) => ({ token: word.token, factor: writtenLength / word.spellingLength }));
|
|
900
|
+
}
|
|
901
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
902
|
+
const readers = [];
|
|
903
|
+
for (const sub of subs) {
|
|
904
|
+
const reader = sub.partialWords.readerFor(analysis);
|
|
905
|
+
if (reader !== null) readers.push(reader);
|
|
906
|
+
}
|
|
907
|
+
return requests.map((request) => readers.length === 0 ? [] : addedWordsFor(subs, readers, request));
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// src/core/partition/partial-words/constants.ts
|
|
911
|
+
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
912
|
+
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
913
|
+
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
914
|
+
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
915
|
+
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
916
|
+
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
917
|
+
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
918
|
+
|
|
919
|
+
// src/core/partition/partial-words/spellings.ts
|
|
920
|
+
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
921
|
+
function codePointCountOf(text) {
|
|
922
|
+
let count = 0;
|
|
923
|
+
for (let index = 0; index < text.length; count++) {
|
|
924
|
+
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
925
|
+
}
|
|
926
|
+
return count;
|
|
927
|
+
}
|
|
928
|
+
function holdsRun(spelling) {
|
|
929
|
+
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
930
|
+
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
931
|
+
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
932
|
+
}
|
|
933
|
+
function countSpelling(counts, token, spelling, change) {
|
|
934
|
+
if (!holdsRun(spelling)) return;
|
|
935
|
+
let byToken = counts.get(token);
|
|
936
|
+
if (byToken === void 0) {
|
|
937
|
+
if (change <= 0) return;
|
|
938
|
+
byToken = /* @__PURE__ */ new Map();
|
|
939
|
+
counts.set(token, byToken);
|
|
940
|
+
}
|
|
941
|
+
const next = (byToken.get(spelling) ?? 0) + change;
|
|
942
|
+
if (next > 0) {
|
|
943
|
+
byToken.set(spelling, next);
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
byToken.delete(spelling);
|
|
947
|
+
if (byToken.size === 0) counts.delete(token);
|
|
948
|
+
}
|
|
949
|
+
function textsOf(value) {
|
|
950
|
+
if (typeof value === "string") return [value];
|
|
951
|
+
if (!Array.isArray(value)) return [];
|
|
952
|
+
return value.filter((element) => typeof element === "string");
|
|
953
|
+
}
|
|
954
|
+
function visitFieldTexts(document, fields, visit) {
|
|
955
|
+
for (const field of fields) {
|
|
956
|
+
for (const text of textsOf(getNestedValue(document, field))) visit(text);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
function visitDocumentSpellings(document, analysis, visit) {
|
|
960
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
961
|
+
const visitOccurrence = (token, _position, surface) => {
|
|
962
|
+
visit(token, surface ?? token);
|
|
963
|
+
};
|
|
964
|
+
visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
|
|
965
|
+
}
|
|
966
|
+
function visitDocumentSurfaceSpellings(document, analysis, visit) {
|
|
967
|
+
if (analysis.options?.customTokenizer !== void 0) return;
|
|
968
|
+
visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
|
|
969
|
+
}
|
|
970
|
+
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
971
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => countSpelling(counts, token, spelling, direction));
|
|
972
|
+
}
|
|
973
|
+
function spellingCountsOf(documents, analysis) {
|
|
974
|
+
const counts = /* @__PURE__ */ new Map();
|
|
975
|
+
for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
|
|
976
|
+
return counts;
|
|
977
|
+
}
|
|
978
|
+
function encodePartialSpellings(fields, counts) {
|
|
979
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
980
|
+
for (const [token, byToken] of counts) {
|
|
981
|
+
for (const [spelling, count] of byToken) {
|
|
982
|
+
encoded.tokens.push(token);
|
|
983
|
+
encoded.spellings.push(spelling);
|
|
984
|
+
encoded.counts.push(count);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
return encoded;
|
|
988
|
+
}
|
|
989
|
+
function sameFields(left, right) {
|
|
990
|
+
if (left.length !== right.length) return false;
|
|
991
|
+
for (let at = 0; at < left.length; at++) {
|
|
992
|
+
if (left[at] !== right[at]) return false;
|
|
993
|
+
}
|
|
994
|
+
return true;
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// src/core/partition/partial-words/frozen-table.ts
|
|
998
|
+
var decoder = new TextDecoder();
|
|
999
|
+
function compareEntries(left, right) {
|
|
1000
|
+
return compareCodePoints(left.spelling, right.spelling) || left.tokenIndex - right.tokenIndex;
|
|
1001
|
+
}
|
|
1002
|
+
function orderedEntriesOf(source, tokenTable) {
|
|
1003
|
+
const entries = [];
|
|
1004
|
+
for (let at = 0; at < source.tokens.length; at++) {
|
|
1005
|
+
if (source.counts[at] <= 0) continue;
|
|
1006
|
+
const tokenIndex = tokenTable.find(source.tokens[at]);
|
|
1007
|
+
if (tokenIndex >= 0) entries.push({ spelling: source.spellings[at], tokenIndex, count: source.counts[at] });
|
|
1008
|
+
}
|
|
1009
|
+
entries.sort(compareEntries);
|
|
1010
|
+
const distinct = [];
|
|
1011
|
+
for (const entry of entries) {
|
|
1012
|
+
const last = distinct[distinct.length - 1];
|
|
1013
|
+
if (last !== void 0 && compareEntries(last, entry) === 0) last.count += entry.count;
|
|
1014
|
+
else distinct.push(entry);
|
|
1015
|
+
}
|
|
1016
|
+
return distinct;
|
|
1017
|
+
}
|
|
1018
|
+
function buildPartialWordTable(source, tokenTable) {
|
|
1019
|
+
const spellings = [];
|
|
1020
|
+
const spellingTokens = [];
|
|
1021
|
+
const spellingCounts = [];
|
|
1022
|
+
const runLists = /* @__PURE__ */ new Map();
|
|
1023
|
+
const addSpelling = (spelling, tokenIndex, count) => {
|
|
1024
|
+
const points = codePointsOf(spelling);
|
|
1025
|
+
if (points.length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
1026
|
+
const entry = spellings.length;
|
|
1027
|
+
spellings.push(spelling);
|
|
1028
|
+
spellingTokens.push(tokenIndex);
|
|
1029
|
+
spellingCounts.push(count);
|
|
1030
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
1031
|
+
const key = runKeyAt(points, at);
|
|
1032
|
+
const list = runLists.get(key);
|
|
1033
|
+
if (list === void 0) runLists.set(key, [entry]);
|
|
1034
|
+
else if (list[list.length - 1] !== entry) list.push(entry);
|
|
1035
|
+
}
|
|
1036
|
+
};
|
|
1037
|
+
for (const entry of orderedEntriesOf(source, tokenTable)) addSpelling(entry.spelling, entry.tokenIndex, entry.count);
|
|
1038
|
+
const runKeys = [...runLists.keys()].sort(compareCodePoints);
|
|
1039
|
+
const runs = new Uint32Array(runKeys.length * PARTIAL_WORD_RUN_CODE_POINTS);
|
|
1040
|
+
const runOffsets = new Uint32Array(runKeys.length + 1);
|
|
1041
|
+
let memberCount = 0;
|
|
1042
|
+
for (const key of runKeys) memberCount += runLists.get(key)?.length ?? 0;
|
|
1043
|
+
const runEntries = new Uint32Array(memberCount);
|
|
1044
|
+
let cursor = 0;
|
|
1045
|
+
for (let run = 0; run < runKeys.length; run++) {
|
|
1046
|
+
const points = codePointsOf(runKeys[run]);
|
|
1047
|
+
runs.set(points, run * PARTIAL_WORD_RUN_CODE_POINTS);
|
|
1048
|
+
for (const entry of runLists.get(runKeys[run]) ?? []) runEntries[cursor++] = entry;
|
|
1049
|
+
runOffsets[run + 1] = cursor;
|
|
1050
|
+
}
|
|
1051
|
+
const spellingBlob = encodeStringBlob(spellings);
|
|
1052
|
+
return {
|
|
1053
|
+
fields: [...source.fields],
|
|
1054
|
+
spellings: spellingBlob.blob,
|
|
1055
|
+
spellingOffsets: spellingBlob.offsets,
|
|
1056
|
+
spellingTokens: Uint32Array.from(spellingTokens),
|
|
1057
|
+
spellingCounts: Uint32Array.from(spellingCounts),
|
|
1058
|
+
runs,
|
|
1059
|
+
runOffsets,
|
|
1060
|
+
runEntries
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
function countTableSpellings(counts, table, tokenTable) {
|
|
1064
|
+
const { spellings, spellingOffsets, spellingTokens, spellingCounts } = table;
|
|
1065
|
+
for (let entry = 0; entry < spellingTokens.length; entry++) {
|
|
1066
|
+
const spelling = decoder.decode(spellings.subarray(spellingOffsets[entry], spellingOffsets[entry + 1]));
|
|
1067
|
+
countSpelling(counts, tokenTable.tokenAt(spellingTokens[entry]), spelling, spellingCounts[entry]);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
function findRun(runs, first, second, third) {
|
|
1071
|
+
let low = 0;
|
|
1072
|
+
let high = runs.length / PARTIAL_WORD_RUN_CODE_POINTS;
|
|
1073
|
+
while (low < high) {
|
|
1074
|
+
const middle = low + high >>> 1;
|
|
1075
|
+
const at = middle * PARTIAL_WORD_RUN_CODE_POINTS;
|
|
1076
|
+
const difference = runs[at] - first || runs[at + 1] - second || runs[at + 2] - third;
|
|
1077
|
+
if (difference === 0) return middle;
|
|
1078
|
+
if (difference < 0) low = middle + 1;
|
|
1079
|
+
else high = middle;
|
|
1080
|
+
}
|
|
1081
|
+
return -1;
|
|
1082
|
+
}
|
|
1083
|
+
function partialWordTableBytes(table) {
|
|
1084
|
+
return table.spellings.byteLength + table.spellingOffsets.byteLength + table.spellingTokens.byteLength + table.spellingCounts.byteLength + table.runs.byteLength + table.runOffsets.byteLength + table.runEntries.byteLength;
|
|
1085
|
+
}
|
|
1086
|
+
function readPartialWordTable(table, tokenTable, removedCounts) {
|
|
1087
|
+
const { spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries } = table;
|
|
1088
|
+
function narrowestRun(points) {
|
|
1089
|
+
let narrowest = -1;
|
|
1090
|
+
let narrowestLength = Number.POSITIVE_INFINITY;
|
|
1091
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
1092
|
+
const run = findRun(runs, points[at], points[at + 1], points[at + 2]);
|
|
1093
|
+
if (run < 0) return -1;
|
|
1094
|
+
const length = runOffsets[run + 1] - runOffsets[run];
|
|
1095
|
+
if (length < narrowestLength) {
|
|
1096
|
+
narrowest = run;
|
|
1097
|
+
narrowestLength = length;
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
return narrowest;
|
|
1101
|
+
}
|
|
1102
|
+
return {
|
|
1103
|
+
forEachContaining(probe, visit) {
|
|
1104
|
+
if (probe.codePoints.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
1105
|
+
const run = narrowestRun(probe.codePoints);
|
|
1106
|
+
if (run < 0) return;
|
|
1107
|
+
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
1108
|
+
const entry = runEntries[member];
|
|
1109
|
+
const start = spellingOffsets[entry];
|
|
1110
|
+
const end = spellingOffsets[entry + 1];
|
|
1111
|
+
if (end - start <= probe.utf8.length) continue;
|
|
1112
|
+
if ((removedCounts.get(entry) ?? 0) >= spellingCounts[entry]) continue;
|
|
1113
|
+
if (!utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
1114
|
+
const length = utf8CodePointCount(spellings, start, end);
|
|
1115
|
+
if (length <= probe.codePoints.length) continue;
|
|
1116
|
+
visit(tokenTable.tokenAt(spellingTokens[entry]), length);
|
|
1117
|
+
}
|
|
1118
|
+
},
|
|
1119
|
+
entryOf(spelling) {
|
|
1120
|
+
const probe = partialWordProbe(spelling);
|
|
1121
|
+
if (probe.codePoints.length <= PARTIAL_WORD_RUN_CODE_POINTS) return -1;
|
|
1122
|
+
const run = narrowestRun(probe.codePoints);
|
|
1123
|
+
if (run < 0) return -1;
|
|
1124
|
+
let exhausted = -1;
|
|
1125
|
+
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
1126
|
+
const entry = runEntries[member];
|
|
1127
|
+
const start = spellingOffsets[entry];
|
|
1128
|
+
const end = spellingOffsets[entry + 1];
|
|
1129
|
+
if (end - start !== probe.utf8.length || !utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
1130
|
+
if ((removedCounts.get(entry) ?? 0) < spellingCounts[entry]) return entry;
|
|
1131
|
+
if (exhausted < 0) exhausted = entry;
|
|
1132
|
+
}
|
|
1133
|
+
return exhausted;
|
|
1134
|
+
},
|
|
1135
|
+
bytes() {
|
|
1136
|
+
return partialWordTableBytes(table);
|
|
1137
|
+
}
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
// src/core/partition/partial-words/frozen-slot.ts
|
|
1142
|
+
function tombstonedOrdinalsOf(documents) {
|
|
1143
|
+
let marks = null;
|
|
1144
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
1145
|
+
if (!documents.isTombstoned(ordinal)) continue;
|
|
1146
|
+
marks ??= new Uint8Array(documents.count);
|
|
1147
|
+
marks[ordinal] = 1;
|
|
1148
|
+
}
|
|
1149
|
+
return marks;
|
|
1150
|
+
}
|
|
1151
|
+
function addCounts(target, source) {
|
|
1152
|
+
for (const [token, bySpelling] of source) {
|
|
1153
|
+
for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
function countRemovedSpellings(removed, document, analysis) {
|
|
1157
|
+
const count = (spelling) => {
|
|
1158
|
+
removed.set(spelling, (removed.get(spelling) ?? 0) + 1);
|
|
1159
|
+
};
|
|
1160
|
+
if (analysis.options?.customTokenizer !== void 0) {
|
|
1161
|
+
visitDocumentSpellings(document, analysis, (_token, spelling) => count(spelling));
|
|
1162
|
+
return;
|
|
1163
|
+
}
|
|
1164
|
+
visitDocumentSurfaceSpellings(document, analysis, count);
|
|
1165
|
+
}
|
|
1166
|
+
function unstemmedSpellingsOf(analysis) {
|
|
1167
|
+
const { language } = analysis;
|
|
1168
|
+
if (language.normalizer === void 0 || language.tokenizer?.normalizeDiacritics === true) return () => null;
|
|
1169
|
+
const stopWords = resolveStopWords(language, analysis.options?.stopWordOverride);
|
|
1170
|
+
return (spelling) => {
|
|
1171
|
+
if (stopWords.has(spelling)) return null;
|
|
1172
|
+
const normalized = transformToken(spelling, language, false, false);
|
|
1173
|
+
return normalized === spelling ? null : normalized;
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
function subtractRemovedSpellings(counts, removed, analysis) {
|
|
1177
|
+
const tokenOf = /* @__PURE__ */ new Map();
|
|
1178
|
+
for (const [token, bySpelling] of counts) {
|
|
1179
|
+
for (const spelling of bySpelling.keys()) tokenOf.set(spelling, token);
|
|
1180
|
+
}
|
|
1181
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
1182
|
+
for (const [spelling, count] of removed) {
|
|
1183
|
+
const stored = tokenOf.has(spelling) ? spelling : unstemmedSpelling(spelling) ?? spelling;
|
|
1184
|
+
const token = tokenOf.get(stored);
|
|
1185
|
+
if (token !== void 0) countSpelling(counts, token, stored, -count);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
1189
|
+
let table = initial;
|
|
1190
|
+
let uncounted = null;
|
|
1191
|
+
let removedCounts = /* @__PURE__ */ new Map();
|
|
1192
|
+
let reader = initial === null ? null : readPartialWordTable(initial, tokenTable, removedCounts);
|
|
1193
|
+
let pendingRemovals = [];
|
|
1194
|
+
function isCounted(ordinal) {
|
|
1195
|
+
return uncounted === null || uncounted[ordinal] === 0;
|
|
1196
|
+
}
|
|
1197
|
+
function subtractPendingRemovals(current, analysis) {
|
|
1198
|
+
if (pendingRemovals.length === 0) return;
|
|
1199
|
+
const removed = /* @__PURE__ */ new Map();
|
|
1200
|
+
for (const ordinal of pendingRemovals) countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
1201
|
+
pendingRemovals = [];
|
|
1202
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
1203
|
+
for (const [spelling, count] of removed) {
|
|
1204
|
+
let entry = current.entryOf(spelling);
|
|
1205
|
+
if (entry < 0) {
|
|
1206
|
+
const fallback = unstemmedSpelling(spelling);
|
|
1207
|
+
if (fallback !== null) entry = current.entryOf(fallback);
|
|
1208
|
+
}
|
|
1209
|
+
if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
function rebuild(analysis) {
|
|
1213
|
+
uncounted = tombstonedOrdinalsOf(documents);
|
|
1214
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1215
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
1216
|
+
if (isCounted(ordinal)) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
1217
|
+
}
|
|
1218
|
+
const rebuilt = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
|
|
1219
|
+
table = rebuilt;
|
|
1220
|
+
removedCounts = /* @__PURE__ */ new Map();
|
|
1221
|
+
pendingRemovals = [];
|
|
1222
|
+
const rebuiltReader = readPartialWordTable(rebuilt, tokenTable, removedCounts);
|
|
1223
|
+
reader = rebuiltReader;
|
|
1224
|
+
return rebuiltReader;
|
|
1225
|
+
}
|
|
1226
|
+
function correctionsAgainstTable(mergedOrdinals) {
|
|
1227
|
+
let corrections = 0;
|
|
1228
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1229
|
+
if (mergedOrdinals[ordinal] >= 0 !== isCounted(ordinal)) corrections += 1;
|
|
1230
|
+
}
|
|
1231
|
+
return corrections;
|
|
1232
|
+
}
|
|
1233
|
+
function keptCount(mergedOrdinals) {
|
|
1234
|
+
let kept = 0;
|
|
1235
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1236
|
+
if (mergedOrdinals[ordinal] >= 0) kept += 1;
|
|
1237
|
+
}
|
|
1238
|
+
return kept;
|
|
1239
|
+
}
|
|
1240
|
+
return {
|
|
1241
|
+
readerFor(analysis) {
|
|
1242
|
+
if (analysis.fields.length === 0) return null;
|
|
1243
|
+
if (table === null || reader === null || !sameFields(table.fields, analysis.fields)) return rebuild(analysis);
|
|
1244
|
+
subtractPendingRemovals(reader, analysis);
|
|
1245
|
+
return reader;
|
|
1246
|
+
},
|
|
1247
|
+
afterRemoval(ordinal) {
|
|
1248
|
+
if (table !== null) pendingRemovals.push(ordinal);
|
|
1249
|
+
},
|
|
1250
|
+
tableOfEveryDocument(analysis) {
|
|
1251
|
+
if (table === null || uncounted !== null || !sameFields(table.fields, analysis.fields)) return null;
|
|
1252
|
+
return table;
|
|
1253
|
+
},
|
|
1254
|
+
countKeptSpellings(counts, analysis, mergedOrdinals) {
|
|
1255
|
+
if (table === null || !sameFields(table.fields, analysis.fields) || correctionsAgainstTable(mergedOrdinals) > keptCount(mergedOrdinals)) {
|
|
1256
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1257
|
+
if (mergedOrdinals[ordinal] >= 0) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
1258
|
+
}
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
const corrected = /* @__PURE__ */ new Map();
|
|
1262
|
+
countTableSpellings(corrected, table, tokenTable);
|
|
1263
|
+
const removed = /* @__PURE__ */ new Map();
|
|
1264
|
+
const returned = [];
|
|
1265
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1266
|
+
const kept = mergedOrdinals[ordinal] >= 0;
|
|
1267
|
+
if (kept === isCounted(ordinal)) continue;
|
|
1268
|
+
if (kept) returned.push(ordinal);
|
|
1269
|
+
else countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
1270
|
+
}
|
|
1271
|
+
subtractRemovedSpellings(corrected, removed, analysis);
|
|
1272
|
+
for (const ordinal of returned) countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, 1);
|
|
1273
|
+
addCounts(counts, corrected);
|
|
1274
|
+
},
|
|
1275
|
+
spellings() {
|
|
1276
|
+
return null;
|
|
1277
|
+
},
|
|
1278
|
+
bytes() {
|
|
1279
|
+
if (table === null) return 0;
|
|
1280
|
+
return partialWordTableBytes(table) + removedCounts.size * FROZEN_PARTIAL_REMOVED_ENTRY_BYTES;
|
|
1281
|
+
}
|
|
1282
|
+
};
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
// src/core/partition/partial-words/growing-slot.ts
|
|
1286
|
+
function createGrowingPartialWordSlot(view) {
|
|
1287
|
+
let fields = [];
|
|
1288
|
+
let everyDocumentPending = false;
|
|
1289
|
+
const pending = /* @__PURE__ */ new Set();
|
|
1290
|
+
const entriesByToken = /* @__PURE__ */ new Map();
|
|
1291
|
+
const entrySpellings = [];
|
|
1292
|
+
const entryTokens = [];
|
|
1293
|
+
const entryLengths = [];
|
|
1294
|
+
const entryCounts = [];
|
|
1295
|
+
const entryInRuns = [];
|
|
1296
|
+
const freeEntries = [];
|
|
1297
|
+
const entriesOutsideRuns = [];
|
|
1298
|
+
const runs = /* @__PURE__ */ new Map();
|
|
1299
|
+
let liveEntries = 0;
|
|
1300
|
+
let runMembers = 0;
|
|
1301
|
+
function resetSpellings() {
|
|
1302
|
+
pending.clear();
|
|
1303
|
+
entriesByToken.clear();
|
|
1304
|
+
entrySpellings.length = 0;
|
|
1305
|
+
entryTokens.length = 0;
|
|
1306
|
+
entryLengths.length = 0;
|
|
1307
|
+
entryCounts.length = 0;
|
|
1308
|
+
entryInRuns.length = 0;
|
|
1309
|
+
freeEntries.length = 0;
|
|
1310
|
+
entriesOutsideRuns.length = 0;
|
|
1311
|
+
runs.clear();
|
|
1312
|
+
liveEntries = 0;
|
|
1313
|
+
runMembers = 0;
|
|
1314
|
+
}
|
|
1315
|
+
function addEntry(byToken, token, spelling, count) {
|
|
1316
|
+
const length = codePointCountOf(spelling);
|
|
1317
|
+
if (length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
1318
|
+
const entry = freeEntries.pop() ?? entrySpellings.length;
|
|
1319
|
+
entrySpellings[entry] = spelling;
|
|
1320
|
+
entryTokens[entry] = token;
|
|
1321
|
+
entryLengths[entry] = length;
|
|
1322
|
+
entryCounts[entry] = count;
|
|
1323
|
+
entryInRuns[entry] = false;
|
|
1324
|
+
entriesOutsideRuns.push(entry);
|
|
1325
|
+
if (entriesOutsideRuns.length > GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY * entrySpellings.length) {
|
|
1326
|
+
dropFreedAndRepeatedPendingEntries();
|
|
1327
|
+
}
|
|
1328
|
+
byToken.set(spelling, entry);
|
|
1329
|
+
liveEntries++;
|
|
1330
|
+
}
|
|
1331
|
+
function dropFreedAndRepeatedPendingEntries() {
|
|
1332
|
+
let kept = 0;
|
|
1333
|
+
for (const entry of entriesOutsideRuns) {
|
|
1334
|
+
if (entryLengths[entry] === 0 || entryInRuns[entry]) continue;
|
|
1335
|
+
entryInRuns[entry] = true;
|
|
1336
|
+
entriesOutsideRuns[kept++] = entry;
|
|
1337
|
+
}
|
|
1338
|
+
entriesOutsideRuns.length = kept;
|
|
1339
|
+
for (let at = 0; at < kept; at++) entryInRuns[entriesOutsideRuns[at]] = false;
|
|
1340
|
+
}
|
|
1341
|
+
function addToRuns(entry) {
|
|
1342
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
1343
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
1344
|
+
const key = packedRunKeyAt(points, at);
|
|
1345
|
+
let members = runs.get(key);
|
|
1346
|
+
if (members === void 0) {
|
|
1347
|
+
members = /* @__PURE__ */ new Set();
|
|
1348
|
+
runs.set(key, members);
|
|
1349
|
+
}
|
|
1350
|
+
const size = members.size;
|
|
1351
|
+
members.add(entry);
|
|
1352
|
+
if (members.size !== size) runMembers++;
|
|
1353
|
+
}
|
|
1354
|
+
entryInRuns[entry] = true;
|
|
1355
|
+
}
|
|
1356
|
+
function addEntriesToRuns() {
|
|
1357
|
+
for (const entry of entriesOutsideRuns) {
|
|
1358
|
+
if (entryLengths[entry] > 0 && !entryInRuns[entry]) addToRuns(entry);
|
|
1359
|
+
}
|
|
1360
|
+
entriesOutsideRuns.length = 0;
|
|
1361
|
+
}
|
|
1362
|
+
function removeFromRuns(entry) {
|
|
1363
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
1364
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
1365
|
+
const key = packedRunKeyAt(points, at);
|
|
1366
|
+
const members = runs.get(key);
|
|
1367
|
+
if (members === void 0 || !members.delete(entry)) continue;
|
|
1368
|
+
runMembers--;
|
|
1369
|
+
if (members.size === 0) runs.delete(key);
|
|
1370
|
+
}
|
|
1371
|
+
entryInRuns[entry] = false;
|
|
1372
|
+
}
|
|
1373
|
+
function removeEntry(byToken, token, spelling, entry) {
|
|
1374
|
+
if (entryInRuns[entry]) removeFromRuns(entry);
|
|
1375
|
+
byToken.delete(spelling);
|
|
1376
|
+
if (byToken.size === 0) entriesByToken.delete(token);
|
|
1377
|
+
entrySpellings[entry] = "";
|
|
1378
|
+
entryTokens[entry] = "";
|
|
1379
|
+
entryLengths[entry] = 0;
|
|
1380
|
+
entryCounts[entry] = 0;
|
|
1381
|
+
freeEntries.push(entry);
|
|
1382
|
+
liveEntries--;
|
|
1383
|
+
}
|
|
1384
|
+
function change(token, spelling, delta) {
|
|
1385
|
+
let byToken = entriesByToken.get(token);
|
|
1386
|
+
const entry = byToken?.get(spelling);
|
|
1387
|
+
if (byToken !== void 0 && entry !== void 0) {
|
|
1388
|
+
const after = entryCounts[entry] + delta;
|
|
1389
|
+
if (after > 0) entryCounts[entry] = after;
|
|
1390
|
+
else removeEntry(byToken, token, spelling, entry);
|
|
1391
|
+
return;
|
|
1392
|
+
}
|
|
1393
|
+
if (delta <= 0) return;
|
|
1394
|
+
if (byToken === void 0) {
|
|
1395
|
+
byToken = /* @__PURE__ */ new Map();
|
|
1396
|
+
entriesByToken.set(token, byToken);
|
|
1397
|
+
}
|
|
1398
|
+
addEntry(byToken, token, spelling, delta);
|
|
1399
|
+
if (byToken.size === 0) entriesByToken.delete(token);
|
|
1400
|
+
}
|
|
1401
|
+
function encodedSpellings() {
|
|
1402
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
1403
|
+
for (const [token, byToken] of entriesByToken) {
|
|
1404
|
+
for (const [spelling, entry] of byToken) {
|
|
1405
|
+
encoded.tokens.push(token);
|
|
1406
|
+
encoded.spellings.push(spelling);
|
|
1407
|
+
encoded.counts.push(entryCounts[entry]);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
return encoded;
|
|
1411
|
+
}
|
|
1412
|
+
function applyCounts(source, direction) {
|
|
1413
|
+
for (const [token, byToken] of source) {
|
|
1414
|
+
for (const [spelling, count] of byToken) change(token, spelling, count * direction);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
function adoptFields(next) {
|
|
1418
|
+
if (sameFields(fields, next)) return;
|
|
1419
|
+
fields = [...next];
|
|
1420
|
+
resetSpellings();
|
|
1421
|
+
everyDocumentPending = fields.length > 0;
|
|
1422
|
+
}
|
|
1423
|
+
function documentAt(internalId) {
|
|
1424
|
+
const docId = view.docStore.getExternalId(internalId);
|
|
1425
|
+
return docId === void 0 ? void 0 : view.docStore.get(docId)?.fields;
|
|
1426
|
+
}
|
|
1427
|
+
function resolvePending(analysis) {
|
|
1428
|
+
if (everyDocumentPending) {
|
|
1429
|
+
resetSpellings();
|
|
1430
|
+
const derived2 = /* @__PURE__ */ new Map();
|
|
1431
|
+
for (const [, stored] of view.docStore.all()) countDocumentSpellings(derived2, stored.fields, analysis, 1);
|
|
1432
|
+
applyCounts(derived2, 1);
|
|
1433
|
+
everyDocumentPending = false;
|
|
1434
|
+
return;
|
|
1435
|
+
}
|
|
1436
|
+
if (pending.size === 0) return;
|
|
1437
|
+
const derived = /* @__PURE__ */ new Map();
|
|
1438
|
+
for (const internalId of pending) {
|
|
1439
|
+
const document = documentAt(internalId);
|
|
1440
|
+
if (document !== void 0) countDocumentSpellings(derived, document, analysis, 1);
|
|
1441
|
+
}
|
|
1442
|
+
pending.clear();
|
|
1443
|
+
applyCounts(derived, 1);
|
|
1444
|
+
}
|
|
1445
|
+
const reader = {
|
|
1446
|
+
forEachContaining(probe, visit) {
|
|
1447
|
+
const points = probe.codePoints;
|
|
1448
|
+
if (points.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
1449
|
+
let narrowest = null;
|
|
1450
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
1451
|
+
const members = runs.get(packedRunKeyAt(points, at));
|
|
1452
|
+
if (members === void 0) return;
|
|
1453
|
+
if (narrowest === null || members.size < narrowest.size) narrowest = members;
|
|
1454
|
+
}
|
|
1455
|
+
if (narrowest === null) return;
|
|
1456
|
+
for (const entry of narrowest) {
|
|
1457
|
+
const length = entryLengths[entry];
|
|
1458
|
+
if (length <= points.length) continue;
|
|
1459
|
+
if (!entrySpellings[entry].includes(probe.text)) continue;
|
|
1460
|
+
visit(entryTokens[entry], length);
|
|
1461
|
+
}
|
|
1462
|
+
},
|
|
1463
|
+
bytes() {
|
|
1464
|
+
return slot.bytes();
|
|
1465
|
+
}
|
|
1466
|
+
};
|
|
1467
|
+
const slot = {
|
|
1468
|
+
readerFor(analysis) {
|
|
1469
|
+
if (analysis.fields.length === 0) return null;
|
|
1470
|
+
adoptFields(analysis.fields);
|
|
1471
|
+
resolvePending(analysis);
|
|
1472
|
+
addEntriesToRuns();
|
|
1473
|
+
return reader;
|
|
1474
|
+
},
|
|
1475
|
+
spellings() {
|
|
1476
|
+
if (fields.length === 0 || everyDocumentPending || pending.size > 0) return null;
|
|
1477
|
+
return encodedSpellings();
|
|
1478
|
+
},
|
|
1479
|
+
afterDocument(documentFields, internalId, spellings, direction) {
|
|
1480
|
+
if (documentFields.length === 0) return;
|
|
1481
|
+
if (!sameFields(fields, documentFields)) {
|
|
1482
|
+
const holdsOnlyThisDocument = direction === 1 && view.docStore.count() === 1;
|
|
1483
|
+
adoptFields(documentFields);
|
|
1484
|
+
if (holdsOnlyThisDocument) everyDocumentPending = false;
|
|
1485
|
+
}
|
|
1486
|
+
if (everyDocumentPending) return;
|
|
1487
|
+
if (direction === -1 && pending.delete(internalId)) return;
|
|
1488
|
+
if (spellings !== null) {
|
|
1489
|
+
applyCounts(spellings, direction);
|
|
1490
|
+
} else if (direction === 1) {
|
|
1491
|
+
pending.add(internalId);
|
|
1492
|
+
} else {
|
|
1493
|
+
everyDocumentPending = true;
|
|
1494
|
+
}
|
|
1495
|
+
},
|
|
1496
|
+
afterMerge(spellings, internalIds) {
|
|
1497
|
+
if (internalIds.length === 0) return;
|
|
1498
|
+
if (fields.length === 0 && spellings !== null && view.docStore.count() === internalIds.length) {
|
|
1499
|
+
adoptFields(spellings.fields);
|
|
1500
|
+
everyDocumentPending = false;
|
|
1501
|
+
}
|
|
1502
|
+
if (fields.length === 0 || everyDocumentPending) return;
|
|
1503
|
+
if (spellings === null || !sameFields(fields, spellings.fields)) {
|
|
1504
|
+
for (const internalId of internalIds) pending.add(internalId);
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
for (let at = 0; at < spellings.tokens.length; at++) {
|
|
1508
|
+
change(spellings.tokens[at], spellings.spellings[at], spellings.counts[at]);
|
|
1509
|
+
}
|
|
1510
|
+
},
|
|
1511
|
+
forgetSpellings() {
|
|
1512
|
+
resetSpellings();
|
|
1513
|
+
everyDocumentPending = fields.length > 0;
|
|
1514
|
+
},
|
|
1515
|
+
clear() {
|
|
1516
|
+
fields = [];
|
|
1517
|
+
everyDocumentPending = false;
|
|
1518
|
+
resetSpellings();
|
|
1519
|
+
},
|
|
1520
|
+
bytes() {
|
|
1521
|
+
return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES + entriesOutsideRuns.length * GROWING_PARTIAL_PENDING_ENTRY_BYTES;
|
|
1522
|
+
}
|
|
1523
|
+
};
|
|
1524
|
+
return slot;
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
// src/core/partition/partial-words/highlight.ts
|
|
1528
|
+
function containsPartialWord(spelling, words) {
|
|
1529
|
+
for (const word of words) {
|
|
1530
|
+
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
1531
|
+
}
|
|
1532
|
+
return false;
|
|
1533
|
+
}
|
|
1534
|
+
function partialWordMatcher(text, language, partial) {
|
|
1535
|
+
const indexed = tokenize(text, language, {
|
|
1536
|
+
stem: true,
|
|
1537
|
+
removeStopWords: true,
|
|
1538
|
+
collectSurfaces: true,
|
|
1539
|
+
customTokenizer: partial.customTokenizer,
|
|
1540
|
+
stopWordOverride: partial.stopWordOverride
|
|
1541
|
+
});
|
|
1542
|
+
const positions = /* @__PURE__ */ new Set();
|
|
1543
|
+
const spellings = /* @__PURE__ */ new Set();
|
|
1544
|
+
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
1545
|
+
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
1546
|
+
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
1547
|
+
positions.add(indexed.tokens[at].position);
|
|
1548
|
+
spellings.add(spelling);
|
|
1549
|
+
}
|
|
1550
|
+
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
1551
|
+
return (position) => positions.has(position);
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
// src/core/partition/partial-words/query.ts
|
|
1555
|
+
function addPartialWords(group, added) {
|
|
1556
|
+
const membersByToken = /* @__PURE__ */ new Map();
|
|
1557
|
+
for (const member of group.members) membersByToken.set(member.token, member);
|
|
1558
|
+
for (const word of added) {
|
|
1559
|
+
const member = membersByToken.get(word.token);
|
|
1560
|
+
if (member === void 0) {
|
|
1561
|
+
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
1562
|
+
} else if (word.factor > member.partialFactor) {
|
|
1563
|
+
member.partialFactor = word.factor;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
function partialWordRequests(query) {
|
|
1568
|
+
const eligible = [];
|
|
1569
|
+
for (const queryToken of query.queryTokens) {
|
|
1570
|
+
const word = query.written.get(queryToken.position);
|
|
1571
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
1572
|
+
eligible.push({ token: queryToken.token, written: word });
|
|
1573
|
+
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
1574
|
+
}
|
|
1575
|
+
const count = eligible.length;
|
|
1576
|
+
if (count === 0) return [];
|
|
1577
|
+
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
1578
|
+
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
1579
|
+
return eligible.map((entry, order) => ({
|
|
1580
|
+
token: entry.token,
|
|
1581
|
+
request: {
|
|
1582
|
+
written: entry.written,
|
|
1583
|
+
ownToken: entry.token,
|
|
1584
|
+
share: evenShare + (order < remainder ? 1 : 0),
|
|
1585
|
+
tolerance: entry.token === query.prefixToken ? 0 : query.tolerance,
|
|
1586
|
+
prefixLength: query.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
1587
|
+
}
|
|
1588
|
+
}));
|
|
1589
|
+
}
|
|
1590
|
+
function addPartialWordMembers(query, expand, groupOf) {
|
|
1591
|
+
const requests = partialWordRequests(query);
|
|
1592
|
+
const added = expand(
|
|
1593
|
+
query.analysis,
|
|
1594
|
+
requests.map((entry) => entry.request)
|
|
1595
|
+
);
|
|
1596
|
+
for (let at = 0; at < requests.length; at++) {
|
|
1597
|
+
if (added[at].length === 0) continue;
|
|
1598
|
+
const token = requests[at].token;
|
|
1599
|
+
addPartialWords(groupOf(token, token !== query.prefixToken), added[at]);
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
function partialWordsToHighlight(queryTokens, writtenWords) {
|
|
1603
|
+
const writtenAt = /* @__PURE__ */ new Map();
|
|
1604
|
+
for (const word of writtenWords) {
|
|
1605
|
+
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
1606
|
+
}
|
|
1607
|
+
const seenTokens = /* @__PURE__ */ new Set();
|
|
1608
|
+
const words = [];
|
|
1609
|
+
for (const queryToken of queryTokens) {
|
|
1610
|
+
if (seenTokens.has(queryToken.token)) continue;
|
|
1611
|
+
seenTokens.add(queryToken.token);
|
|
1612
|
+
const word = writtenAt.get(queryToken.position);
|
|
1613
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
1614
|
+
words.push(word);
|
|
1615
|
+
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
1616
|
+
}
|
|
1617
|
+
return words;
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
// src/core/partition/partial-words/module.ts
|
|
1621
|
+
var partialWordSearchModule = {
|
|
1622
|
+
name: "partialWords",
|
|
1623
|
+
createGrowingSlot: createGrowingPartialWordSlot,
|
|
1624
|
+
createFrozenSlot: createFrozenPartialWordSlot,
|
|
1625
|
+
buildTable: buildPartialWordTable,
|
|
1626
|
+
expand: expandPartialWords,
|
|
1627
|
+
countSpelling,
|
|
1628
|
+
encodeSpellings: encodePartialSpellings,
|
|
1629
|
+
spellingCountsOf,
|
|
1630
|
+
highlightMatcher: partialWordMatcher,
|
|
1631
|
+
addTermMembers: addPartialWordMembers,
|
|
1632
|
+
highlightWords: partialWordsToHighlight
|
|
1633
|
+
};
|
|
1634
|
+
|
|
1635
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, encodeDocument, encodeDocumentReply, expandTermPrefix, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isLatitude, isLongitude, isQuantizationMode, isStorageMode, messagePackUpperBoundBytes, parseVectorDimension, partialFieldsOf, partialWordSearchModule, partitionDocumentFrequency, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireValidFieldTypes, sameFields, schemaFieldsOf, suggestDisplayTerms, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, visitDocumentSpellings, visitDocumentSurfaceSpellings };
|