@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
3
|
+
|
|
4
|
+
// src/persistence/object-store.ts
|
|
5
|
+
function objectKeyPrefix(prefix, adapter) {
|
|
6
|
+
if (prefix === void 0 || prefix.length === 0) return "";
|
|
7
|
+
if (prefix.startsWith("/") || prefix.includes("\0")) {
|
|
8
|
+
throw new NarsilError(
|
|
9
|
+
ErrorCodes.CONFIG_INVALID,
|
|
10
|
+
`The ${adapter} key prefix must not start with '/' or hold a null byte`,
|
|
11
|
+
{ prefix }
|
|
12
|
+
);
|
|
13
|
+
}
|
|
14
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
15
|
+
}
|
|
16
|
+
function requireBucket(bucket, adapter) {
|
|
17
|
+
if (typeof bucket !== "string" || bucket.trim().length === 0) {
|
|
18
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The ${adapter} adapter needs a bucket name`, {});
|
|
19
|
+
}
|
|
20
|
+
return bucket;
|
|
21
|
+
}
|
|
22
|
+
function requireObjectKey(key) {
|
|
23
|
+
if (key.length === 0 || key.includes("\0")) {
|
|
24
|
+
throw new NarsilError(
|
|
25
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
26
|
+
"Invalid key: a key must be non-empty and hold no null byte",
|
|
27
|
+
{
|
|
28
|
+
key
|
|
29
|
+
}
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function refuseOversizedObject(key, bytes) {
|
|
34
|
+
if (bytes > MAX_FILE_BYTES) {
|
|
35
|
+
throw new NarsilError(
|
|
36
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
37
|
+
`The object "${key}" holds ${bytes} bytes, which passes the ${MAX_FILE_BYTES}-byte limit of one stored file`,
|
|
38
|
+
{ key, bytes, limit: MAX_FILE_BYTES }
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function objectStoreFailure(code, action, key, error) {
|
|
43
|
+
if (error instanceof NarsilError) return error;
|
|
44
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
45
|
+
return new NarsilError(code, `The object store could not ${action} "${key}": ${cause}`, { key, cause });
|
|
46
|
+
}
|
|
47
|
+
function lengthMismatch(key, length, yielded) {
|
|
48
|
+
return new NarsilError(
|
|
49
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
50
|
+
`The stream for "${key}" yielded ${yielded} bytes where the caller declared ${length}`,
|
|
51
|
+
{ key, length, yielded }
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
function requireStreamLength(key, length) {
|
|
55
|
+
if (!Number.isSafeInteger(length) || length < 0 || length > MAX_FILE_BYTES) {
|
|
56
|
+
throw new NarsilError(
|
|
57
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
58
|
+
`A stream for "${key}" declares ${length} bytes, which is not a byte count from 0 to ${MAX_FILE_BYTES}`,
|
|
59
|
+
{ key, length }
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async function* countedChunks(key, data, length) {
|
|
64
|
+
let yielded = 0;
|
|
65
|
+
for await (const chunk of data) {
|
|
66
|
+
yielded += chunk.byteLength;
|
|
67
|
+
if (yielded > length) throw lengthMismatch(key, length, yielded);
|
|
68
|
+
if (chunk.byteLength > 0) yield chunk;
|
|
69
|
+
}
|
|
70
|
+
if (yielded !== length) throw lengthMismatch(key, length, yielded);
|
|
71
|
+
}
|
|
72
|
+
async function* partsOf(chunks, partBytes) {
|
|
73
|
+
let pending = [];
|
|
74
|
+
let pendingBytes = 0;
|
|
75
|
+
for await (const chunk of chunks) {
|
|
76
|
+
pending.push(chunk);
|
|
77
|
+
pendingBytes += chunk.byteLength;
|
|
78
|
+
if (pendingBytes < partBytes) continue;
|
|
79
|
+
yield joinChunks(pending, pendingBytes);
|
|
80
|
+
pending = [];
|
|
81
|
+
pendingBytes = 0;
|
|
82
|
+
}
|
|
83
|
+
if (pendingBytes > 0 || pending.length > 0) yield joinChunks(pending, pendingBytes);
|
|
84
|
+
}
|
|
85
|
+
async function readWhole(chunks, length) {
|
|
86
|
+
const joined = new Uint8Array(length);
|
|
87
|
+
let offset = 0;
|
|
88
|
+
for await (const chunk of chunks) {
|
|
89
|
+
joined.set(chunk, offset);
|
|
90
|
+
offset += chunk.byteLength;
|
|
91
|
+
}
|
|
92
|
+
return joined;
|
|
93
|
+
}
|
|
94
|
+
function joinChunks(chunks, total) {
|
|
95
|
+
const joined = new Uint8Array(total);
|
|
96
|
+
let offset = 0;
|
|
97
|
+
for (const chunk of chunks) {
|
|
98
|
+
joined.set(chunk, offset);
|
|
99
|
+
offset += chunk.byteLength;
|
|
100
|
+
}
|
|
101
|
+
return joined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export { countedChunks, joinChunks, objectKeyPrefix, objectStoreFailure, partsOf, readWhole, refuseOversizedObject, requireBucket, requireObjectKey, requireStreamLength };
|
|
@@ -68,7 +68,12 @@ var ClusterMessageTypes = {
|
|
|
68
68
|
RESHARD_FORWARD: "cluster.reshard_forward",
|
|
69
69
|
RESHARD_CUTOVER: "cluster.reshard_cutover",
|
|
70
70
|
RESHARD_PURGE: "cluster.reshard_purge",
|
|
71
|
-
INDEX_SETTINGS: "cluster.index_settings"
|
|
71
|
+
INDEX_SETTINGS: "cluster.index_settings",
|
|
72
|
+
BACKUP: "cluster.backup",
|
|
73
|
+
BACKUP_RESTORE: "cluster.backup_restore",
|
|
74
|
+
BACKUP_DELETE: "cluster.backup_delete",
|
|
75
|
+
BACKUP_PARTITION: "cluster.backup_partition",
|
|
76
|
+
RESTORE_PARTITION: "cluster.restore_partition"
|
|
72
77
|
};
|
|
73
78
|
|
|
74
79
|
export { ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, QueryMessageTypes, ReplicationMessageTypes, TransportError, TransportErrorCodes };
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
1
|
+
import { answerRefusalOf } from './chunk-LDGGR7YT.mjs';
|
|
3
2
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
|
+
import { readRecentMemoryFigures, readMemoryFigures } from './chunk-6F6USYII.mjs';
|
|
4
4
|
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
5
|
+
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-XIHUI4KB.mjs';
|
|
5
6
|
import { ANALYSIS_CHANGE_MEMBERS } from './chunk-EEBCDP36.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
7
|
+
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXNQEW43.mjs';
|
|
8
|
+
import { loadNativeCore } from './chunk-NVTFQFUL.mjs';
|
|
9
|
+
import { MAX_FILTER_DEPTH } from './chunk-U46IZNCW.mjs';
|
|
8
10
|
import { MAX_SORT_FIELDS } from './chunk-FQBFZAWG.mjs';
|
|
9
|
-
import {
|
|
10
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
11
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
11
12
|
import { randomUUID } from 'crypto';
|
|
12
13
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
13
14
|
import { tmpdir } from 'os';
|
|
@@ -202,6 +203,7 @@ var STATUS_BY_CODE = {
|
|
|
202
203
|
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
203
204
|
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
204
205
|
RESTORE_MEMORY_EXCEEDED: 507,
|
|
206
|
+
MEMORY_WRITE_BLOCKED: 429,
|
|
205
207
|
ANALYSIS_CHANGE_REPLACED: 409,
|
|
206
208
|
ANALYSIS_CHANGE_NOT_READY: 503,
|
|
207
209
|
ANALYSIS_GENERATION_MISSING: 503,
|
|
@@ -217,6 +219,15 @@ var STATUS_BY_CODE = {
|
|
|
217
219
|
PERSISTENCE_CRC_MISMATCH: 503,
|
|
218
220
|
PERSISTENCE_WAL_CORRUPT: 503,
|
|
219
221
|
PERSISTENCE_FSYNC_FAILED: 503,
|
|
222
|
+
PERSISTENCE_DEPENDENCY_MISSING: 500,
|
|
223
|
+
BACKUP_NOT_FOUND: 404,
|
|
224
|
+
BACKUP_ALREADY_EXISTS: 409,
|
|
225
|
+
BACKUP_DAMAGED: 422,
|
|
226
|
+
BACKUP_DURABILITY_REQUIRED: 409,
|
|
227
|
+
BACKUP_IN_PROGRESS: 409,
|
|
228
|
+
BACKUP_STORE_CLAIMED: 409,
|
|
229
|
+
BACKUP_STORE_NOT_CONFIGURED: 501,
|
|
230
|
+
BACKUP_NOT_READY: 503,
|
|
220
231
|
EMBEDDING_FAILED: 503,
|
|
221
232
|
RERANK_FAILED: 503,
|
|
222
233
|
QUERY_ROUTING_FAILED: 503,
|
|
@@ -241,6 +252,7 @@ var STATUS_BY_CODE = {
|
|
|
241
252
|
PARTITION_UNASSIGNED: 503,
|
|
242
253
|
INSUFFICIENT_REPLICAS: 503,
|
|
243
254
|
TRANSPORT_DEPENDENCY_MISSING: 500,
|
|
255
|
+
NATIVE_DEPENDENCY_MISSING: 500,
|
|
244
256
|
ALLOCATION_INVALID_CONFIG: 400,
|
|
245
257
|
ALLOCATION_NO_DATA_NODES: 503,
|
|
246
258
|
ALLOCATION_FAILED: 503,
|
|
@@ -906,14 +918,19 @@ function validateRebalance(body) {
|
|
|
906
918
|
}
|
|
907
919
|
return null;
|
|
908
920
|
}
|
|
909
|
-
var
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
921
|
+
var TASK_TYPE_SET = {
|
|
922
|
+
optimizeVectors: true,
|
|
923
|
+
rebalance: true,
|
|
924
|
+
restore: true,
|
|
925
|
+
import: true,
|
|
926
|
+
importExport: true,
|
|
927
|
+
rebuildAnalysis: true,
|
|
928
|
+
changeAnalysis: true,
|
|
929
|
+
backup: true,
|
|
930
|
+
backupRestore: true,
|
|
931
|
+
backupDelete: true
|
|
932
|
+
};
|
|
933
|
+
var TASK_TYPES = Object.keys(TASK_TYPE_SET);
|
|
917
934
|
var TASK_STATUSES = ["running", "succeeded", "failed", "cancelled"];
|
|
918
935
|
function parseCountParam(raw, field, minimum) {
|
|
919
936
|
if (raw === null) return minimum;
|
|
@@ -1571,6 +1588,57 @@ function createRouteRunner(deps) {
|
|
|
1571
1588
|
};
|
|
1572
1589
|
}
|
|
1573
1590
|
|
|
1591
|
+
// src/server/routes/backups.ts
|
|
1592
|
+
function backupRoutes(shared) {
|
|
1593
|
+
const { bare, named, namedBody } = shared;
|
|
1594
|
+
return [
|
|
1595
|
+
{
|
|
1596
|
+
method: "post",
|
|
1597
|
+
path: "/indexes/:name/backups",
|
|
1598
|
+
handler: "createBackup",
|
|
1599
|
+
options: namedBody,
|
|
1600
|
+
operationId: "createBackup",
|
|
1601
|
+
summary: "Back up an index to the store outside the cluster, as a task",
|
|
1602
|
+
tags: ["Cluster"],
|
|
1603
|
+
request: "CreateBackupBody",
|
|
1604
|
+
responses: { 202: "TaskRecord" }
|
|
1605
|
+
},
|
|
1606
|
+
{
|
|
1607
|
+
method: "get",
|
|
1608
|
+
path: "/backups",
|
|
1609
|
+
handler: "listBackups",
|
|
1610
|
+
options: bare,
|
|
1611
|
+
operationId: "listBackups",
|
|
1612
|
+
summary: "List the backups in the store outside the cluster",
|
|
1613
|
+
tags: ["Cluster"],
|
|
1614
|
+
request: null,
|
|
1615
|
+
responses: { 200: "BackupListResponse" }
|
|
1616
|
+
},
|
|
1617
|
+
{
|
|
1618
|
+
method: "post",
|
|
1619
|
+
path: "/backups/:backup/_restore",
|
|
1620
|
+
handler: "restoreBackup",
|
|
1621
|
+
options: namedBody,
|
|
1622
|
+
operationId: "restoreBackup",
|
|
1623
|
+
summary: "Restore a backup as a new index, as a task",
|
|
1624
|
+
tags: ["Cluster"],
|
|
1625
|
+
request: "RestoreBackupBody",
|
|
1626
|
+
responses: { 202: "TaskRecord" }
|
|
1627
|
+
},
|
|
1628
|
+
{
|
|
1629
|
+
method: "del",
|
|
1630
|
+
path: "/backups/:backup",
|
|
1631
|
+
handler: "deleteBackup",
|
|
1632
|
+
options: named,
|
|
1633
|
+
operationId: "deleteBackup",
|
|
1634
|
+
summary: "Delete a backup and the files that no other backup lists, as a task",
|
|
1635
|
+
tags: ["Cluster"],
|
|
1636
|
+
request: null,
|
|
1637
|
+
responses: { 202: "TaskRecord" }
|
|
1638
|
+
}
|
|
1639
|
+
];
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1574
1642
|
// src/server/routes/documents.ts
|
|
1575
1643
|
function documentRoutes(shared) {
|
|
1576
1644
|
const { named, document, writeBody, readBody: readBody2, batchBody, importBody, documentWriteBody } = shared;
|
|
@@ -2305,7 +2373,8 @@ function serverRoutes(limits) {
|
|
|
2305
2373
|
...indexRoutes(shared),
|
|
2306
2374
|
...documentRoutes(shared),
|
|
2307
2375
|
...searchRoutes(shared),
|
|
2308
|
-
...maintenanceRoutes(shared)
|
|
2376
|
+
...maintenanceRoutes(shared),
|
|
2377
|
+
...backupRoutes(shared)
|
|
2309
2378
|
];
|
|
2310
2379
|
}
|
|
2311
2380
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-4UMZ7KQL.mjs';
|
|
2
|
+
import { createNarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
2
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
3
|
-
import { createNarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
4
4
|
|
|
5
5
|
// src/core/tokenizer/constants.ts
|
|
6
6
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { WORKER_YOUNG_GENERATION_MB } from './chunk-3MXV3VHR.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
3
|
+
|
|
4
|
+
// src/runtime/detect.ts
|
|
5
|
+
function detectRuntimeName() {
|
|
6
|
+
if ("Bun" in globalThis) {
|
|
7
|
+
return "bun";
|
|
8
|
+
}
|
|
9
|
+
if ("Deno" in globalThis) {
|
|
10
|
+
return "deno";
|
|
11
|
+
}
|
|
12
|
+
const proc = globalThis.process;
|
|
13
|
+
if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
|
|
14
|
+
return "node";
|
|
15
|
+
}
|
|
16
|
+
return "browser";
|
|
17
|
+
}
|
|
18
|
+
function getCpuCount() {
|
|
19
|
+
const nav = globalThis.navigator;
|
|
20
|
+
if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
|
|
21
|
+
return nav.hardwareConcurrency;
|
|
22
|
+
}
|
|
23
|
+
return 1;
|
|
24
|
+
}
|
|
25
|
+
function detectRuntime() {
|
|
26
|
+
const runtime = detectRuntimeName();
|
|
27
|
+
return {
|
|
28
|
+
runtime,
|
|
29
|
+
cpuCount: getCpuCount(),
|
|
30
|
+
supportsWorkerThreads: runtime === "node" || runtime === "bun",
|
|
31
|
+
supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
|
|
32
|
+
supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
|
|
33
|
+
supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
|
|
34
|
+
supportsBroadcastChannel: "BroadcastChannel" in globalThis
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// src/workers/entry-point.ts
|
|
39
|
+
var DIST_SEGMENT = "/dist/";
|
|
40
|
+
function resolveWorkerEntry(moduleUrl, sourceDirectory, distEntry) {
|
|
41
|
+
const distIndex = moduleUrl.lastIndexOf(DIST_SEGMENT);
|
|
42
|
+
const entry = distIndex !== -1 ? new URL(distEntry, moduleUrl.slice(0, distIndex + DIST_SEGMENT.length)).href : sourceDirectory.test(moduleUrl) ? moduleUrl.replace(sourceDirectory, `${DIST_SEGMENT}${distEntry}`) : null;
|
|
43
|
+
return entry === moduleUrl ? null : entry;
|
|
44
|
+
}
|
|
45
|
+
function missingWorkerEntry(moduleUrl) {
|
|
46
|
+
return new NarsilError(
|
|
47
|
+
ErrorCodes.CONFIG_INVALID,
|
|
48
|
+
`The engine finds no worker entry beside its own module at "${moduleUrl}", which happens where a bundler folds @delali/narsil into an application bundle. Keep @delali/narsil outside the bundle, or set workers.enabled to false`,
|
|
49
|
+
{ moduleUrl }
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// src/workers/resource-limits.ts
|
|
54
|
+
function workerResourceLimits() {
|
|
55
|
+
return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export { detectRuntime, missingWorkerEntry, resolveWorkerEntry, workerResourceLimits };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { packChecksummedEnvelope } from './chunk-
|
|
2
|
-
import { fnv1a } from './chunk-
|
|
3
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { packChecksummedEnvelope } from './chunk-VQDZVWN3.mjs';
|
|
2
|
+
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
3
|
+
import { ErrorCodes, NarsilError } from './chunk-MDF6PDIA.mjs';
|
|
4
4
|
import { encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/persistence/durability/segment/store.ts
|
|
@@ -1,15 +1,109 @@
|
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, readOldGenerationStatistics, readResidentMemory, partitionsIn, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-3HV7M5OJ.mjs';
|
|
2
|
+
import { FNV_OFFSET_BASIS, fnv1aContinue, fnv1aContinueTaggedNumbers } from './chunk-KAKQ74LS.mjs';
|
|
3
|
+
import { admittedHeapBytes } from './chunk-XIHUI4KB.mjs';
|
|
4
|
+
import { engineThreadYoungGenerationBytes } from './chunk-Q2UL2TPC.mjs';
|
|
1
5
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
6
|
+
import { MIN_OVERSAMPLE } from './chunk-NVTFQFUL.mjs';
|
|
7
|
+
import { turnIsSpent, yieldTurn } from './chunk-LQG5G2LY.mjs';
|
|
8
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-BTPGQC6Z.mjs';
|
|
9
|
+
import { flattenSchema, partialFieldsOf } from './chunk-SOUZMPME.mjs';
|
|
5
10
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import {
|
|
11
|
+
import { tokenize } from './chunk-65WCV5JO.mjs';
|
|
12
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType, fuzzyTermMatches } from './chunk-KYVRZAYP.mjs';
|
|
13
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
9
15
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
16
|
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
|
|
18
|
+
// src/engine/memory/constants.ts
|
|
19
|
+
var DEFAULT_BLOCK_WRITES_ABOVE = 0.9;
|
|
20
|
+
var DEFAULT_RESUME_WRITES_BELOW = 0.8;
|
|
21
|
+
var RECENT_MEMORY_FIGURES_MAX_AGE_MS = 10;
|
|
22
|
+
var RECENT_MEMORY_FIGURES_MAX_ADMITTED_FRACTION = 0.01;
|
|
23
|
+
var FULL_MEMORY_COLLECTION_INTERVAL_MS = 5e3;
|
|
24
|
+
var ENGINE_MEMORY_FIGURES_MAX_AGE_MS = 1e3;
|
|
25
|
+
var DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE = 24;
|
|
26
|
+
var DOCUMENT_VALUE_HEAP_BYTES = 256;
|
|
27
|
+
var DOCUMENT_PROPERTY_HEAP_BYTES = 128;
|
|
28
|
+
var DISTINCT_TERM_HEAP_BYTES = 2048;
|
|
29
|
+
var TERM_UNIT_HEAP_BYTES = 24;
|
|
30
|
+
var PARTIAL_SPELLING_HEAP_BYTES = 512;
|
|
31
|
+
var PATTERN_RUN_HEAP_BYTES = 512;
|
|
32
|
+
var ENUM_VALUE_HEAP_BYTES = 256;
|
|
33
|
+
var VECTOR_COMPONENT_HEAP_BYTES = 80;
|
|
34
|
+
var VECTOR_DOCUMENT_HEAP_BYTES = 5120;
|
|
35
|
+
var NORMALIZATION_EXPANSION_FACTOR = 3;
|
|
36
|
+
var CASE_FOLD_EXPANSION_FACTOR = 3;
|
|
37
|
+
|
|
38
|
+
// src/engine/memory/measure.ts
|
|
39
|
+
var lastFigures = null;
|
|
40
|
+
var lastFiguresComplete = false;
|
|
41
|
+
var lastFiguresReadAt = Number.NEGATIVE_INFINITY;
|
|
42
|
+
var admittedBytesAtLastFigures = 0;
|
|
43
|
+
var recentFigures = {
|
|
44
|
+
processBytes: 0,
|
|
45
|
+
processLimitBytes: 0,
|
|
46
|
+
heapBytes: 0,
|
|
47
|
+
heapLimitBytes: 0,
|
|
48
|
+
youngHeapBytes: 0,
|
|
49
|
+
heapCommittedBytes: 0,
|
|
50
|
+
youngCeilingBytes: 0,
|
|
51
|
+
youngReserveBytes: 0
|
|
52
|
+
};
|
|
53
|
+
function readFreshFigures() {
|
|
54
|
+
try {
|
|
55
|
+
const heap = readOldGenerationStatistics();
|
|
56
|
+
const resident = readResidentMemory();
|
|
57
|
+
lastFiguresComplete = heap !== null && resident !== null;
|
|
58
|
+
if (heap === null && resident === null) return null;
|
|
59
|
+
return {
|
|
60
|
+
processBytes: resident?.residentBytes ?? 0,
|
|
61
|
+
processLimitBytes: resident?.limitBytes ?? Number.POSITIVE_INFINITY,
|
|
62
|
+
heapBytes: heap?.usedBytes ?? 0,
|
|
63
|
+
heapLimitBytes: heap?.limitBytes ?? Number.POSITIVE_INFINITY,
|
|
64
|
+
youngHeapBytes: heap?.youngUsedBytes ?? 0,
|
|
65
|
+
heapCommittedBytes: heap === null ? 0 : Math.max(heap.committedBytes, heap.usedBytes),
|
|
66
|
+
youngCeilingBytes: heap?.youngCeilingBytes ?? 0,
|
|
67
|
+
youngReserveBytes: (heap?.youngReserveBytes ?? 0) + engineThreadYoungGenerationBytes()
|
|
68
|
+
};
|
|
69
|
+
} catch {
|
|
70
|
+
lastFiguresComplete = false;
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function measure() {
|
|
75
|
+
const figures = readFreshFigures();
|
|
76
|
+
lastFigures = figures;
|
|
77
|
+
lastFiguresReadAt = performance.now();
|
|
78
|
+
admittedBytesAtLastFigures = admittedHeapBytes();
|
|
79
|
+
return figures;
|
|
80
|
+
}
|
|
81
|
+
function recent(maxAgeMs) {
|
|
82
|
+
const admittedSince = admittedHeapBytes() - admittedBytesAtLastFigures;
|
|
83
|
+
if (lastFigures === null || performance.now() - lastFiguresReadAt >= maxAgeMs || admittedSince > RECENT_MEMORY_FIGURES_MAX_ADMITTED_FRACTION * lastFigures.heapLimitBytes) {
|
|
84
|
+
return measure();
|
|
85
|
+
}
|
|
86
|
+
recentFigures.processLimitBytes = lastFigures.processLimitBytes;
|
|
87
|
+
recentFigures.heapLimitBytes = lastFigures.heapLimitBytes;
|
|
88
|
+
recentFigures.processBytes = lastFigures.processBytes + admittedSince;
|
|
89
|
+
recentFigures.heapBytes = lastFigures.heapBytes + admittedSince;
|
|
90
|
+
recentFigures.youngHeapBytes = lastFigures.youngHeapBytes;
|
|
91
|
+
recentFigures.heapCommittedBytes = lastFigures.heapCommittedBytes + admittedSince;
|
|
92
|
+
recentFigures.youngCeilingBytes = lastFigures.youngCeilingBytes;
|
|
93
|
+
recentFigures.youngReserveBytes = lastFigures.youngReserveBytes;
|
|
94
|
+
return recentFigures;
|
|
95
|
+
}
|
|
96
|
+
function readMemoryFigures() {
|
|
97
|
+
const figures = measure();
|
|
98
|
+
return lastFiguresComplete ? figures : null;
|
|
99
|
+
}
|
|
100
|
+
function readRecentMemoryFigures() {
|
|
101
|
+
const figures = recent(RECENT_MEMORY_FIGURES_MAX_AGE_MS);
|
|
102
|
+
return lastFiguresComplete ? figures : null;
|
|
103
|
+
}
|
|
104
|
+
function readEngineMemoryFigures() {
|
|
105
|
+
return measure();
|
|
106
|
+
}
|
|
13
107
|
|
|
14
108
|
// src/search/cursor-binding.ts
|
|
15
109
|
var ABSENT_TAG = 0;
|
|
@@ -596,6 +690,12 @@ function validateDocId(docId) {
|
|
|
596
690
|
);
|
|
597
691
|
}
|
|
598
692
|
}
|
|
693
|
+
function validateBatchList(given, listed) {
|
|
694
|
+
if (Array.isArray(given)) return;
|
|
695
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `A batch takes its ${listed} as an array`, {
|
|
696
|
+
received: given === null ? "null" : typeof given
|
|
697
|
+
});
|
|
698
|
+
}
|
|
599
699
|
function validatePartitionConfig(partitions) {
|
|
600
700
|
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
601
701
|
if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
@@ -1465,7 +1565,7 @@ var runtimeHost = {
|
|
|
1465
1565
|
function sortsWithoutScores(params) {
|
|
1466
1566
|
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
1467
1567
|
}
|
|
1468
|
-
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1568
|
+
async function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1469
1569
|
const { manager, language, config } = context;
|
|
1470
1570
|
const schema = config.schema;
|
|
1471
1571
|
let anchorKey = null;
|
|
@@ -1491,6 +1591,13 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
1491
1591
|
let count = 0;
|
|
1492
1592
|
const clock = context.patternWork.clock;
|
|
1493
1593
|
for (const partition of partitions) {
|
|
1594
|
+
if (turnIsSpent()) await yieldTurn();
|
|
1595
|
+
partition.prepareSortedPage({ fields, directions, fieldTypes, modes });
|
|
1596
|
+
if (partitionFacetConfig !== void 0) {
|
|
1597
|
+
if (turnIsSpent()) await yieldTurn();
|
|
1598
|
+
partition.prepareFacets(partitionFacetConfig, schema);
|
|
1599
|
+
}
|
|
1600
|
+
if (turnIsSpent()) await yieldTurn();
|
|
1494
1601
|
clock?.check();
|
|
1495
1602
|
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1496
1603
|
clock?.check();
|
|
@@ -1550,7 +1657,7 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1550
1657
|
}
|
|
1551
1658
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1552
1659
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1553
|
-
return import('./runtime-
|
|
1660
|
+
return import('./runtime-L2DZYY6N.mjs').then(
|
|
1554
1661
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1555
1662
|
);
|
|
1556
1663
|
}
|
|
@@ -1560,7 +1667,7 @@ function executeQuery(request, callerContext) {
|
|
|
1560
1667
|
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
1561
1668
|
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
1562
1669
|
}
|
|
1563
|
-
return import('./runtime-
|
|
1670
|
+
return import('./runtime-L2DZYY6N.mjs').then(
|
|
1564
1671
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1565
1672
|
);
|
|
1566
1673
|
}
|
|
@@ -1596,7 +1703,7 @@ async function runQuery(request, callerContext) {
|
|
|
1596
1703
|
let facets;
|
|
1597
1704
|
let groups;
|
|
1598
1705
|
if (sortSignature !== null && hasTerm && !isVectorOnly && !isHybridMode && sortsWithoutScores(params)) {
|
|
1599
|
-
const page = executeSortedQueryPage(params, context, limit, offset, sortSignature);
|
|
1706
|
+
const page = await executeSortedQueryPage(params, context, limit, offset, sortSignature);
|
|
1600
1707
|
paginated = page.hits;
|
|
1601
1708
|
nextCursor = page.cursor;
|
|
1602
1709
|
count = page.count;
|
|
@@ -1861,4 +1968,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1861
1968
|
return best;
|
|
1862
1969
|
}
|
|
1863
1970
|
|
|
1864
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, lazyQueryBinding, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, storedBM25Params, validateBM25Params, validateBM25Shape, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
1971
|
+
export { CASE_FOLD_EXPANSION_FACTOR, DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_RESUME_WRITES_BELOW, DISTINCT_TERM_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, ENGINE_MEMORY_FIGURES_MAX_AGE_MS, ENUM_VALUE_HEAP_BYTES, FULL_MEMORY_COLLECTION_INTERVAL_MS, NORMALIZATION_EXPANSION_FACTOR, PARTIAL_SPELLING_HEAP_BYTES, PATTERN_RUN_HEAP_BYTES, TERM_UNIT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, lazyQueryBinding, listBindingOf, now, queryBindingOf, readEngineMemoryFigures, readMemoryFigures, readRecentMemoryFigures, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, storedBM25Params, validateBM25Params, validateBM25Shape, validateBatchList, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { nodeThreadId } from './chunk-
|
|
1
|
+
import { nodeThreadId } from './chunk-BEZH7VEH.mjs';
|
|
2
2
|
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
3
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
3
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-QN5WVV5K.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
4
5
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
6
6
|
import { openSync, close, readFileSync } from 'fs';
|
|
7
7
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
8
8
|
import { endianness } from 'os';
|
|
@@ -147,7 +147,7 @@ async function removeStaleLock(lockPath, asidePath, isStale) {
|
|
|
147
147
|
|
|
148
148
|
// src/persistence/durability/directory-lock.ts
|
|
149
149
|
var LOCK_FILE_NAME = ".narsil.lock";
|
|
150
|
-
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
150
|
+
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
151
151
|
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
152
152
|
function errnoOf2(err) {
|
|
153
153
|
return err?.code;
|
|
@@ -483,9 +483,9 @@ function createDurableDirectory(root) {
|
|
|
483
483
|
if (stat.size === 0) {
|
|
484
484
|
return new Uint8Array(0);
|
|
485
485
|
}
|
|
486
|
-
const
|
|
487
|
-
await handle.read(
|
|
488
|
-
return
|
|
486
|
+
const bytes = new Uint8Array(stat.size);
|
|
487
|
+
await handle.read(bytes, 0, stat.size, 0);
|
|
488
|
+
return bytes;
|
|
489
489
|
},
|
|
490
490
|
async writeSlot(offset, slot) {
|
|
491
491
|
try {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { crc32 } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { crc32 } from './chunk-VQDZVWN3.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
3
3
|
import { decode, encode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
var PAGED_SEGMENT_ENVELOPE_VERSION = 6;
|
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import { encodeDocumentJson } from './chunk-
|
|
2
|
-
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-
|
|
1
|
+
import { encodeDocumentJson } from './chunk-3SBJVQLL.mjs';
|
|
2
|
+
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-7N223GWV.mjs';
|
|
3
3
|
import { createVectorFileReader } from './chunk-FOCV4CBZ.mjs';
|
|
4
|
-
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-
|
|
5
|
-
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-
|
|
6
|
-
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-
|
|
7
|
-
import { wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-
|
|
8
|
-
import { isCompositePartition, buildPartialWordTableFrom, createPartitionIndex, createCompositePartition, mergeDocFrequencies, scanPostingBlocks } from './chunk-
|
|
9
|
-
import {
|
|
10
|
-
import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
|
|
11
|
-
import { flattenSchema, encodeDocument } from './chunk-KGOXHLLZ.mjs';
|
|
12
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
+
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-QN5WVV5K.mjs';
|
|
5
|
+
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-U2726TL5.mjs';
|
|
6
|
+
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-VQDZVWN3.mjs';
|
|
7
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-O4KUNDVF.mjs';
|
|
8
|
+
import { isCompositePartition, buildPartialWordTableFrom, createPartitionIndex, createCompositePartition, mergeDocFrequencies, scanPostingBlocks } from './chunk-3HV7M5OJ.mjs';
|
|
9
|
+
import { flattenSchema, parseVectorDimension, encodeDocument } from './chunk-SOUZMPME.mjs';
|
|
13
10
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
11
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
12
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
13
|
import { decode, encode } from '@msgpack/msgpack';
|
|
16
14
|
import { deflateRawSync, inflateRawSync } from 'zlib';
|
|
17
15
|
|
|
@@ -1309,7 +1307,7 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
1309
1307
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
1310
1308
|
if (dimension === null) return null;
|
|
1311
1309
|
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
1312
|
-
const { openColdVectorField } = await import('./cold-module-
|
|
1310
|
+
const { openColdVectorField } = await import('./cold-module-2WWGBFJC.mjs');
|
|
1313
1311
|
return openColdVectorField({
|
|
1314
1312
|
fieldPath: field.fieldPath,
|
|
1315
1313
|
dimension,
|