@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createBitSet, bitsetIsEmpty, GrowableUint32, PHRASE_ROWS_START, bitsetHas, bitsetSet, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import {
|
|
2
|
+
import { MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-WXLBZJMT.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
3
4
|
|
|
4
5
|
// src/core/partition/phrase-matching.ts
|
|
5
6
|
var UINT32_BITS = 32;
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { createRequestId, fnv1a } from './chunk-MPK76GVS.mjs';
|
|
2
|
+
import { resolveWorkerCount, SHUTDOWN_TIMEOUT_MS } from './chunk-TY7L6OXO.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
4
|
+
|
|
5
|
+
// src/workers/pool-memory.ts
|
|
6
|
+
function toFinite(value) {
|
|
7
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
8
|
+
}
|
|
9
|
+
function toHeapLimit(value) {
|
|
10
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
11
|
+
}
|
|
12
|
+
async function memoryStatsOf(slots) {
|
|
13
|
+
const indices = [];
|
|
14
|
+
const pending = [];
|
|
15
|
+
for (const [slotIndex, slot] of slots) {
|
|
16
|
+
indices.push(slotIndex);
|
|
17
|
+
pending.push(
|
|
18
|
+
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
const reports = await Promise.all(pending);
|
|
22
|
+
const stats = [];
|
|
23
|
+
for (let n = 0; n < indices.length; n++) {
|
|
24
|
+
const report = reports[n];
|
|
25
|
+
stats.push({
|
|
26
|
+
workerId: indices[n],
|
|
27
|
+
heapUsed: toFinite(report?.heapUsed),
|
|
28
|
+
heapTotal: toFinite(report?.heapTotal),
|
|
29
|
+
heapLimit: toHeapLimit(report?.heapLimit),
|
|
30
|
+
external: toFinite(report?.external)
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return stats;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// src/workers/pool.ts
|
|
37
|
+
function createWorkerPool(config) {
|
|
38
|
+
const workerCount = resolveWorkerCount(config.count);
|
|
39
|
+
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
40
|
+
const workers = /* @__PURE__ */ new Map();
|
|
41
|
+
const deadSlots = /* @__PURE__ */ new Set();
|
|
42
|
+
const goneSlots = /* @__PURE__ */ new Set();
|
|
43
|
+
const threadsStillRunning = /* @__PURE__ */ new Set();
|
|
44
|
+
const waitingForEveryThread = [];
|
|
45
|
+
const indexAssignment = /* @__PURE__ */ new Map();
|
|
46
|
+
let isShutdown = false;
|
|
47
|
+
function handleWorkerDeath(slotIndex, error) {
|
|
48
|
+
if (isShutdown) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const slot = workers.get(slotIndex);
|
|
52
|
+
if (slot !== void 0 && !slot.serving) {
|
|
53
|
+
workers.delete(slotIndex);
|
|
54
|
+
goneSlots.delete(slotIndex);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (deadSlots.has(slotIndex)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
deadSlots.add(slotIndex);
|
|
61
|
+
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
62
|
+
workers.delete(slotIndex);
|
|
63
|
+
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
64
|
+
}
|
|
65
|
+
function handleWorkerGone(slotIndex) {
|
|
66
|
+
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
goneSlots.add(slotIndex);
|
|
70
|
+
config.onWorkerGone?.(slotIndex);
|
|
71
|
+
}
|
|
72
|
+
function spawnSlot(slotIndex, serving) {
|
|
73
|
+
const slot = {
|
|
74
|
+
executor: config.workerFactory(
|
|
75
|
+
slotIndex,
|
|
76
|
+
(error) => handleWorkerDeath(slotIndex, error),
|
|
77
|
+
() => {
|
|
78
|
+
noteThreadGone(slot.executor);
|
|
79
|
+
handleWorkerGone(slotIndex);
|
|
80
|
+
}
|
|
81
|
+
),
|
|
82
|
+
indexes: /* @__PURE__ */ new Set(),
|
|
83
|
+
inFlight: 0,
|
|
84
|
+
serving
|
|
85
|
+
};
|
|
86
|
+
threadsStillRunning.add(slot.executor);
|
|
87
|
+
workers.set(slotIndex, slot);
|
|
88
|
+
return slot;
|
|
89
|
+
}
|
|
90
|
+
function noteThreadGone(executor) {
|
|
91
|
+
threadsStillRunning.delete(executor);
|
|
92
|
+
if (threadsStillRunning.size > 0) return;
|
|
93
|
+
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
94
|
+
}
|
|
95
|
+
function whenEveryThreadIsGone() {
|
|
96
|
+
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
97
|
+
return new Promise((resolve) => {
|
|
98
|
+
waitingForEveryThread.push(resolve);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
function shutDownUntilTheThreadIsGone(executor) {
|
|
102
|
+
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
103
|
+
}
|
|
104
|
+
function ensureWorker(slotIndex) {
|
|
105
|
+
const slot = workers.get(slotIndex);
|
|
106
|
+
if (slot !== void 0) return slot;
|
|
107
|
+
if (deadSlots.has(slotIndex)) return void 0;
|
|
108
|
+
return spawnSlot(slotIndex, true);
|
|
109
|
+
}
|
|
110
|
+
function deadWorkerIds() {
|
|
111
|
+
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
112
|
+
}
|
|
113
|
+
function spawnReplacement(slotIndex) {
|
|
114
|
+
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
115
|
+
const slot = spawnSlot(slotIndex, false);
|
|
116
|
+
return {
|
|
117
|
+
workerId: slotIndex,
|
|
118
|
+
executor: slot.executor,
|
|
119
|
+
hold(indexName) {
|
|
120
|
+
slot.indexes.add(indexName);
|
|
121
|
+
},
|
|
122
|
+
admit() {
|
|
123
|
+
if (workers.get(slotIndex) !== slot) return;
|
|
124
|
+
deadSlots.delete(slotIndex);
|
|
125
|
+
goneSlots.delete(slotIndex);
|
|
126
|
+
slot.serving = true;
|
|
127
|
+
},
|
|
128
|
+
abandon() {
|
|
129
|
+
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
130
|
+
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
function isLongQueryWorker(workerId) {
|
|
135
|
+
return workerId >= queryWorkerCount;
|
|
136
|
+
}
|
|
137
|
+
function assignSlot(indexName) {
|
|
138
|
+
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
139
|
+
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
140
|
+
const candidate = (preferred + probe) % queryWorkerCount;
|
|
141
|
+
if (!deadSlots.has(candidate)) {
|
|
142
|
+
return candidate;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
146
|
+
}
|
|
147
|
+
function addIndex(indexName) {
|
|
148
|
+
if (isShutdown) {
|
|
149
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
150
|
+
}
|
|
151
|
+
if (indexAssignment.has(indexName)) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const slotIndex = assignSlot(indexName);
|
|
155
|
+
const slot = ensureWorker(slotIndex);
|
|
156
|
+
if (!slot) {
|
|
157
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
158
|
+
}
|
|
159
|
+
slot.indexes.add(indexName);
|
|
160
|
+
indexAssignment.set(indexName, slotIndex);
|
|
161
|
+
}
|
|
162
|
+
function removeIndex(indexName) {
|
|
163
|
+
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
164
|
+
indexAssignment.delete(indexName);
|
|
165
|
+
}
|
|
166
|
+
function getExecutor(indexName) {
|
|
167
|
+
if (isShutdown) {
|
|
168
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
169
|
+
}
|
|
170
|
+
const slotIndex = indexAssignment.get(indexName);
|
|
171
|
+
if (slotIndex === void 0) {
|
|
172
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
173
|
+
}
|
|
174
|
+
const slot = workers.get(slotIndex);
|
|
175
|
+
if (slot?.serving) {
|
|
176
|
+
return slot.executor;
|
|
177
|
+
}
|
|
178
|
+
for (const survivor of workers.values()) {
|
|
179
|
+
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
180
|
+
return survivor.executor;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
184
|
+
}
|
|
185
|
+
async function shutdown() {
|
|
186
|
+
if (isShutdown) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
isShutdown = true;
|
|
190
|
+
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
191
|
+
let timeoutId;
|
|
192
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
193
|
+
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
194
|
+
});
|
|
195
|
+
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
196
|
+
() => clearTimeout(timeoutId)
|
|
197
|
+
);
|
|
198
|
+
});
|
|
199
|
+
await Promise.allSettled(shutdownPromises);
|
|
200
|
+
indexAssignment.clear();
|
|
201
|
+
}
|
|
202
|
+
function getAllExecutors() {
|
|
203
|
+
return [...workers.values()].map((slot) => slot.executor);
|
|
204
|
+
}
|
|
205
|
+
function executorEntries() {
|
|
206
|
+
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
207
|
+
}
|
|
208
|
+
function executorsHolding(indexName) {
|
|
209
|
+
const holders = [];
|
|
210
|
+
for (const slot of workers.values()) {
|
|
211
|
+
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
212
|
+
}
|
|
213
|
+
return holders;
|
|
214
|
+
}
|
|
215
|
+
function lease(workerId, slot) {
|
|
216
|
+
slot.inFlight += 1;
|
|
217
|
+
let released = false;
|
|
218
|
+
return {
|
|
219
|
+
workerId,
|
|
220
|
+
executor: slot.executor,
|
|
221
|
+
release() {
|
|
222
|
+
if (released) return;
|
|
223
|
+
released = true;
|
|
224
|
+
slot.inFlight -= 1;
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function leaseLeastBusy(indexName) {
|
|
229
|
+
let chosenId = -1;
|
|
230
|
+
let chosen = null;
|
|
231
|
+
for (const [workerId, slot] of workers) {
|
|
232
|
+
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
233
|
+
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
234
|
+
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
235
|
+
chosenId = workerId;
|
|
236
|
+
chosen = slot;
|
|
237
|
+
}
|
|
238
|
+
if (chosen.inFlight === 0) break;
|
|
239
|
+
}
|
|
240
|
+
return chosen === null ? null : lease(chosenId, chosen);
|
|
241
|
+
}
|
|
242
|
+
function queriesInFlight() {
|
|
243
|
+
let total = 0;
|
|
244
|
+
for (const [workerId, slot] of workers) {
|
|
245
|
+
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
246
|
+
}
|
|
247
|
+
return total;
|
|
248
|
+
}
|
|
249
|
+
function leaseIdle(limit) {
|
|
250
|
+
const leases = [];
|
|
251
|
+
for (const [workerId, slot] of workers) {
|
|
252
|
+
if (leases.length >= limit) break;
|
|
253
|
+
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
254
|
+
}
|
|
255
|
+
return leases;
|
|
256
|
+
}
|
|
257
|
+
function addIndexToAll(indexName) {
|
|
258
|
+
if (isShutdown) {
|
|
259
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
260
|
+
}
|
|
261
|
+
for (let i = 0; i < workerCount; i++) {
|
|
262
|
+
ensureWorker(i)?.indexes.add(indexName);
|
|
263
|
+
}
|
|
264
|
+
indexAssignment.set(indexName, assignSlot(indexName));
|
|
265
|
+
}
|
|
266
|
+
function spawnAll() {
|
|
267
|
+
if (isShutdown) {
|
|
268
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
269
|
+
}
|
|
270
|
+
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
271
|
+
}
|
|
272
|
+
return {
|
|
273
|
+
getExecutor,
|
|
274
|
+
getAllExecutors,
|
|
275
|
+
executorEntries,
|
|
276
|
+
executorsHolding,
|
|
277
|
+
leaseLeastBusy,
|
|
278
|
+
leaseIdle,
|
|
279
|
+
isLongQueryWorker,
|
|
280
|
+
queriesInFlight,
|
|
281
|
+
spawnAll,
|
|
282
|
+
deadWorkerIds,
|
|
283
|
+
spawnReplacement,
|
|
284
|
+
get workerCount() {
|
|
285
|
+
return workerCount;
|
|
286
|
+
},
|
|
287
|
+
addIndex,
|
|
288
|
+
addIndexToAll,
|
|
289
|
+
removeIndex,
|
|
290
|
+
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
291
|
+
shutdown,
|
|
292
|
+
whenEveryThreadIsGone
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export { createWorkerPool };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abandonChangeOf } from './chunk-
|
|
1
|
+
import { abandonChangeOf } from './chunk-4DYLCICG.mjs';
|
|
2
2
|
|
|
3
3
|
// src/narsil/restore-queue.ts
|
|
4
4
|
function restoreDepsOf(core, host) {
|
|
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
24
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
25
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
26
26
|
const restored = previous.then(async () => {
|
|
27
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-DQ6A4PQP.mjs');
|
|
28
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
29
29
|
});
|
|
30
30
|
const settled = restored.then(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
-
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-
|
|
3
|
-
import {
|
|
2
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-4RD3AFSB.mjs';
|
|
3
|
+
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-WXLBZJMT.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
4
5
|
|
|
5
6
|
// src/search/hybrid-depth.ts
|
|
6
7
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,18 +1,22 @@
|
|
|
1
|
-
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
2
|
-
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-
|
|
3
|
-
import
|
|
4
|
-
import './chunk-
|
|
5
|
-
import
|
|
6
|
-
import './chunk-
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
1
|
+
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-D6UFGBF5.mjs';
|
|
2
|
+
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-NVEJJP4R.mjs';
|
|
3
|
+
import './chunk-JNOAKDUM.mjs';
|
|
4
|
+
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-W643QV3L.mjs';
|
|
5
|
+
import './chunk-QMNZFFD3.mjs';
|
|
6
|
+
import { vectorsPerFile } from './chunk-BRWY3JCN.mjs';
|
|
7
|
+
import './chunk-ODQP7P3H.mjs';
|
|
8
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-KAEVGLYN.mjs';
|
|
9
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-SRL47USS.mjs';
|
|
10
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-BVX2MEHA.mjs';
|
|
11
|
+
import './chunk-MPK76GVS.mjs';
|
|
10
12
|
import './chunk-JPTQZWJQ.mjs';
|
|
11
13
|
import './chunk-26A5JFT2.mjs';
|
|
12
14
|
import './chunk-OIZS5KIX.mjs';
|
|
13
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
14
|
-
import './chunk-
|
|
15
|
-
import
|
|
15
|
+
import { MAX_FILE_BYTES } from './chunk-IAXZAVYL.mjs';
|
|
16
|
+
import './chunk-TY7L6OXO.mjs';
|
|
17
|
+
import './chunk-4RD3AFSB.mjs';
|
|
18
|
+
import './chunk-WXLBZJMT.mjs';
|
|
19
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
16
20
|
import { decode } from '@msgpack/msgpack';
|
|
17
21
|
|
|
18
22
|
// src/persistence/archive/memory-store.ts
|
|
@@ -82,7 +86,8 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
82
86
|
newFiles: files.length,
|
|
83
87
|
graphParts: null,
|
|
84
88
|
signature: { revision: 0, graphSize: -1, calibration: null },
|
|
85
|
-
readNewFile: (index) => files[index]
|
|
89
|
+
readNewFile: (index) => files[index],
|
|
90
|
+
release: () => void 0
|
|
86
91
|
};
|
|
87
92
|
}
|
|
88
93
|
function capture(input) {
|
|
@@ -556,7 +561,7 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
556
561
|
for (const ref of partition.segments) {
|
|
557
562
|
const { directory } = store;
|
|
558
563
|
if (directory !== void 0) {
|
|
559
|
-
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-
|
|
564
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-CEKA7MQH.mjs');
|
|
560
565
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
561
566
|
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
562
567
|
continue;
|
|
@@ -924,7 +929,7 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
924
929
|
if (manifest === null) missingPart("manifest");
|
|
925
930
|
const { directory } = staging;
|
|
926
931
|
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
927
|
-
const { bytesThatSegmentsHoldInMemory } = await import('./module-
|
|
932
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-CEKA7MQH.mjs');
|
|
928
933
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
929
934
|
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
930
935
|
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import { encodeExactJson, encodeDocumentJson } from './chunk-
|
|
4
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
7
|
-
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2JABMB4S.mjs';
|
|
2
|
+
import './chunk-NVEJJP4R.mjs';
|
|
3
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-6OWX3OP2.mjs';
|
|
4
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-QMNZFFD3.mjs';
|
|
5
|
+
import './chunk-5V74FFZE.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-KAEVGLYN.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-BVX2MEHA.mjs';
|
|
8
|
+
import './chunk-MPK76GVS.mjs';
|
|
8
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
10
|
import './chunk-26A5JFT2.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import
|
|
12
|
-
import './chunk-
|
|
11
|
+
import './chunk-IAXZAVYL.mjs';
|
|
12
|
+
import './chunk-TY7L6OXO.mjs';
|
|
13
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-4RD3AFSB.mjs';
|
|
14
|
+
import './chunk-WXLBZJMT.mjs';
|
|
15
|
+
import './chunk-PIS7ZAP7.mjs';
|
|
13
16
|
import { decode } from '@msgpack/msgpack';
|
|
14
17
|
|
|
15
18
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
@@ -91,7 +94,7 @@ async function wholeFileVectorReads(checkpoint) {
|
|
|
91
94
|
};
|
|
92
95
|
}
|
|
93
96
|
async function* pagedDocumentLines(checkpoint) {
|
|
94
|
-
const { openFileVectorReads, pagedPartitionLines } = await import('./module-
|
|
97
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-CEKA7MQH.mjs');
|
|
95
98
|
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
96
99
|
try {
|
|
97
100
|
for (const partition of checkpoint.manifest.partitions) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { E as ErrorDetail,
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
2
|
+
import { C as ClusterNamespace } from './config-CfhH1IWQ.js';
|
|
3
|
+
import { E as ErrorDetail, j as TaskStore } from './tasks-C4ElXr4P.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
1
|
+
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-65WCV5JO.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
3
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import {
|
|
5
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
5
6
|
|
|
6
7
|
// src/analysis/registry.ts
|
|
7
8
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -78,6 +79,20 @@ function resolveIndexAnalysis(config) {
|
|
|
78
79
|
};
|
|
79
80
|
}
|
|
80
81
|
|
|
82
|
+
// src/languages/registry.ts
|
|
83
|
+
var languages = /* @__PURE__ */ new Map();
|
|
84
|
+
languages.set("english", english);
|
|
85
|
+
function registerLanguage(module) {
|
|
86
|
+
languages.set(module.name, module);
|
|
87
|
+
}
|
|
88
|
+
function getLanguage(name) {
|
|
89
|
+
const lang = languages.get(name);
|
|
90
|
+
if (!lang) {
|
|
91
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
92
|
+
}
|
|
93
|
+
return lang;
|
|
94
|
+
}
|
|
95
|
+
|
|
81
96
|
// src/engine/analysis-change/pending-target.ts
|
|
82
97
|
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
83
98
|
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
@@ -127,6 +142,16 @@ async function oneAtATime(core, indexName, action) {
|
|
|
127
142
|
}
|
|
128
143
|
}
|
|
129
144
|
|
|
145
|
+
// src/core/synonyms/constants.ts
|
|
146
|
+
var MAX_SYNONYM_RULES = 1e5;
|
|
147
|
+
var MAX_SYNONYM_WORDS = 5e5;
|
|
148
|
+
var MAX_SYNONYM_WORD_BYTES = 4194304;
|
|
149
|
+
var MAX_SYNONYM_RULE_WORDS = 100;
|
|
150
|
+
var MIN_TWO_WAY_SYNONYM_WORDS = 2;
|
|
151
|
+
var MIN_ONE_WAY_SYNONYMS = 1;
|
|
152
|
+
var SYNONYMS_ADDED_PER_QUERY = 50;
|
|
153
|
+
var SYNONYM_EXPANSION_CACHE_ENTRIES = 1024;
|
|
154
|
+
|
|
130
155
|
// src/core/synonyms/combined-postings.ts
|
|
131
156
|
var NO_DELETED = { size: 0, has: () => false };
|
|
132
157
|
var FIELD_SLOTS = 65536;
|
|
@@ -384,6 +409,103 @@ function buildSynonymLookup(rules, analysis, raise) {
|
|
|
384
409
|
if (ruleTokenIds.length === 0) return null;
|
|
385
410
|
return packedLookup(interned.tokens, ruleStart, ruleTokenIds, ruleSource);
|
|
386
411
|
}
|
|
412
|
+
var FIRST_PRINTABLE_CODE = 32;
|
|
413
|
+
function holdsControlCharacter(word) {
|
|
414
|
+
for (let i = 0; i < word.length; i++) if (word.charCodeAt(i) < FIRST_PRINTABLE_CODE) return true;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
function utf8Length(word) {
|
|
418
|
+
let bytes = 0;
|
|
419
|
+
for (let i = 0; i < word.length; i++) {
|
|
420
|
+
const unit = word.charCodeAt(i);
|
|
421
|
+
if (unit < 128) bytes += 1;
|
|
422
|
+
else if (unit < 2048) bytes += 2;
|
|
423
|
+
else if (unit >= 55296 && unit < 56320 && i + 1 < word.length) {
|
|
424
|
+
const low = word.charCodeAt(i + 1);
|
|
425
|
+
if (low >= 56320 && low < 57344) {
|
|
426
|
+
bytes += 4;
|
|
427
|
+
i++;
|
|
428
|
+
} else bytes += 3;
|
|
429
|
+
} else bytes += 3;
|
|
430
|
+
}
|
|
431
|
+
return bytes;
|
|
432
|
+
}
|
|
433
|
+
function hasOnlyKeys(value, keys) {
|
|
434
|
+
const own = Object.keys(value);
|
|
435
|
+
return own.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
436
|
+
}
|
|
437
|
+
function readWord(value, rule, tally, raise) {
|
|
438
|
+
if (typeof value !== "string") {
|
|
439
|
+
raise(`Synonym rule ${rule} holds a word that is not a string`, { rule });
|
|
440
|
+
}
|
|
441
|
+
if (holdsControlCharacter(value)) {
|
|
442
|
+
raise(`Synonym rule ${rule} holds a word with a control character`, { rule });
|
|
443
|
+
}
|
|
444
|
+
tally.words++;
|
|
445
|
+
tally.bytes += utf8Length(value);
|
|
446
|
+
if (tally.words > MAX_SYNONYM_WORDS) {
|
|
447
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_WORDS} words`, { maxWords: MAX_SYNONYM_WORDS });
|
|
448
|
+
}
|
|
449
|
+
if (tally.bytes > MAX_SYNONYM_WORD_BYTES) {
|
|
450
|
+
raise(`The words of a synonym set hold at most ${MAX_SYNONYM_WORD_BYTES} bytes in UTF-8`, {
|
|
451
|
+
maxWordBytes: MAX_SYNONYM_WORD_BYTES
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
return value;
|
|
455
|
+
}
|
|
456
|
+
function readWords(value, rule, key, minimum, maximum, tally, raise) {
|
|
457
|
+
if (!Array.isArray(value) || value.length < minimum || value.length > maximum) {
|
|
458
|
+
raise(`Synonym rule ${rule} needs "${key}" as a list of ${minimum} to ${maximum} words`, { rule });
|
|
459
|
+
}
|
|
460
|
+
const words = new Array(value.length);
|
|
461
|
+
for (let i = 0; i < value.length; i++) words[i] = readWord(value[i], rule, tally, raise);
|
|
462
|
+
return words;
|
|
463
|
+
}
|
|
464
|
+
function readRule(value, rule, tally, raise) {
|
|
465
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
466
|
+
raise(`Synonym rule ${rule} must be an object`, { rule });
|
|
467
|
+
}
|
|
468
|
+
if (hasOnlyKeys(value, ["words"])) {
|
|
469
|
+
const words = Reflect.get(value, "words");
|
|
470
|
+
return { words: readWords(words, rule, "words", MIN_TWO_WAY_SYNONYM_WORDS, MAX_SYNONYM_RULE_WORDS, tally, raise) };
|
|
471
|
+
}
|
|
472
|
+
if (hasOnlyKeys(value, ["word", "synonyms"])) {
|
|
473
|
+
const word = readWord(Reflect.get(value, "word"), rule, tally, raise);
|
|
474
|
+
const synonyms = readWords(
|
|
475
|
+
Reflect.get(value, "synonyms"),
|
|
476
|
+
rule,
|
|
477
|
+
"synonyms",
|
|
478
|
+
MIN_ONE_WAY_SYNONYMS,
|
|
479
|
+
MAX_SYNONYM_RULE_WORDS - 1,
|
|
480
|
+
tally,
|
|
481
|
+
raise
|
|
482
|
+
);
|
|
483
|
+
return { word, synonyms };
|
|
484
|
+
}
|
|
485
|
+
raise(`Synonym rule ${rule} must hold either "words" alone, or "word" and "synonyms"`, { rule });
|
|
486
|
+
}
|
|
487
|
+
function parseSynonymRules(value, raise) {
|
|
488
|
+
if (!Array.isArray(value)) raise("A synonym set must be a list of rules", {});
|
|
489
|
+
if (value.length > MAX_SYNONYM_RULES) {
|
|
490
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_RULES} rules`, { maxRules: MAX_SYNONYM_RULES });
|
|
491
|
+
}
|
|
492
|
+
const tally = { words: 0, bytes: 0 };
|
|
493
|
+
const rules = new Array(value.length);
|
|
494
|
+
for (let rule = 0; rule < value.length; rule++) rules[rule] = readRule(value[rule], rule, tally, raise);
|
|
495
|
+
return rules;
|
|
496
|
+
}
|
|
497
|
+
function encodeSynonymRules(rules) {
|
|
498
|
+
return encode(rules);
|
|
499
|
+
}
|
|
500
|
+
function decodeSynonymRules(bytes, raise) {
|
|
501
|
+
let decoded;
|
|
502
|
+
try {
|
|
503
|
+
decoded = decode(bytes);
|
|
504
|
+
} catch {
|
|
505
|
+
raise("A stored synonym set is not valid MessagePack", {});
|
|
506
|
+
}
|
|
507
|
+
return parseSynonymRules(decoded, raise);
|
|
508
|
+
}
|
|
387
509
|
|
|
388
510
|
// src/core/synonyms/prepare.ts
|
|
389
511
|
function invalidIn(source) {
|
|
@@ -467,4 +589,4 @@ var synonymSearchModule = {
|
|
|
467
589
|
applyLoaded: applyLoadedSynonyms
|
|
468
590
|
};
|
|
469
591
|
|
|
470
|
-
export { configOfNamedSettings, configUnderAnalysis, getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, targetOfPending, withOpenIndex };
|
|
592
|
+
export { configOfNamedSettings, configUnderAnalysis, decodeSynonymRules, getLanguage, getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerLanguage, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, targetOfPending, withOpenIndex };
|