@delali/narsil 0.3.1-next.145 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-SLGIAD6B.mjs';
|
|
2
|
+
import './chunk-R4OPUZQ5.mjs';
|
|
3
|
+
import './chunk-BRNAVOKP.mjs';
|
|
4
|
+
import './chunk-UFNCLEQM.mjs';
|
|
5
|
+
import './chunk-B6OD63Z3.mjs';
|
|
6
|
+
import './chunk-7VWQLEFV.mjs';
|
|
7
|
+
import './chunk-YJSMEKRE.mjs';
|
|
8
|
+
import './chunk-LQAKO5QV.mjs';
|
|
9
|
+
import './chunk-NNE4VDMM.mjs';
|
|
10
|
+
import './chunk-L4M7QJNP.mjs';
|
|
11
|
+
import './chunk-IO2SMUQD.mjs';
|
|
12
|
+
import './chunk-AM7RSBPF.mjs';
|
|
13
|
+
import './chunk-ZCKNBPQ4.mjs';
|
|
14
|
+
import './chunk-PU55IOG4.mjs';
|
|
15
|
+
import './chunk-TJMSKBEM.mjs';
|
|
16
|
+
import './chunk-DKJTZ625.mjs';
|
|
17
|
+
import './chunk-3CTWSQRW.mjs';
|
|
18
|
+
import './chunk-2J2727V4.mjs';
|
|
19
|
+
import './chunk-KWRW5U65.mjs';
|
|
20
|
+
import './chunk-PPEPTUVV.mjs';
|
|
21
|
+
import './chunk-LVIKKAA6.mjs';
|
|
22
|
+
import './chunk-AAAETEJK.mjs';
|
|
23
|
+
import './chunk-JV6VQY2S.mjs';
|
|
24
|
+
import './chunk-NFGBOI46.mjs';
|
|
25
|
+
import './chunk-YH7W556M.mjs';
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { createBitSet, bitsetIsEmpty, GrowableUint32, bitsetHas, bitsetSet } from './chunk-LVIKKAA6.mjs';
|
|
2
|
+
import { MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-AAAETEJK.mjs';
|
|
3
|
+
import { PHRASE_ROWS_START, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-JV6VQY2S.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
5
|
+
|
|
6
|
+
// src/core/partition/phrase-matching.ts
|
|
7
|
+
var UINT32_BITS = 32;
|
|
8
|
+
var WorkTally = class {
|
|
9
|
+
constructor(meter) {
|
|
10
|
+
this.meter = meter;
|
|
11
|
+
}
|
|
12
|
+
pending = 0;
|
|
13
|
+
add(units) {
|
|
14
|
+
this.pending += units;
|
|
15
|
+
if (this.meter !== void 0 && this.pending >= this.meter.flushUnits) this.flush();
|
|
16
|
+
}
|
|
17
|
+
tickClock(units) {
|
|
18
|
+
this.meter?.clock?.tick(units);
|
|
19
|
+
}
|
|
20
|
+
flush() {
|
|
21
|
+
const units = this.pending;
|
|
22
|
+
this.pending = 0;
|
|
23
|
+
this.meter?.add(units);
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
function placeRowsByDigit(docIds, from, to, shift, starts, tally) {
|
|
27
|
+
const count = from.length;
|
|
28
|
+
const mask = starts.length - 1;
|
|
29
|
+
starts.fill(0);
|
|
30
|
+
for (let row = 0; row < count; row++) starts[docIds[row] >>> shift & mask]++;
|
|
31
|
+
let next = 0;
|
|
32
|
+
for (let digit = 0; digit < starts.length; digit++) {
|
|
33
|
+
const held = starts[digit];
|
|
34
|
+
starts[digit] = next;
|
|
35
|
+
next += held;
|
|
36
|
+
}
|
|
37
|
+
for (let begin = 0; begin < count; begin += PHRASE_SORT_CLOCK_ROWS) {
|
|
38
|
+
const end = Math.min(count, begin + PHRASE_SORT_CLOCK_ROWS);
|
|
39
|
+
for (let at = begin; at < end; at++) {
|
|
40
|
+
const row = from[at];
|
|
41
|
+
to[starts[docIds[row] >>> shift & mask]++] = row;
|
|
42
|
+
}
|
|
43
|
+
tally.tickClock(end - begin);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function sortedRowOrder(docIds, count, tally) {
|
|
47
|
+
let order = new Uint32Array(count);
|
|
48
|
+
for (let row = 0; row < count; row++) order[row] = row;
|
|
49
|
+
let spare = new Uint32Array(count);
|
|
50
|
+
const starts = new Uint32Array(1 << PHRASE_SORT_RADIX_BITS);
|
|
51
|
+
for (let shift = 0; shift < UINT32_BITS; shift += PHRASE_SORT_RADIX_BITS) {
|
|
52
|
+
placeRowsByDigit(docIds, order, spare, shift, starts, tally);
|
|
53
|
+
const placed = spare;
|
|
54
|
+
spare = order;
|
|
55
|
+
order = placed;
|
|
56
|
+
}
|
|
57
|
+
return order;
|
|
58
|
+
}
|
|
59
|
+
function tokenRowsOf(list, tally) {
|
|
60
|
+
const store = list.positionStore ?? null;
|
|
61
|
+
if (store === null) return null;
|
|
62
|
+
const count = list.length;
|
|
63
|
+
const docIds = list.docIds;
|
|
64
|
+
if (list.ordered) {
|
|
65
|
+
return { list, keys: docIds, order: null, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
66
|
+
}
|
|
67
|
+
tally.add(count);
|
|
68
|
+
const order = sortedRowOrder(docIds, count, tally);
|
|
69
|
+
const keys = new Uint32Array(count);
|
|
70
|
+
for (let row = 0; row < count; row++) keys[row] = docIds[order[row]];
|
|
71
|
+
return { list, keys, order, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
72
|
+
}
|
|
73
|
+
function resolvePhrase(index, phrase, sharedRows, tally) {
|
|
74
|
+
const entryRows = [];
|
|
75
|
+
const rows = [];
|
|
76
|
+
for (const token of phrase.tokens) {
|
|
77
|
+
let found = sharedRows.get(token);
|
|
78
|
+
if (found === void 0) {
|
|
79
|
+
const list = index.lookupReadOnce?.(token) ?? index.lookup(token);
|
|
80
|
+
if (list === void 0) return null;
|
|
81
|
+
const built = tokenRowsOf(list, tally);
|
|
82
|
+
if (built === null) return null;
|
|
83
|
+
found = built;
|
|
84
|
+
sharedRows.set(token, found);
|
|
85
|
+
}
|
|
86
|
+
if (!rows.includes(found)) rows.push(found);
|
|
87
|
+
entryRows.push(found);
|
|
88
|
+
}
|
|
89
|
+
let leadEntry = 0;
|
|
90
|
+
for (let entry = 1; entry < entryRows.length; entry++) {
|
|
91
|
+
if (entryRows[entry].count < entryRows[leadEntry].count) leadEntry = entry;
|
|
92
|
+
}
|
|
93
|
+
return { phrase, rows, entryRows, lead: entryRows[leadEntry], leadOffset: phrase.offsets[leadEntry] };
|
|
94
|
+
}
|
|
95
|
+
function firstRowFor(rows, doc) {
|
|
96
|
+
const { keys, count } = rows;
|
|
97
|
+
let low = rows.cursor < count && keys[rows.cursor] <= doc ? rows.cursor : 0;
|
|
98
|
+
let high = low;
|
|
99
|
+
let step = 1;
|
|
100
|
+
while (high < count && keys[high] < doc) {
|
|
101
|
+
low = high + 1;
|
|
102
|
+
high += step;
|
|
103
|
+
step *= 2;
|
|
104
|
+
}
|
|
105
|
+
if (high > count) high = count;
|
|
106
|
+
while (low < high) {
|
|
107
|
+
const middle = low + high >>> 1;
|
|
108
|
+
if (keys[middle] < doc) low = middle + 1;
|
|
109
|
+
else high = middle;
|
|
110
|
+
}
|
|
111
|
+
rows.cursor = low;
|
|
112
|
+
return low;
|
|
113
|
+
}
|
|
114
|
+
function positionRowFor(rows, doc, field) {
|
|
115
|
+
const { keys, order, fields, count } = rows;
|
|
116
|
+
for (let at = firstRowFor(rows, doc); at < count && keys[at] === doc; at++) {
|
|
117
|
+
const row = order === null ? at : order[at];
|
|
118
|
+
if (fields[row] === field) return row;
|
|
119
|
+
}
|
|
120
|
+
return -1;
|
|
121
|
+
}
|
|
122
|
+
function countPhraseStarts(resolved, leadRow, cursors, ends, tally) {
|
|
123
|
+
const { entryRows, lead, leadOffset, phrase } = resolved;
|
|
124
|
+
const offsets = phrase.offsets;
|
|
125
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
126
|
+
const rows = entryRows[entry];
|
|
127
|
+
const start = rows.list.positionStartAt?.(rows.positionRow) ?? 0;
|
|
128
|
+
cursors[entry] = start;
|
|
129
|
+
ends[entry] = start + (rows.list.positionCountAt?.(rows.positionRow) ?? 0);
|
|
130
|
+
}
|
|
131
|
+
const leadStore = lead.store;
|
|
132
|
+
const leadStart = lead.list.positionStartAt?.(leadRow) ?? 0;
|
|
133
|
+
const leadEnd = leadStart + (lead.list.positionCountAt?.(leadRow) ?? 0);
|
|
134
|
+
let starts = 0;
|
|
135
|
+
for (let at = leadStart; at < leadEnd; at++) {
|
|
136
|
+
const anchor = leadStore[at] - leadOffset;
|
|
137
|
+
if (anchor < 0) continue;
|
|
138
|
+
let holdsEveryEntry = true;
|
|
139
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
140
|
+
const store = entryRows[entry].store;
|
|
141
|
+
const wanted = anchor + offsets[entry];
|
|
142
|
+
let cursor = cursors[entry];
|
|
143
|
+
const end = ends[entry];
|
|
144
|
+
let compared = 1;
|
|
145
|
+
while (cursor < end && store[cursor] < wanted) {
|
|
146
|
+
cursor++;
|
|
147
|
+
compared++;
|
|
148
|
+
}
|
|
149
|
+
cursors[entry] = cursor;
|
|
150
|
+
tally.add(compared);
|
|
151
|
+
if (cursor === end) {
|
|
152
|
+
return starts;
|
|
153
|
+
}
|
|
154
|
+
if (store[cursor] !== wanted) {
|
|
155
|
+
holdsEveryEntry = false;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (holdsEveryEntry) starts++;
|
|
160
|
+
}
|
|
161
|
+
return starts;
|
|
162
|
+
}
|
|
163
|
+
function collectPhraseRows(resolved, searchable, allowed, capacity, keepRows, tally) {
|
|
164
|
+
const hits = createBitSet(capacity);
|
|
165
|
+
const collected = {
|
|
166
|
+
phrase: resolved.phrase,
|
|
167
|
+
lists: resolved.entryRows.map((rows2) => rows2.list),
|
|
168
|
+
docIds: new GrowableUint32(PHRASE_ROWS_START),
|
|
169
|
+
fieldIndices: new GrowableUint32(PHRASE_ROWS_START),
|
|
170
|
+
frequencies: new GrowableUint32(PHRASE_ROWS_START)
|
|
171
|
+
};
|
|
172
|
+
const { lead, rows } = resolved;
|
|
173
|
+
const leadList = lead.list;
|
|
174
|
+
const docIds = leadList.docIds;
|
|
175
|
+
const fields = leadList.fieldNameIndices;
|
|
176
|
+
const deleted = leadList.deletedDocs;
|
|
177
|
+
const hasDeleted = deleted.size > 0;
|
|
178
|
+
const cursors = new Int32Array(resolved.entryRows.length);
|
|
179
|
+
const ends = new Int32Array(resolved.entryRows.length);
|
|
180
|
+
for (const each of rows) each.cursor = 0;
|
|
181
|
+
for (let leadRow = 0; leadRow < lead.count; leadRow++) {
|
|
182
|
+
tally.add(1);
|
|
183
|
+
const doc = docIds[leadRow];
|
|
184
|
+
const field = fields[leadRow];
|
|
185
|
+
if (searchable[field] === 0) continue;
|
|
186
|
+
if (hasDeleted && deleted.has(doc)) continue;
|
|
187
|
+
if (allowed !== void 0 && !bitsetHas(allowed, doc)) continue;
|
|
188
|
+
let holdsEveryToken = true;
|
|
189
|
+
for (const each of rows) {
|
|
190
|
+
if (each === lead) {
|
|
191
|
+
each.positionRow = leadRow;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const row = positionRowFor(each, doc, field);
|
|
195
|
+
if (row === -1) {
|
|
196
|
+
holdsEveryToken = false;
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
each.positionRow = row;
|
|
200
|
+
}
|
|
201
|
+
if (!holdsEveryToken) continue;
|
|
202
|
+
const starts = countPhraseStarts(resolved, leadRow, cursors, ends, tally);
|
|
203
|
+
if (starts === 0) continue;
|
|
204
|
+
bitsetSet(hits, doc);
|
|
205
|
+
if (keepRows) {
|
|
206
|
+
collected.docIds.push(doc);
|
|
207
|
+
collected.fieldIndices.push(field);
|
|
208
|
+
collected.frequencies.push(starts);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return { hits, rows: collected };
|
|
212
|
+
}
|
|
213
|
+
function matchPhrases(index, phrases, searchable, filterBitset, capacity, meter, keepRows) {
|
|
214
|
+
const tally = new WorkTally(meter);
|
|
215
|
+
const sharedRows = /* @__PURE__ */ new Map();
|
|
216
|
+
const resolved = [];
|
|
217
|
+
for (const phrase of phrases) {
|
|
218
|
+
const found = resolvePhrase(index, phrase, sharedRows, tally);
|
|
219
|
+
if (found === null) {
|
|
220
|
+
tally.flush();
|
|
221
|
+
return { matched: createBitSet(capacity), rows: [] };
|
|
222
|
+
}
|
|
223
|
+
resolved.push(found);
|
|
224
|
+
}
|
|
225
|
+
resolved.sort((first, second) => first.lead.count - second.lead.count);
|
|
226
|
+
const rows = [];
|
|
227
|
+
let allowed = filterBitset;
|
|
228
|
+
for (const each of resolved) {
|
|
229
|
+
const collected = collectPhraseRows(each, searchable, allowed, capacity, keepRows, tally);
|
|
230
|
+
allowed = collected.hits;
|
|
231
|
+
rows.push(collected.rows);
|
|
232
|
+
if (bitsetIsEmpty(allowed)) break;
|
|
233
|
+
}
|
|
234
|
+
tally.flush();
|
|
235
|
+
return { matched: allowed ?? createBitSet(capacity), rows };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// src/search/phrase-analysis.ts
|
|
239
|
+
function phraseKeyOf(tokens, offsets) {
|
|
240
|
+
const slots = [];
|
|
241
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
242
|
+
const gap = index === 0 ? 0 : offsets[index] - offsets[index - 1] - 1;
|
|
243
|
+
for (let missing = 0; missing < gap; missing++) slots.push("?");
|
|
244
|
+
slots.push(tokens[index]);
|
|
245
|
+
}
|
|
246
|
+
return `"${slots.join(" ")}"`;
|
|
247
|
+
}
|
|
248
|
+
function identityOf(tokens, offsets) {
|
|
249
|
+
return JSON.stringify([tokens, offsets]);
|
|
250
|
+
}
|
|
251
|
+
function analyseQueryPhrases(parts, analyseText) {
|
|
252
|
+
const wordParts = [];
|
|
253
|
+
const phrases = [];
|
|
254
|
+
const seen = /* @__PURE__ */ new Set();
|
|
255
|
+
let keptPhraseCount = 0;
|
|
256
|
+
let largestPhraseSize = 0;
|
|
257
|
+
let largestTokenCount = 0;
|
|
258
|
+
let wordsAfterLastPhrase = "";
|
|
259
|
+
for (let index = 0; index < parts.length; index++) {
|
|
260
|
+
const part = parts[index];
|
|
261
|
+
if (index % 2 === 0) {
|
|
262
|
+
wordParts.push(part);
|
|
263
|
+
wordsAfterLastPhrase = `${wordsAfterLastPhrase} ${part}`;
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
if (part.trim().length === 0) continue;
|
|
267
|
+
const entries = analyseText(part);
|
|
268
|
+
if (entries.length === 0) continue;
|
|
269
|
+
const ordered = [...entries].sort((first, second) => first.position - second.position);
|
|
270
|
+
const firstPosition = ordered[0].position;
|
|
271
|
+
const size = ordered[ordered.length - 1].position - firstPosition + 1;
|
|
272
|
+
keptPhraseCount++;
|
|
273
|
+
wordsAfterLastPhrase = "";
|
|
274
|
+
if (size > largestPhraseSize) largestPhraseSize = size;
|
|
275
|
+
if (ordered.length > largestTokenCount) largestTokenCount = ordered.length;
|
|
276
|
+
const tokens = ordered.map((entry) => entry.token);
|
|
277
|
+
const offsets = ordered.map((entry) => entry.position - firstPosition);
|
|
278
|
+
const identity = identityOf(tokens, offsets);
|
|
279
|
+
if (seen.has(identity)) continue;
|
|
280
|
+
seen.add(identity);
|
|
281
|
+
phrases.push({ key: phraseKeyOf(tokens, offsets), tokens, offsets });
|
|
282
|
+
}
|
|
283
|
+
const endsWithPhrase = keptPhraseCount > 0 && (wordsAfterLastPhrase.trim().length === 0 || analyseText(wordsAfterLastPhrase).length === 0);
|
|
284
|
+
return {
|
|
285
|
+
words: wordParts.join(" "),
|
|
286
|
+
phrases,
|
|
287
|
+
keptPhraseCount,
|
|
288
|
+
largestPhraseSize,
|
|
289
|
+
largestTokenCount,
|
|
290
|
+
endsWithPhrase
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
function withWordsOnly(params, phrases) {
|
|
294
|
+
return { ...params, term: phrases.words, prefix: phrases.endsWithPhrase ? false : params.prefix };
|
|
295
|
+
}
|
|
296
|
+
function refusePhraseUntilRebuilt(indexName, rebuildMode) {
|
|
297
|
+
const remedy = rebuildMode === "manual" ? "This engine leaves the rebuild to its caller, so somebody has to start it with rebuildAnalysis" : "Try the query again once the rebuild finishes";
|
|
298
|
+
throw new NarsilError(
|
|
299
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
300
|
+
`Index "${indexName}" holds array fields whose terms need a rebuild, so it cannot match a phrase yet. ${remedy}`,
|
|
301
|
+
{ indexName, rebuild: rebuildMode }
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
function requirePhrasesWithinLimits(phrases, trackPositions) {
|
|
305
|
+
if (phrases.keptPhraseCount === 0) return;
|
|
306
|
+
if (phrases.keptPhraseCount > MAX_QUERY_PHRASES) {
|
|
307
|
+
throw new NarsilError(
|
|
308
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
309
|
+
`A query takes at most ${MAX_QUERY_PHRASES} phrases, while this query holds ${phrases.keptPhraseCount}`,
|
|
310
|
+
{ limit: MAX_QUERY_PHRASES, phrases: phrases.keptPhraseCount }
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
if (phrases.largestPhraseSize > MAX_PHRASE_WORDS || phrases.largestTokenCount > MAX_PHRASE_WORDS) {
|
|
314
|
+
throw new NarsilError(
|
|
315
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
316
|
+
`A phrase takes at most ${MAX_PHRASE_WORDS} words, counting the stop words between its first and last word, while this query holds one of ${Math.max(phrases.largestPhraseSize, phrases.largestTokenCount)}`,
|
|
317
|
+
{ limit: MAX_PHRASE_WORDS, words: Math.max(phrases.largestPhraseSize, phrases.largestTokenCount) }
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
if (!trackPositions) {
|
|
321
|
+
throw new NarsilError(
|
|
322
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
323
|
+
"This index stores no word positions because its trackPositions setting is off, so it cannot match a phrase. Remove the quotes, or create the index with trackPositions on",
|
|
324
|
+
{ trackPositions: false }
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export { analyseQueryPhrases, matchPhrases, refusePhraseUntilRebuilt, requirePhrasesWithinLimits, withWordsOnly };
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { createRequestId } from './chunk-NNE4VDMM.mjs';
|
|
2
|
+
import { resolveWorkerCount, SHUTDOWN_TIMEOUT_MS, fnv1a } from './chunk-L4M7QJNP.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
4
|
+
|
|
5
|
+
// src/workers/pool-memory.ts
|
|
6
|
+
function toFinite(value) {
|
|
7
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
8
|
+
}
|
|
9
|
+
function toHeapLimit(value) {
|
|
10
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
11
|
+
}
|
|
12
|
+
async function memoryStatsOf(slots) {
|
|
13
|
+
const indices = [];
|
|
14
|
+
const pending = [];
|
|
15
|
+
for (const [slotIndex, slot] of slots) {
|
|
16
|
+
indices.push(slotIndex);
|
|
17
|
+
pending.push(
|
|
18
|
+
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
const reports = await Promise.all(pending);
|
|
22
|
+
const stats = [];
|
|
23
|
+
for (let n = 0; n < indices.length; n++) {
|
|
24
|
+
const report = reports[n];
|
|
25
|
+
stats.push({
|
|
26
|
+
workerId: indices[n],
|
|
27
|
+
heapUsed: toFinite(report?.heapUsed),
|
|
28
|
+
heapTotal: toFinite(report?.heapTotal),
|
|
29
|
+
heapLimit: toHeapLimit(report?.heapLimit),
|
|
30
|
+
external: toFinite(report?.external)
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return stats;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// src/workers/pool.ts
|
|
37
|
+
function createWorkerPool(config) {
|
|
38
|
+
const workerCount = resolveWorkerCount(config.count);
|
|
39
|
+
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
40
|
+
const workers = /* @__PURE__ */ new Map();
|
|
41
|
+
const deadSlots = /* @__PURE__ */ new Set();
|
|
42
|
+
const goneSlots = /* @__PURE__ */ new Set();
|
|
43
|
+
const threadsStillRunning = /* @__PURE__ */ new Set();
|
|
44
|
+
const waitingForEveryThread = [];
|
|
45
|
+
const indexAssignment = /* @__PURE__ */ new Map();
|
|
46
|
+
let isShutdown = false;
|
|
47
|
+
function handleWorkerDeath(slotIndex, error) {
|
|
48
|
+
if (isShutdown) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const slot = workers.get(slotIndex);
|
|
52
|
+
if (slot !== void 0 && !slot.serving) {
|
|
53
|
+
workers.delete(slotIndex);
|
|
54
|
+
goneSlots.delete(slotIndex);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (deadSlots.has(slotIndex)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
deadSlots.add(slotIndex);
|
|
61
|
+
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
62
|
+
workers.delete(slotIndex);
|
|
63
|
+
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
64
|
+
}
|
|
65
|
+
function handleWorkerGone(slotIndex) {
|
|
66
|
+
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
goneSlots.add(slotIndex);
|
|
70
|
+
config.onWorkerGone?.(slotIndex);
|
|
71
|
+
}
|
|
72
|
+
function spawnSlot(slotIndex, serving) {
|
|
73
|
+
const slot = {
|
|
74
|
+
executor: config.workerFactory(
|
|
75
|
+
slotIndex,
|
|
76
|
+
(error) => handleWorkerDeath(slotIndex, error),
|
|
77
|
+
() => {
|
|
78
|
+
noteThreadGone(slot.executor);
|
|
79
|
+
handleWorkerGone(slotIndex);
|
|
80
|
+
}
|
|
81
|
+
),
|
|
82
|
+
indexes: /* @__PURE__ */ new Set(),
|
|
83
|
+
inFlight: 0,
|
|
84
|
+
serving
|
|
85
|
+
};
|
|
86
|
+
threadsStillRunning.add(slot.executor);
|
|
87
|
+
workers.set(slotIndex, slot);
|
|
88
|
+
return slot;
|
|
89
|
+
}
|
|
90
|
+
function noteThreadGone(executor) {
|
|
91
|
+
threadsStillRunning.delete(executor);
|
|
92
|
+
if (threadsStillRunning.size > 0) return;
|
|
93
|
+
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
94
|
+
}
|
|
95
|
+
function whenEveryThreadIsGone() {
|
|
96
|
+
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
97
|
+
return new Promise((resolve) => {
|
|
98
|
+
waitingForEveryThread.push(resolve);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
function shutDownUntilTheThreadIsGone(executor) {
|
|
102
|
+
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
103
|
+
}
|
|
104
|
+
function ensureWorker(slotIndex) {
|
|
105
|
+
const slot = workers.get(slotIndex);
|
|
106
|
+
if (slot !== void 0) return slot;
|
|
107
|
+
if (deadSlots.has(slotIndex)) return void 0;
|
|
108
|
+
return spawnSlot(slotIndex, true);
|
|
109
|
+
}
|
|
110
|
+
function deadWorkerIds() {
|
|
111
|
+
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
112
|
+
}
|
|
113
|
+
function spawnReplacement(slotIndex) {
|
|
114
|
+
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
115
|
+
const slot = spawnSlot(slotIndex, false);
|
|
116
|
+
return {
|
|
117
|
+
workerId: slotIndex,
|
|
118
|
+
executor: slot.executor,
|
|
119
|
+
hold(indexName) {
|
|
120
|
+
slot.indexes.add(indexName);
|
|
121
|
+
},
|
|
122
|
+
admit() {
|
|
123
|
+
if (workers.get(slotIndex) !== slot) return;
|
|
124
|
+
deadSlots.delete(slotIndex);
|
|
125
|
+
goneSlots.delete(slotIndex);
|
|
126
|
+
slot.serving = true;
|
|
127
|
+
},
|
|
128
|
+
abandon() {
|
|
129
|
+
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
130
|
+
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
function isLongQueryWorker(workerId) {
|
|
135
|
+
return workerId >= queryWorkerCount;
|
|
136
|
+
}
|
|
137
|
+
function assignSlot(indexName) {
|
|
138
|
+
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
139
|
+
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
140
|
+
const candidate = (preferred + probe) % queryWorkerCount;
|
|
141
|
+
if (!deadSlots.has(candidate)) {
|
|
142
|
+
return candidate;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
146
|
+
}
|
|
147
|
+
function addIndex(indexName) {
|
|
148
|
+
if (isShutdown) {
|
|
149
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
150
|
+
}
|
|
151
|
+
if (indexAssignment.has(indexName)) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const slotIndex = assignSlot(indexName);
|
|
155
|
+
const slot = ensureWorker(slotIndex);
|
|
156
|
+
if (!slot) {
|
|
157
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
158
|
+
}
|
|
159
|
+
slot.indexes.add(indexName);
|
|
160
|
+
indexAssignment.set(indexName, slotIndex);
|
|
161
|
+
}
|
|
162
|
+
function removeIndex(indexName) {
|
|
163
|
+
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
164
|
+
indexAssignment.delete(indexName);
|
|
165
|
+
}
|
|
166
|
+
function getExecutor(indexName) {
|
|
167
|
+
if (isShutdown) {
|
|
168
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
169
|
+
}
|
|
170
|
+
const slotIndex = indexAssignment.get(indexName);
|
|
171
|
+
if (slotIndex === void 0) {
|
|
172
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
173
|
+
}
|
|
174
|
+
const slot = workers.get(slotIndex);
|
|
175
|
+
if (slot?.serving) {
|
|
176
|
+
return slot.executor;
|
|
177
|
+
}
|
|
178
|
+
for (const survivor of workers.values()) {
|
|
179
|
+
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
180
|
+
return survivor.executor;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
184
|
+
}
|
|
185
|
+
async function shutdown() {
|
|
186
|
+
if (isShutdown) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
isShutdown = true;
|
|
190
|
+
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
191
|
+
let timeoutId;
|
|
192
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
193
|
+
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
194
|
+
});
|
|
195
|
+
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
196
|
+
() => clearTimeout(timeoutId)
|
|
197
|
+
);
|
|
198
|
+
});
|
|
199
|
+
await Promise.allSettled(shutdownPromises);
|
|
200
|
+
indexAssignment.clear();
|
|
201
|
+
}
|
|
202
|
+
function getAllExecutors() {
|
|
203
|
+
return [...workers.values()].map((slot) => slot.executor);
|
|
204
|
+
}
|
|
205
|
+
function executorEntries() {
|
|
206
|
+
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
207
|
+
}
|
|
208
|
+
function executorsHolding(indexName) {
|
|
209
|
+
const holders = [];
|
|
210
|
+
for (const slot of workers.values()) {
|
|
211
|
+
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
212
|
+
}
|
|
213
|
+
return holders;
|
|
214
|
+
}
|
|
215
|
+
function lease(workerId, slot) {
|
|
216
|
+
slot.inFlight += 1;
|
|
217
|
+
let released = false;
|
|
218
|
+
return {
|
|
219
|
+
workerId,
|
|
220
|
+
executor: slot.executor,
|
|
221
|
+
release() {
|
|
222
|
+
if (released) return;
|
|
223
|
+
released = true;
|
|
224
|
+
slot.inFlight -= 1;
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function leaseLeastBusy(indexName) {
|
|
229
|
+
let chosenId = -1;
|
|
230
|
+
let chosen = null;
|
|
231
|
+
for (const [workerId, slot] of workers) {
|
|
232
|
+
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
233
|
+
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
234
|
+
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
235
|
+
chosenId = workerId;
|
|
236
|
+
chosen = slot;
|
|
237
|
+
}
|
|
238
|
+
if (chosen.inFlight === 0) break;
|
|
239
|
+
}
|
|
240
|
+
return chosen === null ? null : lease(chosenId, chosen);
|
|
241
|
+
}
|
|
242
|
+
function queriesInFlight() {
|
|
243
|
+
let total = 0;
|
|
244
|
+
for (const [workerId, slot] of workers) {
|
|
245
|
+
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
246
|
+
}
|
|
247
|
+
return total;
|
|
248
|
+
}
|
|
249
|
+
function leaseIdle(limit) {
|
|
250
|
+
const leases = [];
|
|
251
|
+
for (const [workerId, slot] of workers) {
|
|
252
|
+
if (leases.length >= limit) break;
|
|
253
|
+
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
254
|
+
}
|
|
255
|
+
return leases;
|
|
256
|
+
}
|
|
257
|
+
function addIndexToAll(indexName) {
|
|
258
|
+
if (isShutdown) {
|
|
259
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
260
|
+
}
|
|
261
|
+
for (let i = 0; i < workerCount; i++) {
|
|
262
|
+
ensureWorker(i)?.indexes.add(indexName);
|
|
263
|
+
}
|
|
264
|
+
indexAssignment.set(indexName, assignSlot(indexName));
|
|
265
|
+
}
|
|
266
|
+
function spawnAll() {
|
|
267
|
+
if (isShutdown) {
|
|
268
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
269
|
+
}
|
|
270
|
+
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
271
|
+
}
|
|
272
|
+
return {
|
|
273
|
+
getExecutor,
|
|
274
|
+
getAllExecutors,
|
|
275
|
+
executorEntries,
|
|
276
|
+
executorsHolding,
|
|
277
|
+
leaseLeastBusy,
|
|
278
|
+
leaseIdle,
|
|
279
|
+
isLongQueryWorker,
|
|
280
|
+
queriesInFlight,
|
|
281
|
+
spawnAll,
|
|
282
|
+
deadWorkerIds,
|
|
283
|
+
spawnReplacement,
|
|
284
|
+
get workerCount() {
|
|
285
|
+
return workerCount;
|
|
286
|
+
},
|
|
287
|
+
addIndex,
|
|
288
|
+
addIndexToAll,
|
|
289
|
+
removeIndex,
|
|
290
|
+
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
291
|
+
shutdown,
|
|
292
|
+
whenEveryThreadIsGone
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export { createWorkerPool };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// src/engine/orchestration/release.ts
|
|
2
|
+
async function releaseIndexCopies(state, indexName, steps) {
|
|
3
|
+
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
4
|
+
const loading = state.copyTransitions.get(indexName);
|
|
5
|
+
if (loading !== void 0) await loading.done.catch(() => void 0);
|
|
6
|
+
await steps.awaitReplicationIdle(state, indexName);
|
|
7
|
+
steps.cancelIdleMerge(state, indexName);
|
|
8
|
+
const reloads = wasScaledOut || loading !== void 0 && loading.kind !== "drop";
|
|
9
|
+
const pool = state.workerPool;
|
|
10
|
+
if (pool === null) return reloads;
|
|
11
|
+
pool.removeIndex(indexName);
|
|
12
|
+
state.segmentLedger.delete(indexName);
|
|
13
|
+
state.unbroadcastSegments.delete(indexName);
|
|
14
|
+
await Promise.all(
|
|
15
|
+
pool.executorEntries().map(
|
|
16
|
+
({ executor }) => executor.execute({ type: "dropIndex", indexName, requestId: `restore-release-${indexName}` }).catch((error) => {
|
|
17
|
+
if (!steps.heldNowhere(error)) throw error;
|
|
18
|
+
})
|
|
19
|
+
)
|
|
20
|
+
);
|
|
21
|
+
return reloads;
|
|
22
|
+
}
|
|
23
|
+
async function endCopyRelease(state, indexName, reloads, steps) {
|
|
24
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
25
|
+
if (hold === void 0) return steps.resyncIndex(indexName, reloads);
|
|
26
|
+
hold.holders -= 1;
|
|
27
|
+
hold.reloads ||= reloads;
|
|
28
|
+
if (hold.holders > 0) return;
|
|
29
|
+
state.copyReleaseHolds.delete(indexName);
|
|
30
|
+
await steps.resyncIndex(indexName, hold.reloads);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export { endCopyRelease, releaseIndexCopies };
|