@delali/narsil 0.3.1-next.145 → 0.3.1-next.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createBitSet, bitsetIsEmpty, GrowableUint32, PHRASE_ROWS_START, bitsetHas, bitsetSet, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import {
|
|
2
|
+
import { MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-WXLBZJMT.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
3
4
|
|
|
4
5
|
// src/core/partition/phrase-matching.ts
|
|
5
6
|
var UINT32_BITS = 32;
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { createRequestId, fnv1a } from './chunk-MPK76GVS.mjs';
|
|
2
|
+
import { resolveWorkerCount, SHUTDOWN_TIMEOUT_MS } from './chunk-TY7L6OXO.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
4
|
+
|
|
5
|
+
// src/workers/pool-memory.ts
|
|
6
|
+
function toFinite(value) {
|
|
7
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
8
|
+
}
|
|
9
|
+
function toHeapLimit(value) {
|
|
10
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
11
|
+
}
|
|
12
|
+
async function memoryStatsOf(slots) {
|
|
13
|
+
const indices = [];
|
|
14
|
+
const pending = [];
|
|
15
|
+
for (const [slotIndex, slot] of slots) {
|
|
16
|
+
indices.push(slotIndex);
|
|
17
|
+
pending.push(
|
|
18
|
+
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
const reports = await Promise.all(pending);
|
|
22
|
+
const stats = [];
|
|
23
|
+
for (let n = 0; n < indices.length; n++) {
|
|
24
|
+
const report = reports[n];
|
|
25
|
+
stats.push({
|
|
26
|
+
workerId: indices[n],
|
|
27
|
+
heapUsed: toFinite(report?.heapUsed),
|
|
28
|
+
heapTotal: toFinite(report?.heapTotal),
|
|
29
|
+
heapLimit: toHeapLimit(report?.heapLimit),
|
|
30
|
+
external: toFinite(report?.external)
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return stats;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// src/workers/pool.ts
|
|
37
|
+
function createWorkerPool(config) {
|
|
38
|
+
const workerCount = resolveWorkerCount(config.count);
|
|
39
|
+
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
40
|
+
const workers = /* @__PURE__ */ new Map();
|
|
41
|
+
const deadSlots = /* @__PURE__ */ new Set();
|
|
42
|
+
const goneSlots = /* @__PURE__ */ new Set();
|
|
43
|
+
const threadsStillRunning = /* @__PURE__ */ new Set();
|
|
44
|
+
const waitingForEveryThread = [];
|
|
45
|
+
const indexAssignment = /* @__PURE__ */ new Map();
|
|
46
|
+
let isShutdown = false;
|
|
47
|
+
function handleWorkerDeath(slotIndex, error) {
|
|
48
|
+
if (isShutdown) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const slot = workers.get(slotIndex);
|
|
52
|
+
if (slot !== void 0 && !slot.serving) {
|
|
53
|
+
workers.delete(slotIndex);
|
|
54
|
+
goneSlots.delete(slotIndex);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (deadSlots.has(slotIndex)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
deadSlots.add(slotIndex);
|
|
61
|
+
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
62
|
+
workers.delete(slotIndex);
|
|
63
|
+
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
64
|
+
}
|
|
65
|
+
function handleWorkerGone(slotIndex) {
|
|
66
|
+
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
goneSlots.add(slotIndex);
|
|
70
|
+
config.onWorkerGone?.(slotIndex);
|
|
71
|
+
}
|
|
72
|
+
function spawnSlot(slotIndex, serving) {
|
|
73
|
+
const slot = {
|
|
74
|
+
executor: config.workerFactory(
|
|
75
|
+
slotIndex,
|
|
76
|
+
(error) => handleWorkerDeath(slotIndex, error),
|
|
77
|
+
() => {
|
|
78
|
+
noteThreadGone(slot.executor);
|
|
79
|
+
handleWorkerGone(slotIndex);
|
|
80
|
+
}
|
|
81
|
+
),
|
|
82
|
+
indexes: /* @__PURE__ */ new Set(),
|
|
83
|
+
inFlight: 0,
|
|
84
|
+
serving
|
|
85
|
+
};
|
|
86
|
+
threadsStillRunning.add(slot.executor);
|
|
87
|
+
workers.set(slotIndex, slot);
|
|
88
|
+
return slot;
|
|
89
|
+
}
|
|
90
|
+
function noteThreadGone(executor) {
|
|
91
|
+
threadsStillRunning.delete(executor);
|
|
92
|
+
if (threadsStillRunning.size > 0) return;
|
|
93
|
+
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
94
|
+
}
|
|
95
|
+
function whenEveryThreadIsGone() {
|
|
96
|
+
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
97
|
+
return new Promise((resolve) => {
|
|
98
|
+
waitingForEveryThread.push(resolve);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
function shutDownUntilTheThreadIsGone(executor) {
|
|
102
|
+
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
103
|
+
}
|
|
104
|
+
function ensureWorker(slotIndex) {
|
|
105
|
+
const slot = workers.get(slotIndex);
|
|
106
|
+
if (slot !== void 0) return slot;
|
|
107
|
+
if (deadSlots.has(slotIndex)) return void 0;
|
|
108
|
+
return spawnSlot(slotIndex, true);
|
|
109
|
+
}
|
|
110
|
+
function deadWorkerIds() {
|
|
111
|
+
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
112
|
+
}
|
|
113
|
+
function spawnReplacement(slotIndex) {
|
|
114
|
+
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
115
|
+
const slot = spawnSlot(slotIndex, false);
|
|
116
|
+
return {
|
|
117
|
+
workerId: slotIndex,
|
|
118
|
+
executor: slot.executor,
|
|
119
|
+
hold(indexName) {
|
|
120
|
+
slot.indexes.add(indexName);
|
|
121
|
+
},
|
|
122
|
+
admit() {
|
|
123
|
+
if (workers.get(slotIndex) !== slot) return;
|
|
124
|
+
deadSlots.delete(slotIndex);
|
|
125
|
+
goneSlots.delete(slotIndex);
|
|
126
|
+
slot.serving = true;
|
|
127
|
+
},
|
|
128
|
+
abandon() {
|
|
129
|
+
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
130
|
+
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
function isLongQueryWorker(workerId) {
|
|
135
|
+
return workerId >= queryWorkerCount;
|
|
136
|
+
}
|
|
137
|
+
function assignSlot(indexName) {
|
|
138
|
+
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
139
|
+
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
140
|
+
const candidate = (preferred + probe) % queryWorkerCount;
|
|
141
|
+
if (!deadSlots.has(candidate)) {
|
|
142
|
+
return candidate;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
146
|
+
}
|
|
147
|
+
function addIndex(indexName) {
|
|
148
|
+
if (isShutdown) {
|
|
149
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
150
|
+
}
|
|
151
|
+
if (indexAssignment.has(indexName)) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const slotIndex = assignSlot(indexName);
|
|
155
|
+
const slot = ensureWorker(slotIndex);
|
|
156
|
+
if (!slot) {
|
|
157
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
158
|
+
}
|
|
159
|
+
slot.indexes.add(indexName);
|
|
160
|
+
indexAssignment.set(indexName, slotIndex);
|
|
161
|
+
}
|
|
162
|
+
function removeIndex(indexName) {
|
|
163
|
+
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
164
|
+
indexAssignment.delete(indexName);
|
|
165
|
+
}
|
|
166
|
+
function getExecutor(indexName) {
|
|
167
|
+
if (isShutdown) {
|
|
168
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
169
|
+
}
|
|
170
|
+
const slotIndex = indexAssignment.get(indexName);
|
|
171
|
+
if (slotIndex === void 0) {
|
|
172
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
173
|
+
}
|
|
174
|
+
const slot = workers.get(slotIndex);
|
|
175
|
+
if (slot?.serving) {
|
|
176
|
+
return slot.executor;
|
|
177
|
+
}
|
|
178
|
+
for (const survivor of workers.values()) {
|
|
179
|
+
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
180
|
+
return survivor.executor;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
184
|
+
}
|
|
185
|
+
async function shutdown() {
|
|
186
|
+
if (isShutdown) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
isShutdown = true;
|
|
190
|
+
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
191
|
+
let timeoutId;
|
|
192
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
193
|
+
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
194
|
+
});
|
|
195
|
+
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
196
|
+
() => clearTimeout(timeoutId)
|
|
197
|
+
);
|
|
198
|
+
});
|
|
199
|
+
await Promise.allSettled(shutdownPromises);
|
|
200
|
+
indexAssignment.clear();
|
|
201
|
+
}
|
|
202
|
+
function getAllExecutors() {
|
|
203
|
+
return [...workers.values()].map((slot) => slot.executor);
|
|
204
|
+
}
|
|
205
|
+
function executorEntries() {
|
|
206
|
+
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
207
|
+
}
|
|
208
|
+
function executorsHolding(indexName) {
|
|
209
|
+
const holders = [];
|
|
210
|
+
for (const slot of workers.values()) {
|
|
211
|
+
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
212
|
+
}
|
|
213
|
+
return holders;
|
|
214
|
+
}
|
|
215
|
+
function lease(workerId, slot) {
|
|
216
|
+
slot.inFlight += 1;
|
|
217
|
+
let released = false;
|
|
218
|
+
return {
|
|
219
|
+
workerId,
|
|
220
|
+
executor: slot.executor,
|
|
221
|
+
release() {
|
|
222
|
+
if (released) return;
|
|
223
|
+
released = true;
|
|
224
|
+
slot.inFlight -= 1;
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function leaseLeastBusy(indexName) {
|
|
229
|
+
let chosenId = -1;
|
|
230
|
+
let chosen = null;
|
|
231
|
+
for (const [workerId, slot] of workers) {
|
|
232
|
+
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
233
|
+
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
234
|
+
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
235
|
+
chosenId = workerId;
|
|
236
|
+
chosen = slot;
|
|
237
|
+
}
|
|
238
|
+
if (chosen.inFlight === 0) break;
|
|
239
|
+
}
|
|
240
|
+
return chosen === null ? null : lease(chosenId, chosen);
|
|
241
|
+
}
|
|
242
|
+
function queriesInFlight() {
|
|
243
|
+
let total = 0;
|
|
244
|
+
for (const [workerId, slot] of workers) {
|
|
245
|
+
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
246
|
+
}
|
|
247
|
+
return total;
|
|
248
|
+
}
|
|
249
|
+
function leaseIdle(limit) {
|
|
250
|
+
const leases = [];
|
|
251
|
+
for (const [workerId, slot] of workers) {
|
|
252
|
+
if (leases.length >= limit) break;
|
|
253
|
+
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
254
|
+
}
|
|
255
|
+
return leases;
|
|
256
|
+
}
|
|
257
|
+
function addIndexToAll(indexName) {
|
|
258
|
+
if (isShutdown) {
|
|
259
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
260
|
+
}
|
|
261
|
+
for (let i = 0; i < workerCount; i++) {
|
|
262
|
+
ensureWorker(i)?.indexes.add(indexName);
|
|
263
|
+
}
|
|
264
|
+
indexAssignment.set(indexName, assignSlot(indexName));
|
|
265
|
+
}
|
|
266
|
+
function spawnAll() {
|
|
267
|
+
if (isShutdown) {
|
|
268
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
269
|
+
}
|
|
270
|
+
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
271
|
+
}
|
|
272
|
+
return {
|
|
273
|
+
getExecutor,
|
|
274
|
+
getAllExecutors,
|
|
275
|
+
executorEntries,
|
|
276
|
+
executorsHolding,
|
|
277
|
+
leaseLeastBusy,
|
|
278
|
+
leaseIdle,
|
|
279
|
+
isLongQueryWorker,
|
|
280
|
+
queriesInFlight,
|
|
281
|
+
spawnAll,
|
|
282
|
+
deadWorkerIds,
|
|
283
|
+
spawnReplacement,
|
|
284
|
+
get workerCount() {
|
|
285
|
+
return workerCount;
|
|
286
|
+
},
|
|
287
|
+
addIndex,
|
|
288
|
+
addIndexToAll,
|
|
289
|
+
removeIndex,
|
|
290
|
+
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
291
|
+
shutdown,
|
|
292
|
+
whenEveryThreadIsGone
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export { createWorkerPool };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abandonChangeOf } from './chunk-
|
|
1
|
+
import { abandonChangeOf } from './chunk-4DYLCICG.mjs';
|
|
2
2
|
|
|
3
3
|
// src/narsil/restore-queue.ts
|
|
4
4
|
function restoreDepsOf(core, host) {
|
|
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
24
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
25
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
26
26
|
const restored = previous.then(async () => {
|
|
27
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-DQ6A4PQP.mjs');
|
|
28
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
29
29
|
});
|
|
30
30
|
const settled = restored.then(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
-
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-
|
|
3
|
-
import {
|
|
2
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-4RD3AFSB.mjs';
|
|
3
|
+
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-WXLBZJMT.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
4
5
|
|
|
5
6
|
// src/search/hybrid-depth.ts
|
|
6
7
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,18 +1,22 @@
|
|
|
1
|
-
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
2
|
-
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-
|
|
3
|
-
import
|
|
4
|
-
import './chunk-
|
|
5
|
-
import
|
|
6
|
-
import './chunk-
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
1
|
+
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-D6UFGBF5.mjs';
|
|
2
|
+
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-NVEJJP4R.mjs';
|
|
3
|
+
import './chunk-JNOAKDUM.mjs';
|
|
4
|
+
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-W643QV3L.mjs';
|
|
5
|
+
import './chunk-QMNZFFD3.mjs';
|
|
6
|
+
import { vectorsPerFile } from './chunk-BRWY3JCN.mjs';
|
|
7
|
+
import './chunk-ODQP7P3H.mjs';
|
|
8
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-KAEVGLYN.mjs';
|
|
9
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-SRL47USS.mjs';
|
|
10
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-BVX2MEHA.mjs';
|
|
11
|
+
import './chunk-MPK76GVS.mjs';
|
|
10
12
|
import './chunk-JPTQZWJQ.mjs';
|
|
11
13
|
import './chunk-26A5JFT2.mjs';
|
|
12
14
|
import './chunk-OIZS5KIX.mjs';
|
|
13
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
14
|
-
import './chunk-
|
|
15
|
-
import
|
|
15
|
+
import { MAX_FILE_BYTES } from './chunk-IAXZAVYL.mjs';
|
|
16
|
+
import './chunk-TY7L6OXO.mjs';
|
|
17
|
+
import './chunk-4RD3AFSB.mjs';
|
|
18
|
+
import './chunk-WXLBZJMT.mjs';
|
|
19
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
16
20
|
import { decode } from '@msgpack/msgpack';
|
|
17
21
|
|
|
18
22
|
// src/persistence/archive/memory-store.ts
|
|
@@ -557,7 +561,7 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
557
561
|
for (const ref of partition.segments) {
|
|
558
562
|
const { directory } = store;
|
|
559
563
|
if (directory !== void 0) {
|
|
560
|
-
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-
|
|
564
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-CEKA7MQH.mjs');
|
|
561
565
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
562
566
|
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
563
567
|
continue;
|
|
@@ -925,7 +929,7 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
925
929
|
if (manifest === null) missingPart("manifest");
|
|
926
930
|
const { directory } = staging;
|
|
927
931
|
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
928
|
-
const { bytesThatSegmentsHoldInMemory } = await import('./module-
|
|
932
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-CEKA7MQH.mjs');
|
|
929
933
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
930
934
|
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
931
935
|
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import { encodeExactJson, encodeDocumentJson } from './chunk-
|
|
4
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
7
|
-
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2JABMB4S.mjs';
|
|
2
|
+
import './chunk-NVEJJP4R.mjs';
|
|
3
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-6OWX3OP2.mjs';
|
|
4
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-QMNZFFD3.mjs';
|
|
5
|
+
import './chunk-5V74FFZE.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-KAEVGLYN.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-BVX2MEHA.mjs';
|
|
8
|
+
import './chunk-MPK76GVS.mjs';
|
|
8
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
10
|
import './chunk-26A5JFT2.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import
|
|
12
|
-
import './chunk-
|
|
11
|
+
import './chunk-IAXZAVYL.mjs';
|
|
12
|
+
import './chunk-TY7L6OXO.mjs';
|
|
13
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-4RD3AFSB.mjs';
|
|
14
|
+
import './chunk-WXLBZJMT.mjs';
|
|
15
|
+
import './chunk-PIS7ZAP7.mjs';
|
|
13
16
|
import { decode } from '@msgpack/msgpack';
|
|
14
17
|
|
|
15
18
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
@@ -91,7 +94,7 @@ async function wholeFileVectorReads(checkpoint) {
|
|
|
91
94
|
};
|
|
92
95
|
}
|
|
93
96
|
async function* pagedDocumentLines(checkpoint) {
|
|
94
|
-
const { openFileVectorReads, pagedPartitionLines } = await import('./module-
|
|
97
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-CEKA7MQH.mjs');
|
|
95
98
|
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
96
99
|
try {
|
|
97
100
|
for (const partition of checkpoint.manifest.partitions) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { E as ErrorDetail, j as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CfhH1IWQ.js';
|
|
3
|
+
import { E as ErrorDetail, j as TaskStore } from './tasks-C4ElXr4P.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-65WCV5JO.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
5
|
import { decode, encode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-
|
|
1
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-ZEKDO7YX.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-AWNHDBKV.mjs';
|
|
3
3
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
4
|
import { MAX_DOC_ID_LENGTH } from './chunk-EEBCDP36.mjs';
|
|
5
|
+
import { turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn } from './chunk-LQG5G2LY.mjs';
|
|
5
6
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
7
8
|
|
|
8
9
|
// src/vector/vector-index/constants.ts
|
|
9
10
|
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
@@ -12,6 +13,10 @@ var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
|
12
13
|
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
13
14
|
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
14
15
|
var BUILD_CHUNK_SIZE = 100;
|
|
16
|
+
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
17
|
+
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
18
|
+
var SEARCH_REFILL_RESERVE = 64;
|
|
19
|
+
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
15
20
|
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
16
21
|
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
17
22
|
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
@@ -154,6 +159,7 @@ function graphNeedsRebuild(state) {
|
|
|
154
159
|
return removed / held > REBUILD_REMOVED_RATIO;
|
|
155
160
|
}
|
|
156
161
|
function yieldBuildTurn(state) {
|
|
162
|
+
if (turnsAreBudgeted) return backgroundTurnIsSpent() ? yieldBackgroundTurn() : Promise.resolve();
|
|
157
163
|
return new Promise((resolve) => {
|
|
158
164
|
const takeTurn = () => {
|
|
159
165
|
clearTimeout(timer);
|
|
@@ -671,4 +677,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
671
677
|
state.savedSignature = plan.signature;
|
|
672
678
|
}
|
|
673
679
|
|
|
674
|
-
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
680
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|