@delali/narsil 0.3.1-next.95 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-3U636ADD.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-LULHQXCK.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-J57MZK6L.mjs} +115 -61
- package/dist/browser/{write-ZUBDNLCH.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-7QONURHW.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-54HYC5FT.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-J7SDAOTP.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-Q4UTU5ZE.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-L26LEDHO.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-JFUE6F2Y.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-MASE2LTJ.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-BGt9lI1V.d.ts → engine-BGCEm9E_.d.ts} +50 -20
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-YC6TS4OP.mjs → snapshot-HOGXNO5W.mjs} +138 -83
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-M2OALSA4.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
package/dist/server.mjs
CHANGED
|
@@ -1,27 +1,29 @@
|
|
|
1
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-
|
|
2
|
-
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
3
|
-
import {
|
|
1
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-YA7XLE2R.mjs';
|
|
2
|
+
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-YA7XLE2R.mjs';
|
|
3
|
+
import { rerankOnHost } from './chunk-ZYNAMIYX.mjs';
|
|
4
|
+
import { validateIndexName } from './chunk-2WYMG3OZ.mjs';
|
|
4
5
|
import './chunk-DJHHF2L5.mjs';
|
|
5
6
|
import './chunk-P36MUFEJ.mjs';
|
|
6
|
-
import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-
|
|
7
|
+
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-R7EEKAJE.mjs';
|
|
7
8
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
8
|
-
import './chunk-
|
|
9
|
-
import './chunk-
|
|
9
|
+
import './chunk-JFRNNBOM.mjs';
|
|
10
|
+
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
11
|
+
import './chunk-LXWDQLTW.mjs';
|
|
10
12
|
import './chunk-TGP5M5R7.mjs';
|
|
11
13
|
import './chunk-U2BKYEC7.mjs';
|
|
12
|
-
import './chunk-
|
|
14
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
13
15
|
import './chunk-WQBCBKY3.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
+
import './chunk-LXPIXDUJ.mjs';
|
|
17
|
+
import './chunk-E74YLAIG.mjs';
|
|
16
18
|
import './chunk-KEHK2O6P.mjs';
|
|
17
|
-
import './chunk-
|
|
19
|
+
import './chunk-OOEXLUGR.mjs';
|
|
18
20
|
import './chunk-7X5JST5L.mjs';
|
|
19
21
|
import './chunk-R6A773WK.mjs';
|
|
20
22
|
import './chunk-YTJCGKM4.mjs';
|
|
21
23
|
import { startIdleHeapCleanup } from './chunk-5XNIRLFC.mjs';
|
|
22
24
|
import './chunk-L5ORED7V.mjs';
|
|
23
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-
|
|
24
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
25
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
26
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
25
27
|
import { randomUUID } from 'crypto';
|
|
26
28
|
import { createReadStream } from 'fs';
|
|
27
29
|
import { rm } from 'fs/promises';
|
|
@@ -813,14 +815,14 @@ function createSearchHandlers(deps) {
|
|
|
813
815
|
async function search(ctx) {
|
|
814
816
|
const params = parseJson(ctx);
|
|
815
817
|
if (!params) return;
|
|
816
|
-
const failure = validateQuery(params, limits
|
|
818
|
+
const failure = validateQuery(params, limits);
|
|
817
819
|
if (failure) {
|
|
818
820
|
rejectInvalid(ctx, failure);
|
|
819
821
|
return;
|
|
820
822
|
}
|
|
821
823
|
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
822
824
|
try {
|
|
823
|
-
respondJson(ctx, await engine.query(ctx.params[0], params));
|
|
825
|
+
respondJson(ctx, await engine.query(ctx.params[0], withServerRerankWindow(params, limits)));
|
|
824
826
|
} catch (err) {
|
|
825
827
|
respondError(ctx, err);
|
|
826
828
|
}
|
|
@@ -828,7 +830,7 @@ function createSearchHandlers(deps) {
|
|
|
828
830
|
async function preflight(ctx) {
|
|
829
831
|
const params = parseJson(ctx);
|
|
830
832
|
if (!params) return;
|
|
831
|
-
const failure = validateQuery(params, limits
|
|
833
|
+
const failure = validateQuery(params, limits);
|
|
832
834
|
if (failure) {
|
|
833
835
|
rejectInvalid(ctx, failure);
|
|
834
836
|
return;
|
|
@@ -868,7 +870,13 @@ var LIMITS_UNBOUNDED_AT_ZERO = [
|
|
|
868
870
|
"maxConcurrentRequests",
|
|
869
871
|
"maxConcurrentTasks"
|
|
870
872
|
];
|
|
871
|
-
var COUNTED_LIMITS = [
|
|
873
|
+
var COUNTED_LIMITS = [
|
|
874
|
+
"importBatchSize",
|
|
875
|
+
"maxResultWindow",
|
|
876
|
+
"maxRerankWindow",
|
|
877
|
+
"maxFetchDocuments",
|
|
878
|
+
"maxTaskPageSize"
|
|
879
|
+
];
|
|
872
880
|
function assertServerEngine(engine) {
|
|
873
881
|
const candidate = engine;
|
|
874
882
|
if (typeof candidate !== "object" || candidate === null) {
|
|
@@ -896,10 +904,21 @@ function assertLimit(field, value, minimum) {
|
|
|
896
904
|
{ field, value, minimum }
|
|
897
905
|
);
|
|
898
906
|
}
|
|
899
|
-
function
|
|
907
|
+
function assertRerankWindowCap(value, engineCap) {
|
|
908
|
+
if (value === void 0) return;
|
|
909
|
+
const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
|
|
910
|
+
if (value <= cap) return;
|
|
911
|
+
throw new NarsilError(
|
|
912
|
+
ErrorCodes.CONFIG_INVALID,
|
|
913
|
+
`limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
|
|
914
|
+
{ field: "maxRerankWindow", value, maximum: cap }
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
function assertServerLimits(limits, engineRerankWindow) {
|
|
900
918
|
if (limits === void 0) return;
|
|
901
919
|
for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
|
|
902
920
|
for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
|
|
921
|
+
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
903
922
|
}
|
|
904
923
|
function assertAllowRegex(allowRegex) {
|
|
905
924
|
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
@@ -1064,7 +1083,14 @@ function toRelayedResponse(requestId, captured) {
|
|
|
1064
1083
|
}
|
|
1065
1084
|
|
|
1066
1085
|
// src/server/request-threads/host.ts
|
|
1067
|
-
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1086
|
+
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1087
|
+
"request",
|
|
1088
|
+
"authorize",
|
|
1089
|
+
"embed",
|
|
1090
|
+
"rerank",
|
|
1091
|
+
"stream-credit",
|
|
1092
|
+
"stream-cancel"
|
|
1093
|
+
]);
|
|
1068
1094
|
function isRelayRequest(message) {
|
|
1069
1095
|
if (typeof message !== "object" || message === null) return false;
|
|
1070
1096
|
const { type, requestId } = message;
|
|
@@ -1142,6 +1168,15 @@ async function answerEmbed(core, request) {
|
|
|
1142
1168
|
return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
|
|
1143
1169
|
}
|
|
1144
1170
|
}
|
|
1171
|
+
async function answerRerank(core, request) {
|
|
1172
|
+
try {
|
|
1173
|
+
const scores = await rerankOnHost(core, request.adapter, request.query, request.documents);
|
|
1174
|
+
return { type: "reranked", requestId: request.requestId, scores, error: null };
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
|
|
1177
|
+
return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1145
1180
|
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
1146
1181
|
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
1147
1182
|
}
|
|
@@ -1184,7 +1219,7 @@ async function startRequestThreads(options) {
|
|
|
1184
1219
|
streams.get(message.requestId)?.cancel();
|
|
1185
1220
|
return;
|
|
1186
1221
|
}
|
|
1187
|
-
const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
|
|
1222
|
+
const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "embed" ? answerEmbed(core, message) : answerRerank(core, message);
|
|
1188
1223
|
void reply.then((answer) => {
|
|
1189
1224
|
if (answer !== null) port.postMessage(answer);
|
|
1190
1225
|
});
|
|
@@ -1226,6 +1261,7 @@ async function startRequestThreads(options) {
|
|
|
1226
1261
|
authorizes: authorize !== void 0,
|
|
1227
1262
|
searchHooks,
|
|
1228
1263
|
patternWorkCap: options.core.patternWorkCap,
|
|
1264
|
+
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1229
1265
|
allowRegex: options.allowRegex
|
|
1230
1266
|
}
|
|
1231
1267
|
},
|
|
@@ -1490,7 +1526,7 @@ var TaskRegistry = class {
|
|
|
1490
1526
|
};
|
|
1491
1527
|
|
|
1492
1528
|
// src/server/create-server.ts
|
|
1493
|
-
function resolveLimits(limits) {
|
|
1529
|
+
function resolveLimits(limits, engineRerankWindow) {
|
|
1494
1530
|
return {
|
|
1495
1531
|
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
1496
1532
|
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
@@ -1499,6 +1535,7 @@ function resolveLimits(limits) {
|
|
|
1499
1535
|
importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
|
|
1500
1536
|
maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
|
|
1501
1537
|
maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
|
|
1538
|
+
maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
|
|
1502
1539
|
maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
|
|
1503
1540
|
maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
|
|
1504
1541
|
maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
|
|
@@ -1595,7 +1632,8 @@ var NarsilHttpServer = class {
|
|
|
1595
1632
|
stopIdleHeapCleanup = () => void 0;
|
|
1596
1633
|
constructor(engine, options = {}) {
|
|
1597
1634
|
assertServerEngine(engine);
|
|
1598
|
-
|
|
1635
|
+
const engineRerankWindow = rerankWindowCapOf(engine);
|
|
1636
|
+
assertServerLimits(options.limits, engineRerankWindow);
|
|
1599
1637
|
assertAllowRegex(options.allowRegex);
|
|
1600
1638
|
this.host = options.host ?? "127.0.0.1";
|
|
1601
1639
|
this.port = options.port ?? 9876;
|
|
@@ -1604,10 +1642,13 @@ var NarsilHttpServer = class {
|
|
|
1604
1642
|
for (const [name, adapter] of Object.entries(options.embeddingAdapters ?? {})) {
|
|
1605
1643
|
engine.registerEmbeddingAdapter(name, adapter);
|
|
1606
1644
|
}
|
|
1645
|
+
for (const [name, adapter] of Object.entries(options.rerankAdapters ?? {})) {
|
|
1646
|
+
engine.registerRerankAdapter(name, adapter);
|
|
1647
|
+
}
|
|
1607
1648
|
engineCoreOf(engine)?.orchestrator.shareMainThread();
|
|
1608
1649
|
const taskStore = options.taskStore ?? new InMemoryTaskStore();
|
|
1609
1650
|
const instanceId = options.instanceId ?? randomUUID();
|
|
1610
|
-
const limits = resolveLimits(options.limits);
|
|
1651
|
+
const limits = resolveLimits(options.limits, engineRerankWindow);
|
|
1611
1652
|
this.deps = {
|
|
1612
1653
|
engine,
|
|
1613
1654
|
tasks: new TaskRegistry(taskStore, instanceId, limits.maxConcurrentTasks),
|
|
@@ -1,47 +1,49 @@
|
|
|
1
|
-
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
2
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, bytesToVectors } from './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-37CD3OKS.mjs';
|
|
1
|
+
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-BKB3XTMI.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-NTRMFARG.mjs';
|
|
3
|
+
import './chunk-COVCQ73F.mjs';
|
|
4
|
+
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-ZG477IPV.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, bytesToVectors } from './chunk-WCJKUGHB.mjs';
|
|
6
|
+
import './chunk-HXJBSHXF.mjs';
|
|
8
7
|
import './chunk-DJHHF2L5.mjs';
|
|
9
|
-
import
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
8
|
+
import './chunk-37CD3OKS.mjs';
|
|
9
|
+
import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-LH4NZNJJ.mjs';
|
|
10
|
+
import './chunk-L3L5YFXC.mjs';
|
|
12
11
|
import './chunk-P36MUFEJ.mjs';
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import './chunk-
|
|
18
|
-
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-WLOF6HSP.mjs';
|
|
12
|
+
import './chunk-OHP7QR33.mjs';
|
|
13
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-VWULPWRJ.mjs';
|
|
14
|
+
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-NKRBEGJZ.mjs';
|
|
15
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-R7EEKAJE.mjs';
|
|
16
|
+
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
19
17
|
import './chunk-AV4IJKD2.mjs';
|
|
20
|
-
import './chunk-
|
|
21
|
-
import
|
|
18
|
+
import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-OWNZYMRE.mjs';
|
|
19
|
+
import './chunk-K25LZA54.mjs';
|
|
20
|
+
import { getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
21
|
+
import './chunk-HJSN7UYG.mjs';
|
|
22
|
+
import './chunk-LXWDQLTW.mjs';
|
|
22
23
|
import './chunk-TGP5M5R7.mjs';
|
|
23
24
|
import './chunk-U2BKYEC7.mjs';
|
|
24
25
|
import './chunk-KOKM4PQN.mjs';
|
|
25
26
|
import './chunk-QY7GC2MW.mjs';
|
|
26
|
-
import './chunk-
|
|
27
|
-
import './chunk-
|
|
27
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
28
|
+
import './chunk-ASY5PHYI.mjs';
|
|
28
29
|
import './chunk-WQBCBKY3.mjs';
|
|
29
|
-
import { parseVectorDimension } from './chunk-
|
|
30
|
-
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-
|
|
31
|
-
import './chunk-
|
|
30
|
+
import { parseVectorDimension } from './chunk-MHOSCQCQ.mjs';
|
|
31
|
+
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-LXPIXDUJ.mjs';
|
|
32
|
+
import './chunk-E74YLAIG.mjs';
|
|
32
33
|
import './chunk-KEHK2O6P.mjs';
|
|
33
34
|
import './chunk-GES7U2PM.mjs';
|
|
34
|
-
import './chunk-
|
|
35
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
35
|
+
import './chunk-OOEXLUGR.mjs';
|
|
36
|
+
import { resolveIndexAnalysis } from './chunk-RONEOEON.mjs';
|
|
36
37
|
import './chunk-X5CV2E2B.mjs';
|
|
37
38
|
import './chunk-7X5JST5L.mjs';
|
|
38
|
-
import './chunk-
|
|
39
|
+
import './chunk-LU2CE5AE.mjs';
|
|
39
40
|
import './chunk-R6A773WK.mjs';
|
|
40
41
|
import './chunk-YTJCGKM4.mjs';
|
|
42
|
+
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
41
43
|
import './chunk-R6PR4WYZ.mjs';
|
|
42
|
-
import { loadVectorCodeFor } from './chunk-
|
|
44
|
+
import { loadVectorCodeFor } from './chunk-PPY2GSIB.mjs';
|
|
43
45
|
import './chunk-L5ORED7V.mjs';
|
|
44
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
46
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
45
47
|
import { decode } from '@msgpack/msgpack';
|
|
46
48
|
|
|
47
49
|
// src/persistence/archive/validate.ts
|
|
@@ -84,43 +86,84 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
84
86
|
}
|
|
85
87
|
|
|
86
88
|
// src/engine/snapshot/constants.ts
|
|
87
|
-
var
|
|
88
|
-
var
|
|
89
|
-
var
|
|
90
|
-
var
|
|
91
|
-
var
|
|
92
|
-
var
|
|
93
|
-
var
|
|
94
|
-
var
|
|
95
|
-
var
|
|
96
|
-
var
|
|
97
|
-
var
|
|
98
|
-
var
|
|
89
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
90
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
91
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
92
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
93
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
94
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
95
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
96
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
97
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
98
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
99
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
100
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
101
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
102
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
103
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
104
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
105
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
106
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
107
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
108
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
109
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
110
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
111
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
112
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
113
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
99
114
|
|
|
100
115
|
// src/engine/snapshot/memory-budget.ts
|
|
101
116
|
var MEGABYTE_BYTES = 1048576;
|
|
102
117
|
var ARCHIVE_RATES = {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
118
|
+
worst: {
|
|
119
|
+
heap: {
|
|
120
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
121
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
122
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
123
|
+
},
|
|
124
|
+
external: {
|
|
125
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
126
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
127
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
128
|
+
}
|
|
107
129
|
},
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
130
|
+
floor: {
|
|
131
|
+
heap: {
|
|
132
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
133
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
134
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
135
|
+
},
|
|
136
|
+
external: {
|
|
137
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
138
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
139
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
140
|
+
}
|
|
112
141
|
}
|
|
113
142
|
};
|
|
114
143
|
var EARLIER_SNAPSHOT_RATES = {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
144
|
+
worst: {
|
|
145
|
+
heap: {
|
|
146
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
147
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
148
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
149
|
+
},
|
|
150
|
+
external: {
|
|
151
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
152
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
153
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
154
|
+
}
|
|
119
155
|
},
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
156
|
+
floor: {
|
|
157
|
+
heap: {
|
|
158
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
159
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
160
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
161
|
+
},
|
|
162
|
+
external: {
|
|
163
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
164
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
165
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
166
|
+
}
|
|
124
167
|
}
|
|
125
168
|
};
|
|
126
169
|
function bytesAt(rate, storedBytes, vectors) {
|
|
@@ -128,10 +171,12 @@ function bytesAt(rate, storedBytes, vectors) {
|
|
|
128
171
|
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
129
172
|
return bytes;
|
|
130
173
|
}
|
|
131
|
-
function footprintAt(rates, storedBytes, vectors) {
|
|
174
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
175
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
176
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
132
177
|
return {
|
|
133
|
-
heapBytes: bytesAt(
|
|
134
|
-
externalBytes: bytesAt(
|
|
178
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
179
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
135
180
|
};
|
|
136
181
|
}
|
|
137
182
|
function deadCountOf(dead) {
|
|
@@ -142,7 +187,7 @@ function deadCountOf(dead) {
|
|
|
142
187
|
}
|
|
143
188
|
return count;
|
|
144
189
|
}
|
|
145
|
-
function archiveFootprint(prepared, manifest) {
|
|
190
|
+
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
146
191
|
let segmentBytes = 0;
|
|
147
192
|
for (const partition of manifest.partitions) {
|
|
148
193
|
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
@@ -153,36 +198,46 @@ function archiveFootprint(prepared, manifest) {
|
|
|
153
198
|
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
154
199
|
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
155
200
|
}
|
|
156
|
-
return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
|
|
201
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
157
202
|
}
|
|
158
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
|
|
203
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
159
204
|
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
160
205
|
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
161
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
|
|
206
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
162
207
|
}
|
|
163
208
|
function inMegabytes(bytes) {
|
|
164
209
|
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
165
210
|
}
|
|
166
|
-
function
|
|
167
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
168
|
-
throw new NarsilError(
|
|
169
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
170
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
|
|
171
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
function requireRestoreMemory(indexName, footprint) {
|
|
211
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
175
212
|
const heap = readHeapStatistics();
|
|
176
213
|
if (heap !== null) {
|
|
177
|
-
const
|
|
178
|
-
|
|
214
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
215
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
216
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
179
217
|
}
|
|
180
218
|
const resident = readResidentMemory();
|
|
181
219
|
if (resident !== null) {
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
|
|
220
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
221
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
185
222
|
}
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
226
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
227
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
228
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
229
|
+
throw new NarsilError(
|
|
230
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
231
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
232
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
236
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
237
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
238
|
+
const collected = collectAndReturnPages();
|
|
239
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
240
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
186
241
|
}
|
|
187
242
|
|
|
188
243
|
// src/workers/detached-index.ts
|
|
@@ -483,9 +538,8 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
483
538
|
manifest: async (manifest2, sourceName) => {
|
|
484
539
|
if (staged.prepared === null) missingPart("configuration");
|
|
485
540
|
requireSameSchema(staged.prepared, manifest2);
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
}
|
|
541
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
542
|
+
requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
|
|
489
543
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
490
544
|
},
|
|
491
545
|
file: async (entryName, _ref, bytes) => {
|
|
@@ -575,7 +629,7 @@ function vectorsOfSnapshot(envelope) {
|
|
|
575
629
|
}
|
|
576
630
|
return byDocument;
|
|
577
631
|
}
|
|
578
|
-
function footprintOf(envelope, prepared) {
|
|
632
|
+
function footprintOf(envelope, prepared, replacesIndex) {
|
|
579
633
|
let partitionBytes = 0;
|
|
580
634
|
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
581
635
|
const vectorCounts = /* @__PURE__ */ new Map();
|
|
@@ -584,7 +638,7 @@ function footprintOf(envelope, prepared) {
|
|
|
584
638
|
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
585
639
|
vectorCounts.set(fieldPath, count);
|
|
586
640
|
}
|
|
587
|
-
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
|
|
641
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
|
|
588
642
|
}
|
|
589
643
|
function documentsOfPartition(envelope, payload) {
|
|
590
644
|
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
@@ -644,7 +698,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
|
|
|
644
698
|
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
645
699
|
const envelope = await decodeEarlierSnapshot(data);
|
|
646
700
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
647
|
-
|
|
701
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
702
|
+
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
648
703
|
const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
|
|
649
704
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
650
705
|
try {
|
package/dist/synonyms.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { synonymSearchModule } from './chunk-
|
|
1
|
+
import { synonymSearchModule } from './chunk-RONEOEON.mjs';
|
|
2
2
|
import './chunk-X5CV2E2B.mjs';
|
|
3
3
|
import './chunk-7X5JST5L.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-LU2CE5AE.mjs';
|
|
5
5
|
import './chunk-R6A773WK.mjs';
|
|
6
6
|
import './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-KV3AZFT3.mjs';
|
|
8
8
|
|
|
9
9
|
// src/synonyms/index.ts
|
|
10
10
|
var synonyms = synonymSearchModule;
|