@delali/narsil 0.3.1-next.96 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-ZGMR46O3.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-KPMIHLOV.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-3MYKLHRO.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-R7P64REL.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-WMYC5ZK6.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-3A3WDNHO.mjs → snapshot-J57MZK6L.mjs} +8 -6
- package/dist/browser/{write-JODZHKIF.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-H75XXPMY.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-2BY6K52M.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-GMY7F2VA.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-VVFLEGZI.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-LJBJQPB5.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-O5WPRAVZ.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-TCYUAUHF.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-2VBNYYDY.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CbZmXUp6.d.ts → engine-BGCEm9E_.d.ts} +48 -18
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-UFIQJTSG.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-XII3ANGW.mjs → snapshot-HOGXNO5W.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-6M3VG4TU.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
package/dist/server.mjs
CHANGED
|
@@ -1,27 +1,29 @@
|
|
|
1
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-
|
|
2
|
-
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
3
|
-
import {
|
|
1
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-YA7XLE2R.mjs';
|
|
2
|
+
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-YA7XLE2R.mjs';
|
|
3
|
+
import { rerankOnHost } from './chunk-ZYNAMIYX.mjs';
|
|
4
|
+
import { validateIndexName } from './chunk-2WYMG3OZ.mjs';
|
|
4
5
|
import './chunk-DJHHF2L5.mjs';
|
|
5
6
|
import './chunk-P36MUFEJ.mjs';
|
|
6
|
-
import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-
|
|
7
|
+
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-R7EEKAJE.mjs';
|
|
7
8
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
8
|
-
import './chunk-
|
|
9
|
-
import './chunk-
|
|
9
|
+
import './chunk-JFRNNBOM.mjs';
|
|
10
|
+
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
11
|
+
import './chunk-LXWDQLTW.mjs';
|
|
10
12
|
import './chunk-TGP5M5R7.mjs';
|
|
11
13
|
import './chunk-U2BKYEC7.mjs';
|
|
12
|
-
import './chunk-
|
|
14
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
13
15
|
import './chunk-WQBCBKY3.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
+
import './chunk-LXPIXDUJ.mjs';
|
|
17
|
+
import './chunk-E74YLAIG.mjs';
|
|
16
18
|
import './chunk-KEHK2O6P.mjs';
|
|
17
|
-
import './chunk-
|
|
19
|
+
import './chunk-OOEXLUGR.mjs';
|
|
18
20
|
import './chunk-7X5JST5L.mjs';
|
|
19
21
|
import './chunk-R6A773WK.mjs';
|
|
20
22
|
import './chunk-YTJCGKM4.mjs';
|
|
21
23
|
import { startIdleHeapCleanup } from './chunk-5XNIRLFC.mjs';
|
|
22
24
|
import './chunk-L5ORED7V.mjs';
|
|
23
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-
|
|
24
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
25
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
26
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
25
27
|
import { randomUUID } from 'crypto';
|
|
26
28
|
import { createReadStream } from 'fs';
|
|
27
29
|
import { rm } from 'fs/promises';
|
|
@@ -813,14 +815,14 @@ function createSearchHandlers(deps) {
|
|
|
813
815
|
async function search(ctx) {
|
|
814
816
|
const params = parseJson(ctx);
|
|
815
817
|
if (!params) return;
|
|
816
|
-
const failure = validateQuery(params, limits
|
|
818
|
+
const failure = validateQuery(params, limits);
|
|
817
819
|
if (failure) {
|
|
818
820
|
rejectInvalid(ctx, failure);
|
|
819
821
|
return;
|
|
820
822
|
}
|
|
821
823
|
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
822
824
|
try {
|
|
823
|
-
respondJson(ctx, await engine.query(ctx.params[0], params));
|
|
825
|
+
respondJson(ctx, await engine.query(ctx.params[0], withServerRerankWindow(params, limits)));
|
|
824
826
|
} catch (err) {
|
|
825
827
|
respondError(ctx, err);
|
|
826
828
|
}
|
|
@@ -828,7 +830,7 @@ function createSearchHandlers(deps) {
|
|
|
828
830
|
async function preflight(ctx) {
|
|
829
831
|
const params = parseJson(ctx);
|
|
830
832
|
if (!params) return;
|
|
831
|
-
const failure = validateQuery(params, limits
|
|
833
|
+
const failure = validateQuery(params, limits);
|
|
832
834
|
if (failure) {
|
|
833
835
|
rejectInvalid(ctx, failure);
|
|
834
836
|
return;
|
|
@@ -868,7 +870,13 @@ var LIMITS_UNBOUNDED_AT_ZERO = [
|
|
|
868
870
|
"maxConcurrentRequests",
|
|
869
871
|
"maxConcurrentTasks"
|
|
870
872
|
];
|
|
871
|
-
var COUNTED_LIMITS = [
|
|
873
|
+
var COUNTED_LIMITS = [
|
|
874
|
+
"importBatchSize",
|
|
875
|
+
"maxResultWindow",
|
|
876
|
+
"maxRerankWindow",
|
|
877
|
+
"maxFetchDocuments",
|
|
878
|
+
"maxTaskPageSize"
|
|
879
|
+
];
|
|
872
880
|
function assertServerEngine(engine) {
|
|
873
881
|
const candidate = engine;
|
|
874
882
|
if (typeof candidate !== "object" || candidate === null) {
|
|
@@ -896,10 +904,21 @@ function assertLimit(field, value, minimum) {
|
|
|
896
904
|
{ field, value, minimum }
|
|
897
905
|
);
|
|
898
906
|
}
|
|
899
|
-
function
|
|
907
|
+
function assertRerankWindowCap(value, engineCap) {
|
|
908
|
+
if (value === void 0) return;
|
|
909
|
+
const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
|
|
910
|
+
if (value <= cap) return;
|
|
911
|
+
throw new NarsilError(
|
|
912
|
+
ErrorCodes.CONFIG_INVALID,
|
|
913
|
+
`limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
|
|
914
|
+
{ field: "maxRerankWindow", value, maximum: cap }
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
function assertServerLimits(limits, engineRerankWindow) {
|
|
900
918
|
if (limits === void 0) return;
|
|
901
919
|
for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
|
|
902
920
|
for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
|
|
921
|
+
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
903
922
|
}
|
|
904
923
|
function assertAllowRegex(allowRegex) {
|
|
905
924
|
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
@@ -1064,7 +1083,14 @@ function toRelayedResponse(requestId, captured) {
|
|
|
1064
1083
|
}
|
|
1065
1084
|
|
|
1066
1085
|
// src/server/request-threads/host.ts
|
|
1067
|
-
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1086
|
+
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1087
|
+
"request",
|
|
1088
|
+
"authorize",
|
|
1089
|
+
"embed",
|
|
1090
|
+
"rerank",
|
|
1091
|
+
"stream-credit",
|
|
1092
|
+
"stream-cancel"
|
|
1093
|
+
]);
|
|
1068
1094
|
function isRelayRequest(message) {
|
|
1069
1095
|
if (typeof message !== "object" || message === null) return false;
|
|
1070
1096
|
const { type, requestId } = message;
|
|
@@ -1142,6 +1168,15 @@ async function answerEmbed(core, request) {
|
|
|
1142
1168
|
return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
|
|
1143
1169
|
}
|
|
1144
1170
|
}
|
|
1171
|
+
async function answerRerank(core, request) {
|
|
1172
|
+
try {
|
|
1173
|
+
const scores = await rerankOnHost(core, request.adapter, request.query, request.documents);
|
|
1174
|
+
return { type: "reranked", requestId: request.requestId, scores, error: null };
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
|
|
1177
|
+
return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1145
1180
|
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
1146
1181
|
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
1147
1182
|
}
|
|
@@ -1184,7 +1219,7 @@ async function startRequestThreads(options) {
|
|
|
1184
1219
|
streams.get(message.requestId)?.cancel();
|
|
1185
1220
|
return;
|
|
1186
1221
|
}
|
|
1187
|
-
const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
|
|
1222
|
+
const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "embed" ? answerEmbed(core, message) : answerRerank(core, message);
|
|
1188
1223
|
void reply.then((answer) => {
|
|
1189
1224
|
if (answer !== null) port.postMessage(answer);
|
|
1190
1225
|
});
|
|
@@ -1226,6 +1261,7 @@ async function startRequestThreads(options) {
|
|
|
1226
1261
|
authorizes: authorize !== void 0,
|
|
1227
1262
|
searchHooks,
|
|
1228
1263
|
patternWorkCap: options.core.patternWorkCap,
|
|
1264
|
+
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1229
1265
|
allowRegex: options.allowRegex
|
|
1230
1266
|
}
|
|
1231
1267
|
},
|
|
@@ -1490,7 +1526,7 @@ var TaskRegistry = class {
|
|
|
1490
1526
|
};
|
|
1491
1527
|
|
|
1492
1528
|
// src/server/create-server.ts
|
|
1493
|
-
function resolveLimits(limits) {
|
|
1529
|
+
function resolveLimits(limits, engineRerankWindow) {
|
|
1494
1530
|
return {
|
|
1495
1531
|
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
1496
1532
|
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
@@ -1499,6 +1535,7 @@ function resolveLimits(limits) {
|
|
|
1499
1535
|
importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
|
|
1500
1536
|
maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
|
|
1501
1537
|
maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
|
|
1538
|
+
maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
|
|
1502
1539
|
maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
|
|
1503
1540
|
maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
|
|
1504
1541
|
maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
|
|
@@ -1595,7 +1632,8 @@ var NarsilHttpServer = class {
|
|
|
1595
1632
|
stopIdleHeapCleanup = () => void 0;
|
|
1596
1633
|
constructor(engine, options = {}) {
|
|
1597
1634
|
assertServerEngine(engine);
|
|
1598
|
-
|
|
1635
|
+
const engineRerankWindow = rerankWindowCapOf(engine);
|
|
1636
|
+
assertServerLimits(options.limits, engineRerankWindow);
|
|
1599
1637
|
assertAllowRegex(options.allowRegex);
|
|
1600
1638
|
this.host = options.host ?? "127.0.0.1";
|
|
1601
1639
|
this.port = options.port ?? 9876;
|
|
@@ -1604,10 +1642,13 @@ var NarsilHttpServer = class {
|
|
|
1604
1642
|
for (const [name, adapter] of Object.entries(options.embeddingAdapters ?? {})) {
|
|
1605
1643
|
engine.registerEmbeddingAdapter(name, adapter);
|
|
1606
1644
|
}
|
|
1645
|
+
for (const [name, adapter] of Object.entries(options.rerankAdapters ?? {})) {
|
|
1646
|
+
engine.registerRerankAdapter(name, adapter);
|
|
1647
|
+
}
|
|
1607
1648
|
engineCoreOf(engine)?.orchestrator.shareMainThread();
|
|
1608
1649
|
const taskStore = options.taskStore ?? new InMemoryTaskStore();
|
|
1609
1650
|
const instanceId = options.instanceId ?? randomUUID();
|
|
1610
|
-
const limits = resolveLimits(options.limits);
|
|
1651
|
+
const limits = resolveLimits(options.limits, engineRerankWindow);
|
|
1611
1652
|
this.deps = {
|
|
1612
1653
|
engine,
|
|
1613
1654
|
tasks: new TaskRegistry(taskStore, instanceId, limits.maxConcurrentTasks),
|
|
@@ -1,48 +1,49 @@
|
|
|
1
|
-
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
2
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, bytesToVectors } from './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-37CD3OKS.mjs';
|
|
1
|
+
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-BKB3XTMI.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-NTRMFARG.mjs';
|
|
3
|
+
import './chunk-COVCQ73F.mjs';
|
|
4
|
+
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-ZG477IPV.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, bytesToVectors } from './chunk-WCJKUGHB.mjs';
|
|
6
|
+
import './chunk-HXJBSHXF.mjs';
|
|
8
7
|
import './chunk-DJHHF2L5.mjs';
|
|
9
|
-
import
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
8
|
+
import './chunk-37CD3OKS.mjs';
|
|
9
|
+
import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-LH4NZNJJ.mjs';
|
|
10
|
+
import './chunk-L3L5YFXC.mjs';
|
|
12
11
|
import './chunk-P36MUFEJ.mjs';
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import './chunk-
|
|
18
|
-
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-WLOF6HSP.mjs';
|
|
12
|
+
import './chunk-OHP7QR33.mjs';
|
|
13
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-VWULPWRJ.mjs';
|
|
14
|
+
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-NKRBEGJZ.mjs';
|
|
15
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-R7EEKAJE.mjs';
|
|
16
|
+
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
19
17
|
import './chunk-AV4IJKD2.mjs';
|
|
20
|
-
import './chunk-
|
|
21
|
-
import
|
|
18
|
+
import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-OWNZYMRE.mjs';
|
|
19
|
+
import './chunk-K25LZA54.mjs';
|
|
20
|
+
import { getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
21
|
+
import './chunk-HJSN7UYG.mjs';
|
|
22
|
+
import './chunk-LXWDQLTW.mjs';
|
|
22
23
|
import './chunk-TGP5M5R7.mjs';
|
|
23
24
|
import './chunk-U2BKYEC7.mjs';
|
|
24
25
|
import './chunk-KOKM4PQN.mjs';
|
|
25
26
|
import './chunk-QY7GC2MW.mjs';
|
|
26
|
-
import './chunk-
|
|
27
|
-
import './chunk-
|
|
27
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
28
|
+
import './chunk-ASY5PHYI.mjs';
|
|
28
29
|
import './chunk-WQBCBKY3.mjs';
|
|
29
|
-
import { parseVectorDimension } from './chunk-
|
|
30
|
-
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-
|
|
31
|
-
import './chunk-
|
|
30
|
+
import { parseVectorDimension } from './chunk-MHOSCQCQ.mjs';
|
|
31
|
+
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-LXPIXDUJ.mjs';
|
|
32
|
+
import './chunk-E74YLAIG.mjs';
|
|
32
33
|
import './chunk-KEHK2O6P.mjs';
|
|
33
34
|
import './chunk-GES7U2PM.mjs';
|
|
34
|
-
import './chunk-
|
|
35
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
35
|
+
import './chunk-OOEXLUGR.mjs';
|
|
36
|
+
import { resolveIndexAnalysis } from './chunk-RONEOEON.mjs';
|
|
36
37
|
import './chunk-X5CV2E2B.mjs';
|
|
37
38
|
import './chunk-7X5JST5L.mjs';
|
|
38
|
-
import './chunk-
|
|
39
|
+
import './chunk-LU2CE5AE.mjs';
|
|
39
40
|
import './chunk-R6A773WK.mjs';
|
|
40
41
|
import './chunk-YTJCGKM4.mjs';
|
|
41
42
|
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
42
43
|
import './chunk-R6PR4WYZ.mjs';
|
|
43
|
-
import { loadVectorCodeFor } from './chunk-
|
|
44
|
+
import { loadVectorCodeFor } from './chunk-PPY2GSIB.mjs';
|
|
44
45
|
import './chunk-L5ORED7V.mjs';
|
|
45
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
46
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
46
47
|
import { decode } from '@msgpack/msgpack';
|
|
47
48
|
|
|
48
49
|
// src/persistence/archive/validate.ts
|
package/dist/synonyms.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { synonymSearchModule } from './chunk-
|
|
1
|
+
import { synonymSearchModule } from './chunk-RONEOEON.mjs';
|
|
2
2
|
import './chunk-X5CV2E2B.mjs';
|
|
3
3
|
import './chunk-7X5JST5L.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-LU2CE5AE.mjs';
|
|
5
5
|
import './chunk-R6A773WK.mjs';
|
|
6
6
|
import './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-KV3AZFT3.mjs';
|
|
8
8
|
|
|
9
9
|
// src/synonyms/index.ts
|
|
10
10
|
var synonyms = synonymSearchModule;
|
|
@@ -1,24 +1,26 @@
|
|
|
1
|
-
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-
|
|
1
|
+
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, withServerRerankWindow, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-YA7XLE2R.mjs';
|
|
2
|
+
import { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
2
3
|
import './chunk-DJHHF2L5.mjs';
|
|
3
|
-
import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-
|
|
4
|
-
import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-
|
|
5
|
-
import './chunk-
|
|
4
|
+
import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-NKRBEGJZ.mjs';
|
|
5
|
+
import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-K25LZA54.mjs';
|
|
6
|
+
import './chunk-HJSN7UYG.mjs';
|
|
7
|
+
import './chunk-LXWDQLTW.mjs';
|
|
6
8
|
import './chunk-QY7GC2MW.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-
|
|
9
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
10
|
+
import './chunk-ASY5PHYI.mjs';
|
|
9
11
|
import './chunk-WQBCBKY3.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import { extractVectorFieldsFromSchema } from './chunk-
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-MHOSCQCQ.mjs';
|
|
13
|
+
import { extractVectorFieldsFromSchema } from './chunk-LXPIXDUJ.mjs';
|
|
14
|
+
import './chunk-E74YLAIG.mjs';
|
|
13
15
|
import './chunk-KEHK2O6P.mjs';
|
|
14
16
|
import './chunk-GES7U2PM.mjs';
|
|
15
|
-
import './chunk-
|
|
17
|
+
import './chunk-OOEXLUGR.mjs';
|
|
16
18
|
import './chunk-X5CV2E2B.mjs';
|
|
17
|
-
import './chunk-
|
|
19
|
+
import './chunk-LU2CE5AE.mjs';
|
|
18
20
|
import './chunk-R6A773WK.mjs';
|
|
19
21
|
import './chunk-YTJCGKM4.mjs';
|
|
20
22
|
import './chunk-L5ORED7V.mjs';
|
|
21
|
-
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-
|
|
23
|
+
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
22
24
|
|
|
23
25
|
// src/server/request-threads/held-vectors.ts
|
|
24
26
|
function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
|
|
@@ -40,7 +42,7 @@ function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
|
|
|
40
42
|
|
|
41
43
|
// src/server/request-threads/read-engine.ts
|
|
42
44
|
function createThreadReadEngine(options) {
|
|
43
|
-
const { executor, relay, searchHooks, patternWorkCap } = options;
|
|
45
|
+
const { executor, relay, searchHooks, patternWorkCap, maxRerankWindow } = options;
|
|
44
46
|
function vectorFieldsOf(context) {
|
|
45
47
|
return [...extractVectorFieldsFromSchema(context.config.schema).keys()];
|
|
46
48
|
}
|
|
@@ -85,7 +87,15 @@ function createThreadReadEngine(options) {
|
|
|
85
87
|
const context = requireAnswerable(indexName, wholeDocumentsContext(indexName));
|
|
86
88
|
return { context, manager: managerOf(indexName, context) };
|
|
87
89
|
}
|
|
88
|
-
|
|
90
|
+
function rerankRunnerFor(params) {
|
|
91
|
+
if (params.rerank === void 0) return void 0;
|
|
92
|
+
return {
|
|
93
|
+
maxWindow: maxRerankWindow,
|
|
94
|
+
score: (adapter, query, documents) => relay.rerank(adapter, query, documents)
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
async function resolvedParams(indexName, request) {
|
|
98
|
+
const params = withRerankQueryText(request);
|
|
89
99
|
if (params.vector?.text === void 0) return params;
|
|
90
100
|
const value = await relay.embed(indexName, params.vector);
|
|
91
101
|
const { text: _text, ...vector } = params.vector;
|
|
@@ -120,7 +130,8 @@ function createThreadReadEngine(options) {
|
|
|
120
130
|
indexName,
|
|
121
131
|
cursorBinding: queryBindingOf(params),
|
|
122
132
|
vectorSearchers: context.vectorSearchers,
|
|
123
|
-
patternWorkCap
|
|
133
|
+
patternWorkCap,
|
|
134
|
+
rerank: rerankRunnerFor(params)
|
|
124
135
|
});
|
|
125
136
|
if (context.analysisStale) result.analysisStale = true;
|
|
126
137
|
return result;
|
|
@@ -180,7 +191,7 @@ function createThreadReadEngine(options) {
|
|
|
180
191
|
function isRelayReply(message) {
|
|
181
192
|
if (typeof message !== "object" || message === null) return false;
|
|
182
193
|
const { type, requestId } = message;
|
|
183
|
-
return typeof requestId === "number" && (type === "response" || type === "authorized" || type === "embedded");
|
|
194
|
+
return typeof requestId === "number" && (type === "response" || type === "authorized" || type === "embedded" || type === "reranked");
|
|
184
195
|
}
|
|
185
196
|
function isStreamPart(message) {
|
|
186
197
|
if (typeof message !== "object" || message === null) return false;
|
|
@@ -222,9 +233,8 @@ function writeStreamPart(port, writer, part) {
|
|
|
222
233
|
return true;
|
|
223
234
|
});
|
|
224
235
|
}
|
|
225
|
-
function
|
|
226
|
-
|
|
227
|
-
return new NarsilError(error.code, error.message, error.details);
|
|
236
|
+
function relayedErrorOf(error, fallback) {
|
|
237
|
+
return error === null ? fallback : new NarsilError(error.code, error.message, error.details);
|
|
228
238
|
}
|
|
229
239
|
function portClosedError() {
|
|
230
240
|
return new NarsilError(ErrorCodes.WORKER_CRASHED, "The port to the main thread closed before it answered");
|
|
@@ -334,10 +344,19 @@ function createRelayClient(port, touchIntervalMs) {
|
|
|
334
344
|
},
|
|
335
345
|
async embed(indexName, vector) {
|
|
336
346
|
const reply = await send((requestId) => ({ type: "embed", requestId, indexName, vector }));
|
|
337
|
-
if (reply.type
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
347
|
+
if (reply.type === "embedded" && reply.value !== null) return reply.value;
|
|
348
|
+
throw relayedErrorOf(
|
|
349
|
+
reply.type === "embedded" ? reply.error : null,
|
|
350
|
+
new NarsilError(ErrorCodes.EMBEDDING_FAILED, "The main thread returned no vector")
|
|
351
|
+
);
|
|
352
|
+
},
|
|
353
|
+
async rerank(adapter, query, documents) {
|
|
354
|
+
const reply = await send((requestId) => ({ type: "rerank", requestId, adapter, query, documents }));
|
|
355
|
+
if (reply.type === "reranked" && reply.scores !== null) return reply.scores;
|
|
356
|
+
throw relayedErrorOf(
|
|
357
|
+
reply.type === "reranked" ? reply.error : null,
|
|
358
|
+
new NarsilError(ErrorCodes.RERANK_FAILED, "The main thread returned no scores")
|
|
359
|
+
);
|
|
341
360
|
},
|
|
342
361
|
touch(indexName) {
|
|
343
362
|
if (closed) return;
|
|
@@ -547,11 +566,11 @@ function mainThreadAnswersInstead(err) {
|
|
|
547
566
|
return isNarsilError(err) && err.code === ErrorCodes.WORKER_BUSY;
|
|
548
567
|
}
|
|
549
568
|
function searchHandler(options) {
|
|
550
|
-
const { fallback,
|
|
569
|
+
const { fallback, limits, allowRegex, staysOnThread, answer } = options;
|
|
551
570
|
return async (ctx) => {
|
|
552
571
|
const params = parseJson(ctx);
|
|
553
572
|
if (!params) return;
|
|
554
|
-
const failure = validateQuery(params,
|
|
573
|
+
const failure = validateQuery(params, limits);
|
|
555
574
|
if (failure) {
|
|
556
575
|
rejectInvalid(ctx, failure);
|
|
557
576
|
return;
|
|
@@ -562,7 +581,7 @@ function searchHandler(options) {
|
|
|
562
581
|
return;
|
|
563
582
|
}
|
|
564
583
|
try {
|
|
565
|
-
respondJson(ctx, await answer(ctx.params[0], params));
|
|
584
|
+
respondJson(ctx, await answer(ctx.params[0], withServerRerankWindow(params, limits)));
|
|
566
585
|
} catch (err) {
|
|
567
586
|
if (staysOnThread(ctx.params[0], params) && !mainThreadAnswersInstead(err)) respondError(ctx, err);
|
|
568
587
|
else await fallback(ctx);
|
|
@@ -604,7 +623,6 @@ function threadHandlers(local, relay, settings) {
|
|
|
604
623
|
const { allowRegex } = settings;
|
|
605
624
|
const reads = createDocumentReadHandlers({ engine: local, limits: settings.limits, allowRegex });
|
|
606
625
|
const forwarded = relayedHandlers(relay, settings.limits);
|
|
607
|
-
const { maxResultWindow } = settings.limits;
|
|
608
626
|
return {
|
|
609
627
|
...forwarded,
|
|
610
628
|
livez: (ctx) => respondJson(ctx, { status: "ok" }),
|
|
@@ -612,14 +630,14 @@ function threadHandlers(local, relay, settings) {
|
|
|
612
630
|
capabilities: (ctx) => respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] }),
|
|
613
631
|
search: searchHandler({
|
|
614
632
|
fallback: forwarded.search,
|
|
615
|
-
|
|
633
|
+
limits: settings.limits,
|
|
616
634
|
allowRegex,
|
|
617
635
|
staysOnThread: (name, params) => local.canAnswer(name, params),
|
|
618
636
|
answer: (name, params) => local.query(name, params)
|
|
619
637
|
}),
|
|
620
638
|
preflight: searchHandler({
|
|
621
639
|
fallback: forwarded.preflight,
|
|
622
|
-
|
|
640
|
+
limits: settings.limits,
|
|
623
641
|
allowRegex,
|
|
624
642
|
staysOnThread: (name, params) => local.canCount(name, params),
|
|
625
643
|
answer: (name, params) => local.preflight(name, params)
|
|
@@ -648,7 +666,8 @@ async function serveRequests(executor, settings) {
|
|
|
648
666
|
executor: departure.guardExecutor(executor),
|
|
649
667
|
relay,
|
|
650
668
|
searchHooks: settings.searchHooks,
|
|
651
|
-
patternWorkCap: settings.patternWorkCap
|
|
669
|
+
patternWorkCap: settings.patternWorkCap,
|
|
670
|
+
maxRerankWindow: settings.maxRerankWindow
|
|
652
671
|
});
|
|
653
672
|
const run = createRouteRunner({
|
|
654
673
|
authorize: settings.authorizes ? (context) => relay.authorize(context) : void 0,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { e as VectorQuantizationMode } from '../schema-
|
|
2
|
-
import '../adapters-
|
|
1
|
+
import { e as VectorQuantizationMode } from '../schema-4W_7tk1f.js';
|
|
2
|
+
import '../adapters-BQFQ4PRz.js';
|
|
3
3
|
import '../synonyms-CpOAHPa-.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -461,6 +461,8 @@ interface VectorSearchRequest {
|
|
|
461
461
|
efSearch?: number;
|
|
462
462
|
/** The worker re-scores this many times the requested count against full precision, or its own default when absent. */
|
|
463
463
|
oversample?: number;
|
|
464
|
+
/** When present, the worker re-scores only this many times the oversample of the nearest candidates against full precision and keeps the estimated score of the rest, or returns the re-scored candidates alone under a similarity floor. */
|
|
465
|
+
rescoreWindow?: number;
|
|
464
466
|
}
|
|
465
467
|
/**
|
|
466
468
|
* Asks the search worker to answer one nearest-neighbour query from a shared
|
|
@@ -489,6 +491,8 @@ interface VectorOrdinalSearchRequest {
|
|
|
489
491
|
efSearch?: number;
|
|
490
492
|
/** The worker re-scores this many times the requested count against full precision, or its own default when absent. */
|
|
491
493
|
oversample?: number;
|
|
494
|
+
/** When present, the worker re-scores only this many times the oversample of the nearest candidates against full precision and keeps the estimated score of the rest, or returns the re-scored candidates alone under a similarity floor. */
|
|
495
|
+
rescoreWindow?: number;
|
|
492
496
|
}
|
|
493
497
|
/**
|
|
494
498
|
* Every message the engine posts to the search worker.
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { openSharedVectorField, createVectorStore } from '../chunk-
|
|
2
|
-
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-
|
|
3
|
-
import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
|
|
1
|
+
import { openSharedVectorField, createVectorStore } from '../chunk-UCDF35PU.mjs';
|
|
2
|
+
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-HXJBSHXF.mjs';
|
|
4
3
|
import '../chunk-DJHHF2L5.mjs';
|
|
4
|
+
import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
|
|
5
5
|
import '../chunk-KOKM4PQN.mjs';
|
|
6
6
|
import { MAX_WORKER_COPIES } from '../chunk-QY7GC2MW.mjs';
|
|
7
|
-
import '../chunk-
|
|
7
|
+
import '../chunk-GGSH6KBQ.mjs';
|
|
8
8
|
import '../chunk-KEHK2O6P.mjs';
|
|
9
9
|
import '../chunk-R6A773WK.mjs';
|
|
10
10
|
import '../chunk-YTJCGKM4.mjs';
|
|
11
|
-
import '../chunk-
|
|
11
|
+
import '../chunk-KV3AZFT3.mjs';
|
|
12
12
|
|
|
13
13
|
// src/vector/worker-copy.ts
|
|
14
14
|
function restoreQuantizer(snapshot, store) {
|
|
@@ -101,7 +101,8 @@ function handleSearch(request) {
|
|
|
101
101
|
const hits = entry.copy.graph.search(request.query, request.k, request.metric, request.minSimilarity, {
|
|
102
102
|
filter: request.filter,
|
|
103
103
|
efSearch: request.efSearch,
|
|
104
|
-
oversample: request.oversample
|
|
104
|
+
oversample: request.oversample,
|
|
105
|
+
rescoreWindow: request.rescoreWindow
|
|
105
106
|
});
|
|
106
107
|
const docIds = [];
|
|
107
108
|
const scores = new Float64Array(hits.length);
|
|
@@ -122,7 +123,8 @@ function handleSearchOrdinals(request) {
|
|
|
122
123
|
const hits = entry.view.searchOrdinals(request.query, request.k, request.metric, request.minSimilarity, {
|
|
123
124
|
filter: request.filter,
|
|
124
125
|
efSearch: request.efSearch,
|
|
125
|
-
oversample: request.oversample
|
|
126
|
+
oversample: request.oversample,
|
|
127
|
+
rescoreWindow: request.rescoreWindow
|
|
126
128
|
});
|
|
127
129
|
const ordinals = new Uint32Array(hits.length);
|
|
128
130
|
const scores = new Float64Array(hits.length);
|
package/dist/workers/entry.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { startIdleHeapCleanup } from '../chunk-5XNIRLFC.mjs';
|
|
2
2
|
import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-R6PR4WYZ.mjs';
|
|
3
|
-
import { loadVectorCode } from '../chunk-
|
|
3
|
+
import { loadVectorCode } from '../chunk-PPY2GSIB.mjs';
|
|
4
4
|
import '../chunk-L5ORED7V.mjs';
|
|
5
|
-
import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-
|
|
5
|
+
import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-KV3AZFT3.mjs';
|
|
6
6
|
|
|
7
7
|
// src/workers/action-handler.ts
|
|
8
8
|
function buildErrorResponse(requestId, code, message, details) {
|
|
@@ -20,7 +20,7 @@ async function importBootstrapModule(moduleUrl) {
|
|
|
20
20
|
await import(moduleUrl);
|
|
21
21
|
}
|
|
22
22
|
async function stopServingRequests() {
|
|
23
|
-
const { closeRequestThread } = await import('../thread-
|
|
23
|
+
const { closeRequestThread } = await import('../thread-B657LQXK.mjs');
|
|
24
24
|
closeRequestThread();
|
|
25
25
|
}
|
|
26
26
|
function holdsCopies(executor) {
|
|
@@ -51,7 +51,7 @@ function createActionHandler(executor) {
|
|
|
51
51
|
if (!holdsCopies(executor)) {
|
|
52
52
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
|
|
53
53
|
}
|
|
54
|
-
const starting = import('../thread-
|
|
54
|
+
const starting = import('../thread-B657LQXK.mjs').then(
|
|
55
55
|
({ serveRequests }) => serveRequests(executor, action.settings)
|
|
56
56
|
);
|
|
57
57
|
serving.start = starting.then(
|
|
@@ -104,7 +104,7 @@ async function setup() {
|
|
|
104
104
|
} catch {
|
|
105
105
|
parentPort = null;
|
|
106
106
|
}
|
|
107
|
-
const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-
|
|
107
|
+
const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-CXZ66ACP.mjs'), loadVectorCode()]);
|
|
108
108
|
const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
|
|
109
109
|
if (parentPort) {
|
|
110
110
|
const port = parentPort;
|
|
@@ -1,38 +1,39 @@
|
|
|
1
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
1
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WCJKUGHB.mjs';
|
|
2
2
|
import './chunk-37CD3OKS.mjs';
|
|
3
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
4
|
-
import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import
|
|
8
|
-
import './chunk-CHOGQSHK.mjs';
|
|
9
|
-
import './chunk-WLOF6HSP.mjs';
|
|
3
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-L3L5YFXC.mjs';
|
|
4
|
+
import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-VWULPWRJ.mjs';
|
|
5
|
+
import './chunk-NKRBEGJZ.mjs';
|
|
6
|
+
import './chunk-R7EEKAJE.mjs';
|
|
7
|
+
import './chunk-AY3AQ3UH.mjs';
|
|
10
8
|
import './chunk-AV4IJKD2.mjs';
|
|
11
|
-
import {
|
|
12
|
-
import './chunk-
|
|
9
|
+
import { decodeStoredDocument } from './chunk-OWNZYMRE.mjs';
|
|
10
|
+
import './chunk-K25LZA54.mjs';
|
|
11
|
+
import './chunk-JFRNNBOM.mjs';
|
|
12
|
+
import './chunk-HJSN7UYG.mjs';
|
|
13
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-LXWDQLTW.mjs';
|
|
13
14
|
import './chunk-TGP5M5R7.mjs';
|
|
14
15
|
import './chunk-U2BKYEC7.mjs';
|
|
15
16
|
import './chunk-KOKM4PQN.mjs';
|
|
16
17
|
import './chunk-QY7GC2MW.mjs';
|
|
17
|
-
import './chunk-
|
|
18
|
-
import './chunk-
|
|
18
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
19
|
+
import './chunk-ASY5PHYI.mjs';
|
|
19
20
|
import './chunk-WQBCBKY3.mjs';
|
|
20
|
-
import './chunk-
|
|
21
|
-
import './chunk-
|
|
22
|
-
import './chunk-
|
|
21
|
+
import './chunk-MHOSCQCQ.mjs';
|
|
22
|
+
import './chunk-LXPIXDUJ.mjs';
|
|
23
|
+
import './chunk-E74YLAIG.mjs';
|
|
23
24
|
import './chunk-KEHK2O6P.mjs';
|
|
24
25
|
import './chunk-GES7U2PM.mjs';
|
|
25
|
-
import './chunk-
|
|
26
|
-
import './chunk-
|
|
26
|
+
import './chunk-OOEXLUGR.mjs';
|
|
27
|
+
import './chunk-RONEOEON.mjs';
|
|
27
28
|
import './chunk-X5CV2E2B.mjs';
|
|
28
29
|
import './chunk-7X5JST5L.mjs';
|
|
29
|
-
import './chunk-
|
|
30
|
+
import './chunk-LU2CE5AE.mjs';
|
|
30
31
|
import './chunk-R6A773WK.mjs';
|
|
31
32
|
import './chunk-YTJCGKM4.mjs';
|
|
32
33
|
import './chunk-R6PR4WYZ.mjs';
|
|
33
|
-
import './chunk-
|
|
34
|
+
import './chunk-PPY2GSIB.mjs';
|
|
34
35
|
import './chunk-L5ORED7V.mjs';
|
|
35
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
36
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
36
37
|
import { decode } from '@msgpack/msgpack';
|
|
37
38
|
|
|
38
39
|
// src/engine/export/json-line.ts
|