@delali/narsil 0.3.1-next.95 → 0.3.1-next.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/{chunk-3U636ADD.mjs → chunk-CN4I4FZU.mjs} +2 -2
  9. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  10. package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
  11. package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-NAFT3K6X.mjs} +240 -135
  12. package/dist/browser/{chunk-LULHQXCK.mjs → chunk-NNG3EEB6.mjs} +5 -4
  13. package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
  14. package/dist/browser/chunk-RELZRJNR.mjs +203 -0
  15. package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
  16. package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
  17. package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
  18. package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
  19. package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
  20. package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-J57MZK6L.mjs} +115 -61
  21. package/dist/browser/{write-ZUBDNLCH.mjs → write-L7FAQA47.mjs} +6 -4
  22. package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
  23. package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
  24. package/dist/{chunk-7QONURHW.mjs → chunk-45AVKHMJ.mjs} +2 -2
  25. package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
  26. package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
  27. package/dist/{chunk-54HYC5FT.mjs → chunk-BKB3XTMI.mjs} +6 -6
  28. package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
  29. package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
  30. package/dist/chunk-FMWRCP2C.mjs +57 -0
  31. package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
  32. package/dist/chunk-HJSN7UYG.mjs +10 -0
  33. package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
  34. package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
  35. package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
  36. package/dist/chunk-K7D4ZTSW.mjs +126 -0
  37. package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
  38. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  39. package/dist/{chunk-J7SDAOTP.mjs → chunk-L3L5YFXC.mjs} +4 -4
  40. package/dist/{chunk-Q4UTU5ZE.mjs → chunk-LH4NZNJJ.mjs} +10 -10
  41. package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
  42. package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
  43. package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
  44. package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
  45. package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
  46. package/dist/{chunk-L26LEDHO.mjs → chunk-NTRMFARG.mjs} +7 -7
  47. package/dist/chunk-NW4XDSP2.mjs +212 -0
  48. package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
  49. package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
  50. package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
  51. package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
  52. package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
  53. package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
  54. package/dist/{chunk-JFUE6F2Y.mjs → chunk-VWULPWRJ.mjs} +60 -21
  55. package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
  56. package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
  57. package/dist/{chunk-MASE2LTJ.mjs → chunk-ZG477IPV.mjs} +7 -7
  58. package/dist/chunk-ZYNAMIYX.mjs +113 -0
  59. package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
  60. package/dist/client.d.ts +9 -9
  61. package/dist/client.mjs +2 -0
  62. package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
  63. package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
  64. package/dist/direct-executor-CXZ66ACP.mjs +22 -0
  65. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  66. package/dist/distribution/coordinator/etcd.mjs +1 -1
  67. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  68. package/dist/distribution/transport/grpc.mjs +1 -1
  69. package/dist/distribution.d.ts +24 -12
  70. package/dist/distribution.mjs +177 -54
  71. package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
  72. package/dist/embeddings/openai.d.ts +2 -2
  73. package/dist/embeddings/openai.mjs +22 -133
  74. package/dist/{engine-BGt9lI1V.d.ts → engine-BGCEm9E_.d.ts} +50 -20
  75. package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
  76. package/dist/index.browser.mjs +4 -2
  77. package/dist/index.d.ts +8 -8
  78. package/dist/index.mjs +19 -18
  79. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  80. package/dist/invalidation/filesystem.d.ts +1 -1
  81. package/dist/invalidation/noop.d.ts +1 -1
  82. package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
  83. package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
  84. package/dist/partial-words.mjs +6 -6
  85. package/dist/pattern.mjs +3 -3
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  87. package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
  88. package/dist/react.d.ts +8 -8
  89. package/dist/react.mjs +2 -0
  90. package/dist/rerank-NWTKK636.mjs +60 -0
  91. package/dist/rerankers/http.d.ts +50 -0
  92. package/dist/rerankers/http.mjs +104 -0
  93. package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-JPRBUYII.mjs} +1 -1
  94. package/dist/runtime-ZKYZFJJC.mjs +177 -0
  95. package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
  96. package/dist/server.d.ts +11 -11
  97. package/dist/server.mjs +63 -22
  98. package/dist/{snapshot-YC6TS4OP.mjs → snapshot-HOGXNO5W.mjs} +138 -83
  99. package/dist/synonyms.mjs +3 -3
  100. package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
  101. package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
  102. package/dist/vector/search-worker.d.ts +6 -2
  103. package/dist/vector/search-worker.mjs +9 -7
  104. package/dist/workers/entry.mjs +5 -5
  105. package/dist/{write-M2OALSA4.mjs → write-XLIGTXBU.mjs} +21 -20
  106. package/package.json +11 -7
  107. package/dist/direct-executor-5KJGVQHM.mjs +0 -22
package/dist/server.mjs CHANGED
@@ -1,27 +1,29 @@
1
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-RUIXODYU.mjs';
2
- export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-RUIXODYU.mjs';
3
- import { validateIndexName } from './chunk-GJZHU4FC.mjs';
1
+ import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-YA7XLE2R.mjs';
2
+ export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-YA7XLE2R.mjs';
3
+ import { rerankOnHost } from './chunk-ZYNAMIYX.mjs';
4
+ import { validateIndexName } from './chunk-2WYMG3OZ.mjs';
4
5
  import './chunk-DJHHF2L5.mjs';
5
6
  import './chunk-P36MUFEJ.mjs';
6
- import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-WLOF6HSP.mjs';
7
+ import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-R7EEKAJE.mjs';
7
8
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
8
- import './chunk-Q5DC6WAB.mjs';
9
- import './chunk-ZHS44YKF.mjs';
9
+ import './chunk-JFRNNBOM.mjs';
10
+ import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
11
+ import './chunk-LXWDQLTW.mjs';
10
12
  import './chunk-TGP5M5R7.mjs';
11
13
  import './chunk-U2BKYEC7.mjs';
12
- import './chunk-ES33L3HO.mjs';
14
+ import './chunk-GGSH6KBQ.mjs';
13
15
  import './chunk-WQBCBKY3.mjs';
14
- import './chunk-4ZJWRETC.mjs';
15
- import './chunk-FGMGUDHX.mjs';
16
+ import './chunk-LXPIXDUJ.mjs';
17
+ import './chunk-E74YLAIG.mjs';
16
18
  import './chunk-KEHK2O6P.mjs';
17
- import './chunk-BB7MASRQ.mjs';
19
+ import './chunk-OOEXLUGR.mjs';
18
20
  import './chunk-7X5JST5L.mjs';
19
21
  import './chunk-R6A773WK.mjs';
20
22
  import './chunk-YTJCGKM4.mjs';
21
23
  import { startIdleHeapCleanup } from './chunk-5XNIRLFC.mjs';
22
24
  import './chunk-L5ORED7V.mjs';
23
- import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-SYWGORQB.mjs';
24
- export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-SYWGORQB.mjs';
25
+ import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
26
+ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
25
27
  import { randomUUID } from 'crypto';
26
28
  import { createReadStream } from 'fs';
27
29
  import { rm } from 'fs/promises';
@@ -813,14 +815,14 @@ function createSearchHandlers(deps) {
813
815
  async function search(ctx) {
814
816
  const params = parseJson(ctx);
815
817
  if (!params) return;
816
- const failure = validateQuery(params, limits.maxResultWindow);
818
+ const failure = validateQuery(params, limits);
817
819
  if (failure) {
818
820
  rejectInvalid(ctx, failure);
819
821
  return;
820
822
  }
821
823
  if (refuseRegex(ctx, params.filters, allowRegex)) return;
822
824
  try {
823
- respondJson(ctx, await engine.query(ctx.params[0], params));
825
+ respondJson(ctx, await engine.query(ctx.params[0], withServerRerankWindow(params, limits)));
824
826
  } catch (err) {
825
827
  respondError(ctx, err);
826
828
  }
@@ -828,7 +830,7 @@ function createSearchHandlers(deps) {
828
830
  async function preflight(ctx) {
829
831
  const params = parseJson(ctx);
830
832
  if (!params) return;
831
- const failure = validateQuery(params, limits.maxResultWindow);
833
+ const failure = validateQuery(params, limits);
832
834
  if (failure) {
833
835
  rejectInvalid(ctx, failure);
834
836
  return;
@@ -868,7 +870,13 @@ var LIMITS_UNBOUNDED_AT_ZERO = [
868
870
  "maxConcurrentRequests",
869
871
  "maxConcurrentTasks"
870
872
  ];
871
- var COUNTED_LIMITS = ["importBatchSize", "maxResultWindow", "maxFetchDocuments", "maxTaskPageSize"];
873
+ var COUNTED_LIMITS = [
874
+ "importBatchSize",
875
+ "maxResultWindow",
876
+ "maxRerankWindow",
877
+ "maxFetchDocuments",
878
+ "maxTaskPageSize"
879
+ ];
872
880
  function assertServerEngine(engine) {
873
881
  const candidate = engine;
874
882
  if (typeof candidate !== "object" || candidate === null) {
@@ -896,10 +904,21 @@ function assertLimit(field, value, minimum) {
896
904
  { field, value, minimum }
897
905
  );
898
906
  }
899
- function assertServerLimits(limits) {
907
+ function assertRerankWindowCap(value, engineCap) {
908
+ if (value === void 0) return;
909
+ const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
910
+ if (value <= cap) return;
911
+ throw new NarsilError(
912
+ ErrorCodes.CONFIG_INVALID,
913
+ `limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
914
+ { field: "maxRerankWindow", value, maximum: cap }
915
+ );
916
+ }
917
+ function assertServerLimits(limits, engineRerankWindow) {
900
918
  if (limits === void 0) return;
901
919
  for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
902
920
  for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
921
+ assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
903
922
  }
904
923
  function assertAllowRegex(allowRegex) {
905
924
  if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
@@ -1064,7 +1083,14 @@ function toRelayedResponse(requestId, captured) {
1064
1083
  }
1065
1084
 
1066
1085
  // src/server/request-threads/host.ts
1067
- var RELAYED_TYPES = /* @__PURE__ */ new Set(["request", "authorize", "embed", "stream-credit", "stream-cancel"]);
1086
+ var RELAYED_TYPES = /* @__PURE__ */ new Set([
1087
+ "request",
1088
+ "authorize",
1089
+ "embed",
1090
+ "rerank",
1091
+ "stream-credit",
1092
+ "stream-cancel"
1093
+ ]);
1068
1094
  function isRelayRequest(message) {
1069
1095
  if (typeof message !== "object" || message === null) return false;
1070
1096
  const { type, requestId } = message;
@@ -1142,6 +1168,15 @@ async function answerEmbed(core, request) {
1142
1168
  return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
1143
1169
  }
1144
1170
  }
1171
+ async function answerRerank(core, request) {
1172
+ try {
1173
+ const scores = await rerankOnHost(core, request.adapter, request.query, request.documents);
1174
+ return { type: "reranked", requestId: request.requestId, scores, error: null };
1175
+ } catch (err) {
1176
+ if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
1177
+ return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
1178
+ }
1179
+ }
1145
1180
  function touchIntervalFor(copyIdleTimeoutMs) {
1146
1181
  return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
1147
1182
  }
@@ -1184,7 +1219,7 @@ async function startRequestThreads(options) {
1184
1219
  streams.get(message.requestId)?.cancel();
1185
1220
  return;
1186
1221
  }
1187
- const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
1222
+ const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "embed" ? answerEmbed(core, message) : answerRerank(core, message);
1188
1223
  void reply.then((answer) => {
1189
1224
  if (answer !== null) port.postMessage(answer);
1190
1225
  });
@@ -1226,6 +1261,7 @@ async function startRequestThreads(options) {
1226
1261
  authorizes: authorize !== void 0,
1227
1262
  searchHooks,
1228
1263
  patternWorkCap: options.core.patternWorkCap,
1264
+ maxRerankWindow: options.core.rerankSettings.maxWindow,
1229
1265
  allowRegex: options.allowRegex
1230
1266
  }
1231
1267
  },
@@ -1490,7 +1526,7 @@ var TaskRegistry = class {
1490
1526
  };
1491
1527
 
1492
1528
  // src/server/create-server.ts
1493
- function resolveLimits(limits) {
1529
+ function resolveLimits(limits, engineRerankWindow) {
1494
1530
  return {
1495
1531
  maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
1496
1532
  maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
@@ -1499,6 +1535,7 @@ function resolveLimits(limits) {
1499
1535
  importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
1500
1536
  maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
1501
1537
  maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
1538
+ maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
1502
1539
  maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
1503
1540
  maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
1504
1541
  maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
@@ -1595,7 +1632,8 @@ var NarsilHttpServer = class {
1595
1632
  stopIdleHeapCleanup = () => void 0;
1596
1633
  constructor(engine, options = {}) {
1597
1634
  assertServerEngine(engine);
1598
- assertServerLimits(options.limits);
1635
+ const engineRerankWindow = rerankWindowCapOf(engine);
1636
+ assertServerLimits(options.limits, engineRerankWindow);
1599
1637
  assertAllowRegex(options.allowRegex);
1600
1638
  this.host = options.host ?? "127.0.0.1";
1601
1639
  this.port = options.port ?? 9876;
@@ -1604,10 +1642,13 @@ var NarsilHttpServer = class {
1604
1642
  for (const [name, adapter] of Object.entries(options.embeddingAdapters ?? {})) {
1605
1643
  engine.registerEmbeddingAdapter(name, adapter);
1606
1644
  }
1645
+ for (const [name, adapter] of Object.entries(options.rerankAdapters ?? {})) {
1646
+ engine.registerRerankAdapter(name, adapter);
1647
+ }
1607
1648
  engineCoreOf(engine)?.orchestrator.shareMainThread();
1608
1649
  const taskStore = options.taskStore ?? new InMemoryTaskStore();
1609
1650
  const instanceId = options.instanceId ?? randomUUID();
1610
- const limits = resolveLimits(options.limits);
1651
+ const limits = resolveLimits(options.limits, engineRerankWindow);
1611
1652
  this.deps = {
1612
1653
  engine,
1613
1654
  tasks: new TaskRegistry(taskStore, instanceId, limits.maxConcurrentTasks),
@@ -1,47 +1,49 @@
1
- import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-54HYC5FT.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-L26LEDHO.mjs';
3
- import './chunk-MWT5WCJ5.mjs';
4
- import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-MASE2LTJ.mjs';
5
- import { decodeVectorIndexParts, bytesToVectors } from './chunk-F52QGNZ2.mjs';
6
- import './chunk-ICL2AELD.mjs';
7
- import './chunk-37CD3OKS.mjs';
1
+ import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-BKB3XTMI.mjs';
2
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-NTRMFARG.mjs';
3
+ import './chunk-COVCQ73F.mjs';
4
+ import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-ZG477IPV.mjs';
5
+ import { decodeVectorIndexParts, bytesToVectors } from './chunk-WCJKUGHB.mjs';
6
+ import './chunk-HXJBSHXF.mjs';
8
7
  import './chunk-DJHHF2L5.mjs';
9
- import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-Q4UTU5ZE.mjs';
10
- import './chunk-J7SDAOTP.mjs';
11
- import './chunk-OHP7QR33.mjs';
8
+ import './chunk-37CD3OKS.mjs';
9
+ import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-LH4NZNJJ.mjs';
10
+ import './chunk-L3L5YFXC.mjs';
12
11
  import './chunk-P36MUFEJ.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-JFUE6F2Y.mjs';
14
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-R36MIGBU.mjs';
15
- import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
16
- import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-3LYY62M6.mjs';
17
- import './chunk-CHOGQSHK.mjs';
18
- import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-WLOF6HSP.mjs';
12
+ import './chunk-OHP7QR33.mjs';
13
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-VWULPWRJ.mjs';
14
+ import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-NKRBEGJZ.mjs';
15
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-R7EEKAJE.mjs';
16
+ import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
19
17
  import './chunk-AV4IJKD2.mjs';
20
- import './chunk-Q5DC6WAB.mjs';
21
- import { getLanguage } from './chunk-ZHS44YKF.mjs';
18
+ import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-OWNZYMRE.mjs';
19
+ import './chunk-K25LZA54.mjs';
20
+ import { getLanguage } from './chunk-JFRNNBOM.mjs';
21
+ import './chunk-HJSN7UYG.mjs';
22
+ import './chunk-LXWDQLTW.mjs';
22
23
  import './chunk-TGP5M5R7.mjs';
23
24
  import './chunk-U2BKYEC7.mjs';
24
25
  import './chunk-KOKM4PQN.mjs';
25
26
  import './chunk-QY7GC2MW.mjs';
26
- import './chunk-ES33L3HO.mjs';
27
- import './chunk-QM3VPX4Z.mjs';
27
+ import './chunk-GGSH6KBQ.mjs';
28
+ import './chunk-ASY5PHYI.mjs';
28
29
  import './chunk-WQBCBKY3.mjs';
29
- import { parseVectorDimension } from './chunk-SBCEHIPL.mjs';
30
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-4ZJWRETC.mjs';
31
- import './chunk-FGMGUDHX.mjs';
30
+ import { parseVectorDimension } from './chunk-MHOSCQCQ.mjs';
31
+ import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-LXPIXDUJ.mjs';
32
+ import './chunk-E74YLAIG.mjs';
32
33
  import './chunk-KEHK2O6P.mjs';
33
34
  import './chunk-GES7U2PM.mjs';
34
- import './chunk-BB7MASRQ.mjs';
35
- import { resolveIndexAnalysis } from './chunk-2GZGBOQI.mjs';
35
+ import './chunk-OOEXLUGR.mjs';
36
+ import { resolveIndexAnalysis } from './chunk-RONEOEON.mjs';
36
37
  import './chunk-X5CV2E2B.mjs';
37
38
  import './chunk-7X5JST5L.mjs';
38
- import './chunk-7KC56574.mjs';
39
+ import './chunk-LU2CE5AE.mjs';
39
40
  import './chunk-R6A773WK.mjs';
40
41
  import './chunk-YTJCGKM4.mjs';
42
+ import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
41
43
  import './chunk-R6PR4WYZ.mjs';
42
- import { loadVectorCodeFor } from './chunk-HBVLF3HA.mjs';
44
+ import { loadVectorCodeFor } from './chunk-PPY2GSIB.mjs';
43
45
  import './chunk-L5ORED7V.mjs';
44
- import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
46
+ import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
45
47
  import { decode } from '@msgpack/msgpack';
46
48
 
47
49
  // src/persistence/archive/validate.ts
@@ -84,43 +86,84 @@ async function validateStagedCheckpoint(store, manifest) {
84
86
  }
85
87
 
86
88
  // src/engine/snapshot/constants.ts
87
- var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
88
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
89
- var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
90
- var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
91
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
92
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
93
- var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
94
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
95
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
96
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
97
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
98
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
89
+ var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
90
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
91
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
92
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
93
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
94
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
95
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
96
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
97
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
98
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
99
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
100
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
101
+ var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
102
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
103
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
104
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
105
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
106
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
107
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
108
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
109
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
110
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
111
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
112
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
113
+ var RESTORE_ESTIMATE_MARGIN = 1.25;
99
114
 
100
115
  // src/engine/snapshot/memory-budget.ts
101
116
  var MEGABYTE_BYTES = 1048576;
102
117
  var ARCHIVE_RATES = {
103
- heap: {
104
- perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
105
- perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
106
- perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
118
+ worst: {
119
+ heap: {
120
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
121
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
122
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
123
+ },
124
+ external: {
125
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
126
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
127
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
128
+ }
107
129
  },
108
- external: {
109
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
110
- perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
111
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
130
+ floor: {
131
+ heap: {
132
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
133
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
134
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
135
+ },
136
+ external: {
137
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
138
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
139
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
140
+ }
112
141
  }
113
142
  };
114
143
  var EARLIER_SNAPSHOT_RATES = {
115
- heap: {
116
- perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
117
- perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
118
- perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
144
+ worst: {
145
+ heap: {
146
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
147
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
148
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
149
+ },
150
+ external: {
151
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
152
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
153
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
154
+ }
119
155
  },
120
- external: {
121
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
122
- perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
123
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
156
+ floor: {
157
+ heap: {
158
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
159
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
160
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
161
+ },
162
+ external: {
163
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
164
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
165
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
166
+ }
124
167
  }
125
168
  };
126
169
  function bytesAt(rate, storedBytes, vectors) {
@@ -128,10 +171,12 @@ function bytesAt(rate, storedBytes, vectors) {
128
171
  for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
129
172
  return bytes;
130
173
  }
131
- function footprintAt(rates, storedBytes, vectors) {
174
+ function footprintAt(rates, replacesIndex, storedBytes, vectors) {
175
+ const chosen = replacesIndex ? rates.worst : rates.floor;
176
+ const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
132
177
  return {
133
- heapBytes: bytesAt(rates.heap, storedBytes, vectors),
134
- externalBytes: bytesAt(rates.external, storedBytes, vectors)
178
+ heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
179
+ externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
135
180
  };
136
181
  }
137
182
  function deadCountOf(dead) {
@@ -142,7 +187,7 @@ function deadCountOf(dead) {
142
187
  }
143
188
  return count;
144
189
  }
145
- function archiveFootprint(prepared, manifest) {
190
+ function archiveFootprint(prepared, manifest, replacesIndex) {
146
191
  let segmentBytes = 0;
147
192
  for (const partition of manifest.partitions) {
148
193
  for (const segment of partition.segments) segmentBytes += segment.bytes;
@@ -153,36 +198,46 @@ function archiveFootprint(prepared, manifest) {
153
198
  const dimension = dimensions.get(field.fieldPath) ?? 0;
154
199
  for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
155
200
  }
156
- return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
201
+ return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
157
202
  }
158
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
203
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
159
204
  const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
160
205
  const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
161
- return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
206
+ return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
162
207
  }
163
208
  function inMegabytes(bytes) {
164
209
  return Math.ceil(bytes / MEGABYTE_BYTES);
165
210
  }
166
- function refuse(indexName, budget, neededBytes, availableBytes) {
167
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
168
- throw new NarsilError(
169
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
170
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
171
- { indexName, budget, neededBytes, availableBytes }
172
- );
173
- }
174
- function requireRestoreMemory(indexName, footprint) {
211
+ function shortfallOf(heapBytes, processBytes, countInUse) {
175
212
  const heap = readHeapStatistics();
176
213
  if (heap !== null) {
177
- const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
178
- if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
214
+ const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
215
+ const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
216
+ if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
179
217
  }
180
218
  const resident = readResidentMemory();
181
219
  if (resident !== null) {
182
- const neededBytes = footprint.heapBytes + footprint.externalBytes;
183
- const availableBytes = resident.limitBytes - resident.residentBytes;
184
- if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
220
+ const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
221
+ if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
185
222
  }
223
+ return null;
224
+ }
225
+ function refuse(indexName, replacesIndex, shortfall) {
226
+ const { budget, neededBytes, availableBytes } = shortfall;
227
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
228
+ const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
229
+ throw new NarsilError(
230
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
231
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
232
+ { indexName, budget, neededBytes, availableBytes }
233
+ );
234
+ }
235
+ function requireRestoreMemory(indexName, footprint, replacesIndex) {
236
+ const processBytes = footprint.heapBytes + footprint.externalBytes;
237
+ if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
238
+ const collected = collectAndReturnPages();
239
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
240
+ if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
186
241
  }
187
242
 
188
243
  // src/workers/detached-index.ts
@@ -483,9 +538,8 @@ async function stageArchive(indexName, reader, staging, deps) {
483
538
  manifest: async (manifest2, sourceName) => {
484
539
  if (staged.prepared === null) missingPart("configuration");
485
540
  requireSameSchema(staged.prepared, manifest2);
486
- if (deps.indexRegistry.has(indexName)) {
487
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
488
- }
541
+ const replacesIndex = deps.indexRegistry.has(indexName);
542
+ requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
489
543
  staged.manifest = renamed(manifest2, sourceName, indexName);
490
544
  },
491
545
  file: async (entryName, _ref, bytes) => {
@@ -575,7 +629,7 @@ function vectorsOfSnapshot(envelope) {
575
629
  }
576
630
  return byDocument;
577
631
  }
578
- function footprintOf(envelope, prepared) {
632
+ function footprintOf(envelope, prepared, replacesIndex) {
579
633
  let partitionBytes = 0;
580
634
  for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
581
635
  const vectorCounts = /* @__PURE__ */ new Map();
@@ -584,7 +638,7 @@ function footprintOf(envelope, prepared) {
584
638
  for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
585
639
  vectorCounts.set(fieldPath, count);
586
640
  }
587
- return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
641
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
588
642
  }
589
643
  function documentsOfPartition(envelope, payload) {
590
644
  return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
@@ -644,7 +698,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
644
698
  async function restoreEarlierSnapshot(indexName, data, deps) {
645
699
  const envelope = await decodeEarlierSnapshot(data);
646
700
  const prepared = prepareRestoredIndex(envelope, deps);
647
- if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
701
+ const replacesIndex = deps.indexRegistry.has(indexName);
702
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
648
703
  const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
649
704
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
650
705
  try {
package/dist/synonyms.mjs CHANGED
@@ -1,10 +1,10 @@
1
- import { synonymSearchModule } from './chunk-2GZGBOQI.mjs';
1
+ import { synonymSearchModule } from './chunk-RONEOEON.mjs';
2
2
  import './chunk-X5CV2E2B.mjs';
3
3
  import './chunk-7X5JST5L.mjs';
4
- import './chunk-7KC56574.mjs';
4
+ import './chunk-LU2CE5AE.mjs';
5
5
  import './chunk-R6A773WK.mjs';
6
6
  import './chunk-YTJCGKM4.mjs';
7
- import './chunk-SYWGORQB.mjs';
7
+ import './chunk-KV3AZFT3.mjs';
8
8
 
9
9
  // src/synonyms/index.ts
10
10
  var synonyms = synonymSearchModule;
@@ -1,4 +1,4 @@
1
- import { M as MemoryStats } from './memory-COdoGthn.js';
1
+ import { M as MemoryStats } from './memory-CUEDwr-F.js';
2
2
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
3
3
 
4
4
  /** Declarative index configuration accepted over HTTP. Function-valued engine