@delali/narsil 0.3.1-next.111 → 0.3.1-next.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/browser/{chunk-6YUUEMQ4.mjs → chunk-3YPLBK4C.mjs} +2 -2
  2. package/dist/browser/{chunk-5BQPBG3G.mjs → chunk-4FVSBSXY.mjs} +1 -1
  3. package/dist/browser/{chunk-HQOYKRMB.mjs → chunk-DC2SZO4D.mjs} +183 -44
  4. package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
  5. package/dist/browser/{durability-wiring-CYAJBYCO.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
  6. package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
  7. package/dist/browser/{restore-queue-RTZGJT4D.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
  8. package/dist/browser/{snapshot-BU6I25CW.mjs → snapshot-BE3F2PNC.mjs} +6 -5
  9. package/dist/browser/{write-RUPYHPBZ.mjs → write-MV7K7OCU.mjs} +2 -2
  10. package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
  11. package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
  12. package/dist/{chunk-6S7YRUPQ.mjs → chunk-27ZEY73B.mjs} +4 -4
  13. package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
  14. package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
  15. package/dist/{chunk-R2MRMH6M.mjs → chunk-5QIRLJH4.mjs} +3 -3
  16. package/dist/{chunk-7JOONQ24.mjs → chunk-5WEBVYOA.mjs} +176 -38
  17. package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
  18. package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
  19. package/dist/{chunk-JBPYFYIU.mjs → chunk-DZXGNLK5.mjs} +3 -3
  20. package/dist/chunk-EBIHRL7B.mjs +21 -0
  21. package/dist/{chunk-UGKO3RG2.mjs → chunk-GLZI636G.mjs} +1 -1
  22. package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
  23. package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
  24. package/dist/chunk-IWVEG5CX.mjs +67 -0
  25. package/dist/{chunk-SXVDBHYJ.mjs → chunk-JUAFWJON.mjs} +3 -3
  26. package/dist/{chunk-PKE2PCTS.mjs → chunk-LVNBY2TS.mjs} +1 -1
  27. package/dist/{chunk-YLE75NQ5.mjs → chunk-OABTAUCW.mjs} +2 -2
  28. package/dist/{chunk-YSR5L3XJ.mjs → chunk-OM5G2J6X.mjs} +1 -1
  29. package/dist/{chunk-IW6UR3AF.mjs → chunk-PABYSTKO.mjs} +13 -10
  30. package/dist/chunk-PELHVTO7.mjs +24 -0
  31. package/dist/{chunk-CUVTRLA4.mjs → chunk-QGVMEMCG.mjs} +4 -4
  32. package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
  33. package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
  34. package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
  35. package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
  36. package/dist/client.d.ts +6 -6
  37. package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
  38. package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
  39. package/dist/{direct-executor-QFAUKTCQ.mjs → direct-executor-R4HOSVPE.mjs} +6 -5
  40. package/dist/distribution.d.ts +5 -5
  41. package/dist/distribution.mjs +74 -37
  42. package/dist/{durability-wiring-NUHTYXHA.mjs → durability-wiring-XZI6355I.mjs} +15 -14
  43. package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
  44. package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
  45. package/dist/index.browser.mjs +1 -1
  46. package/dist/index.d.ts +5 -5
  47. package/dist/index.mjs +8 -7
  48. package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
  49. package/dist/persistence/durability/checkpoint-worker.mjs +12 -11
  50. package/dist/react.d.ts +6 -6
  51. package/dist/{restore-queue-QSLGXZJB.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
  52. package/dist/server.d.ts +8 -8
  53. package/dist/server.mjs +20 -62
  54. package/dist/{snapshot-IV3S2TMI.mjs → snapshot-PDUGHQ25.mjs} +16 -15
  55. package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
  56. package/dist/{thread-THLCKMEB.mjs → thread-S4SUCV4Y.mjs} +10 -7
  57. package/dist/vector/search-worker.mjs +2 -2
  58. package/dist/workers/entry.mjs +6 -6
  59. package/dist/{write-VGRTOS7N.mjs → write-KVJ7YJJ6.mjs} +9 -8
  60. package/package.json +7 -7
@@ -1,10 +1,11 @@
1
- import { readRecentMemoryFigures, readMemoryFigures, collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, answerRefusalOf, reservedHeapBytes, admitHeapBytes } from './chunk-V7G627TA.mjs';
1
+ import { readRecentMemoryFigures, readMemoryFigures, answerRefusalOf } from './chunk-HFJXEFFV.mjs';
2
+ import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
2
3
  import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
3
4
  import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
4
5
  import { loadNativeCore } from './chunk-VK45J47R.mjs';
5
6
  import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
6
7
  import { MAX_SORT_FIELDS } from './chunk-E74YLAIG.mjs';
7
- import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXZ26WSO.mjs';
8
+ import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
8
9
  import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-OBWUWZRA.mjs';
9
10
  import { randomUUID } from 'crypto';
10
11
  import { mkdtemp, open, rm } from 'fs/promises';
@@ -301,6 +302,7 @@ var DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 0.05;
301
302
  var MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 1;
302
303
  var PARSE_HEAP_BLOCK_FRACTION = 0.9;
303
304
  var PARSE_HEAP_READ_BLOCK_FRACTION = 0.95;
305
+ var BATCH_KIND_SCAN_BYTES = 65536;
304
306
  var PARSE_WORST_HEAP_BYTES_PER_BODY_BYTE = 24;
305
307
  var PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE = 2;
306
308
  var PARSE_OBJECT_HEAP_BYTES = 72;
@@ -384,25 +386,34 @@ function scannedParseBytes(raw) {
384
386
  }
385
387
  var WRITE_PARSE_LINE = {
386
388
  heap: Math.min(PARSE_HEAP_BLOCK_FRACTION, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
387
- process: PARSE_HEAP_BLOCK_FRACTION
389
+ process: PARSE_HEAP_BLOCK_FRACTION,
390
+ holdsYoungReserve: true
388
391
  };
389
- var READ_PARSE_LINE = { heap: PARSE_HEAP_READ_BLOCK_FRACTION, process: PARSE_HEAP_READ_BLOCK_FRACTION };
392
+ var READ_PARSE_LINE = {
393
+ heap: PARSE_HEAP_READ_BLOCK_FRACTION,
394
+ process: PARSE_HEAP_READ_BLOCK_FRACTION,
395
+ holdsYoungReserve: false
396
+ };
397
+ function processRoomIn(figures, line) {
398
+ const reserve = line.holdsYoungReserve ? figures.youngReserveBytes : 0;
399
+ return line.process * figures.processLimitBytes - figures.processBytes - reserve;
400
+ }
390
401
  function roomIn(figures, line) {
391
402
  return Math.min(
392
403
  line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes,
393
- line.process * figures.processLimitBytes - figures.processBytes
404
+ processRoomIn(figures, line)
394
405
  ) - reservedHeapBytes();
395
406
  }
396
- function refusal(status, raw, parseBytes, figures, collectedGarbage, line) {
407
+ function refusal(status, bodyBytes, parseBytes, figures, collectedGarbage, line) {
397
408
  const blockBytes = Math.floor(line.heap * figures.heapLimitBytes);
398
- const message = status === 413 ? `Parsing this ${raw.length}-byte body can take up to ${parseBytes} bytes of memory, more than the ${blockBytes} bytes that the server allows its heap even when idle. Send the request in smaller parts.` : `Parsing this ${raw.length}-byte body can take up to ${parseBytes} bytes of memory, which with the ${figures.heapBytes} bytes already in use would pass the server's limit of ${blockBytes} bytes. Retry once the requests in progress finish.`;
409
+ const message = status === 413 ? `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, more than the ${blockBytes} bytes that the server allows its heap even when idle. Send the request in smaller parts.` : `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, which with the ${figures.heapBytes} bytes already in use would pass the server's limit of ${blockBytes} bytes. Retry once the requests in progress finish.`;
399
410
  return {
400
411
  status,
401
412
  code: status === 413 ? ServerErrorCodes.PAYLOAD_TOO_LARGE : ServerErrorCodes.TOO_MANY_REQUESTS,
402
413
  message,
403
414
  details: {
404
415
  reason: status === 413 ? PARSE_EXCEEDS_MEMORY_REASON : PARSE_WOULD_FILL_MEMORY_REASON,
405
- bodyBytes: raw.length,
416
+ bodyBytes,
406
417
  parseBytes,
407
418
  heapBytes: figures.heapBytes,
408
419
  youngHeapBytes: figures.youngHeapBytes,
@@ -424,7 +435,7 @@ function parseRefusalOf(raw, kind = "write") {
424
435
  if (worstBytes <= roomIn(recent, line)) return admitted(worstBytes);
425
436
  const parseBytes = Math.min(scannedParseBytes(raw), worstBytes);
426
437
  if (parseBytes > line.heap * recent.heapLimitBytes) {
427
- return refusal(413, raw, parseBytes, recent, false, line);
438
+ return refusal(413, raw.length, parseBytes, recent, false, line);
428
439
  }
429
440
  if (parseBytes <= roomIn(recent, line)) return admitted(parseBytes);
430
441
  const figures = readMemoryFigures() ?? recent;
@@ -434,7 +445,31 @@ function parseRefusalOf(raw, kind = "write") {
434
445
  const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
435
446
  if (parseBytes <= roomIn(settled, line)) return admitted(parseBytes);
436
447
  if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
437
- return refusal(429, raw, parseBytes, settled, collectedGarbage, line);
448
+ return refusal(429, raw.length, parseBytes, settled, collectedGarbage, line);
449
+ }
450
+ function bodyFits(bodyBytes, figures, line) {
451
+ const reserved = reservedHeapBytes();
452
+ const heapRoom = line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes - reserved;
453
+ const processRoom = processRoomIn(figures, line) - reserved;
454
+ return bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE <= heapRoom && bodyBytes * (PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE + 1) <= processRoom;
455
+ }
456
+ function bodyRefusalOf(bodyBytes, kind) {
457
+ if (bodyBytes <= 0) return null;
458
+ const line = kind === "read" ? READ_PARSE_LINE : WRITE_PARSE_LINE;
459
+ const recent = readRecentMemoryFigures();
460
+ if (recent === null || bodyFits(bodyBytes, recent, line)) return null;
461
+ const leastParseBytes = bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE;
462
+ if (leastParseBytes > line.heap * recent.heapLimitBytes) {
463
+ return refusal(413, bodyBytes, leastParseBytes, recent, false, line);
464
+ }
465
+ const figures = readMemoryFigures() ?? recent;
466
+ if (bodyFits(bodyBytes, figures, line)) return null;
467
+ const collectedGarbage = collectGarbageBeforeRefusal();
468
+ const collectedYoung = !collectedGarbage && collectYoungGenerationBeforeRefusal();
469
+ const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
470
+ if (bodyFits(bodyBytes, settled, line)) return null;
471
+ if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
472
+ return refusal(429, bodyBytes, leastParseBytes, settled, collectedGarbage, line);
438
473
  }
439
474
 
440
475
  // src/json-encoding.ts
@@ -573,18 +608,18 @@ function sendRelayed(res, abort, status, headers, body) {
573
608
 
574
609
  // src/server/handler-utils.ts
575
610
  function refusesParse(ctx, raw, kind) {
576
- const refusal2 = parseRefusalOf(raw, kind);
611
+ const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw, kind));
577
612
  if (refusal2 === null) return false;
578
613
  if (!ctx.abort.aborted) sendError(ctx.res, refusal2.status, refusal2.code, refusal2.message, refusal2.details);
579
614
  return true;
580
615
  }
581
616
  function parseJson(ctx, kind = "write") {
582
617
  const raw = ctx.rawBody;
618
+ if (refusesParse(ctx, raw, kind)) return void 0;
583
619
  if (!raw || raw.length === 0) {
584
620
  sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
585
621
  return void 0;
586
622
  }
587
- if (refusesParse(ctx, raw, kind)) return void 0;
588
623
  try {
589
624
  return JSON.parse(raw.toString("utf8"));
590
625
  } catch {
@@ -594,11 +629,11 @@ function parseJson(ctx, kind = "write") {
594
629
  }
595
630
  async function parseWriteJson(ctx, writeBlock) {
596
631
  const raw = ctx.rawBody;
597
- if (!raw || raw.length === 0 || writeBlock === void 0) return parseJson(ctx);
598
- const refusal2 = parseRefusalOf(raw);
632
+ if (writeBlock === void 0 || ctx.bodyRefusal === null && (!raw || raw.length === 0)) return parseJson(ctx);
633
+ const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw));
599
634
  if (refusal2 === null) {
600
635
  try {
601
- return JSON.parse(raw.toString("utf8"));
636
+ return JSON.parse(raw?.toString("utf8") ?? "");
602
637
  } catch {
603
638
  sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
604
639
  return void 0;
@@ -611,8 +646,8 @@ async function parseWriteJson(ctx, writeBlock) {
611
646
  }
612
647
  function parseJsonOptional(ctx) {
613
648
  const raw = ctx.rawBody;
614
- if (!raw || raw.length === 0) return {};
615
649
  if (refusesParse(ctx, raw, "write")) return void 0;
650
+ if (!raw || raw.length === 0) return {};
616
651
  try {
617
652
  return JSON.parse(raw.toString("utf8"));
618
653
  } catch {
@@ -1104,6 +1139,111 @@ function createVersionHandler(deps) {
1104
1139
  }
1105
1140
  return { report };
1106
1141
  }
1142
+
1143
+ // src/server/read-body.ts
1144
+ var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
1145
+ function bodyWithRoomFor(body, filled, needed, maxBytes) {
1146
+ if (needed <= body.length) return body;
1147
+ const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
1148
+ grown.set(body.subarray(0, filled));
1149
+ return grown;
1150
+ }
1151
+ function bodyOfExactly(body, filled) {
1152
+ if (filled === body.length) return body;
1153
+ const exact = Buffer.allocUnsafeSlow(filled);
1154
+ exact.set(body.subarray(0, filled));
1155
+ return exact;
1156
+ }
1157
+ var NO_BYTES = new Uint8Array(0);
1158
+ var RefusedBody = class extends Error {
1159
+ constructor(refusal2) {
1160
+ super(refusal2.message);
1161
+ this.refusal = refusal2;
1162
+ }
1163
+ };
1164
+ function refuseBodyBytes(res, refusal2) {
1165
+ sendError(
1166
+ res,
1167
+ 429,
1168
+ ServerErrorCodes.TOO_MANY_REQUESTS,
1169
+ `The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
1170
+ { reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
1171
+ );
1172
+ }
1173
+ function readBody(res, maxBytes, abort, hold, declaredBytes = null, kindOf = null) {
1174
+ return new Promise((resolve, reject) => {
1175
+ if (abort.aborted) {
1176
+ reject(new Error("aborted"));
1177
+ return;
1178
+ }
1179
+ let done = false;
1180
+ let total = 0;
1181
+ let filled = 0;
1182
+ let body = null;
1183
+ const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
1184
+ let kind = kindOf?.(NO_BYTES) ?? null;
1185
+ const memoryRefusal = kindOf === null ? null : bodyRefusalOf(reserved, kind ?? "read");
1186
+ if (memoryRefusal !== null) {
1187
+ res.onData(() => void 0);
1188
+ reject(new RefusedBody(memoryRefusal));
1189
+ return;
1190
+ }
1191
+ const refusal2 = hold.grow(reserved);
1192
+ if (refusal2 !== null) {
1193
+ refuseBodyBytes(res, refusal2);
1194
+ reject(new Error("request bytes in flight"));
1195
+ return;
1196
+ }
1197
+ abort.onAbort(() => {
1198
+ if (done) return;
1199
+ done = true;
1200
+ reject(new Error("aborted"));
1201
+ });
1202
+ res.onData((chunk, isLast) => {
1203
+ if (done || abort.aborted) return;
1204
+ total += chunk.byteLength;
1205
+ if (total > maxBytes) {
1206
+ done = true;
1207
+ if (!abort.aborted) {
1208
+ sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
1209
+ }
1210
+ reject(new Error("payload too large"));
1211
+ return;
1212
+ }
1213
+ const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
1214
+ if (chunkRefusal !== null) {
1215
+ done = true;
1216
+ if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
1217
+ reject(new Error("request bytes in flight"));
1218
+ return;
1219
+ }
1220
+ const streamRefusal = kindOf !== null && declaredBytes === null ? bodyRefusalOf(total, kind ?? "read") : null;
1221
+ if (streamRefusal !== null) {
1222
+ done = true;
1223
+ reject(new RefusedBody(streamRefusal));
1224
+ return;
1225
+ }
1226
+ const arrived = new Uint8Array(chunk);
1227
+ const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
1228
+ body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
1229
+ body.set(arrived, filled);
1230
+ filled = total;
1231
+ if (kindOf !== null && kind === null) {
1232
+ kind = kindOf(body.subarray(0, filled));
1233
+ const kindRefusal = kind === null ? null : bodyRefusalOf(declaredBytes ?? total, kind);
1234
+ if (kindRefusal !== null) {
1235
+ done = true;
1236
+ reject(new RefusedBody(kindRefusal));
1237
+ return;
1238
+ }
1239
+ }
1240
+ if (isLast) {
1241
+ done = true;
1242
+ resolve(bodyOfExactly(body, filled));
1243
+ }
1244
+ });
1245
+ });
1246
+ }
1107
1247
  var NotAdmittedError = class extends Error {
1108
1248
  };
1109
1249
  async function removeDirectory(directory) {
@@ -1187,7 +1327,6 @@ function spoolBody(res, maxBytes, abort, admitted2) {
1187
1327
  }
1188
1328
 
1189
1329
  // src/server/request.ts
1190
- var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
1191
1330
  function initAbortHandler(res) {
1192
1331
  const listeners = [];
1193
1332
  let aborted = false;
@@ -1208,79 +1347,6 @@ function initAbortHandler(res) {
1208
1347
  function declaredBodyBytes(header) {
1209
1348
  return /^\d{1,15}$/.test(header) ? Number(header) : null;
1210
1349
  }
1211
- function bodyWithRoomFor(body, filled, needed, maxBytes) {
1212
- if (needed <= body.length) return body;
1213
- const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
1214
- grown.set(body.subarray(0, filled));
1215
- return grown;
1216
- }
1217
- function bodyOfExactly(body, filled) {
1218
- if (filled === body.length) return body;
1219
- const exact = Buffer.allocUnsafeSlow(filled);
1220
- exact.set(body.subarray(0, filled));
1221
- return exact;
1222
- }
1223
- function refuseBodyBytes(res, refusal2) {
1224
- sendError(
1225
- res,
1226
- 429,
1227
- ServerErrorCodes.TOO_MANY_REQUESTS,
1228
- `The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
1229
- { reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
1230
- );
1231
- }
1232
- function readBody(res, maxBytes, abort, hold, declaredBytes = null) {
1233
- return new Promise((resolve, reject) => {
1234
- if (abort.aborted) {
1235
- reject(new Error("aborted"));
1236
- return;
1237
- }
1238
- let done = false;
1239
- let total = 0;
1240
- let filled = 0;
1241
- let body = null;
1242
- const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
1243
- const refusal2 = hold.grow(reserved);
1244
- if (refusal2 !== null) {
1245
- refuseBodyBytes(res, refusal2);
1246
- reject(new Error("request bytes in flight"));
1247
- return;
1248
- }
1249
- abort.onAbort(() => {
1250
- if (done) return;
1251
- done = true;
1252
- reject(new Error("aborted"));
1253
- });
1254
- res.onData((chunk, isLast) => {
1255
- if (done || abort.aborted) return;
1256
- total += chunk.byteLength;
1257
- if (total > maxBytes) {
1258
- done = true;
1259
- if (!abort.aborted) {
1260
- sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
1261
- }
1262
- reject(new Error("payload too large"));
1263
- return;
1264
- }
1265
- const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
1266
- if (chunkRefusal !== null) {
1267
- done = true;
1268
- if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
1269
- reject(new Error("request bytes in flight"));
1270
- return;
1271
- }
1272
- const arrived = new Uint8Array(chunk);
1273
- const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
1274
- body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
1275
- body.set(arrived, filled);
1276
- filled = total;
1277
- if (isLast) {
1278
- done = true;
1279
- resolve(bodyOfExactly(body, filled));
1280
- }
1281
- });
1282
- });
1283
- }
1284
1350
  function decodePathParameter(raw) {
1285
1351
  if (!raw.includes("%")) return raw;
1286
1352
  try {
@@ -1420,7 +1486,12 @@ function createRouteRunner(deps) {
1420
1486
  }
1421
1487
  const release = track?.(res, abort);
1422
1488
  const bodyHold = bodyBudget.hold();
1423
- const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, bodyHold, declaredBytes) : Promise.resolve(null);
1489
+ let bodyRefusal = null;
1490
+ const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, bodyHold, declaredBytes, opts.bodyKind ?? null).catch((err) => {
1491
+ if (!(err instanceof RefusedBody)) throw err;
1492
+ bodyRefusal = err.refusal;
1493
+ return null;
1494
+ }) : Promise.resolve(null);
1424
1495
  const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
1425
1496
  const spoolPromise = opts.spoolsBody === true ? spoolBody(
1426
1497
  res,
@@ -1448,7 +1519,7 @@ function createRouteRunner(deps) {
1448
1519
  return;
1449
1520
  }
1450
1521
  try {
1451
- await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
1522
+ await handler({ res, params, query, contentType, rawBody, bodyRefusal, spooledBody, abort, hookContext });
1452
1523
  } catch (err) {
1453
1524
  if (!abort.aborted) {
1454
1525
  sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
@@ -1463,7 +1534,99 @@ function createRouteRunner(deps) {
1463
1534
  };
1464
1535
  }
1465
1536
 
1537
+ // src/server/batch-scan.ts
1538
+ var QUOTE2 = 34;
1539
+ var BACKSLASH2 = 92;
1540
+ var OPEN_OBJECT2 = 123;
1541
+ var CLOSE_OBJECT = 125;
1542
+ var OPEN_ARRAY2 = 91;
1543
+ var CLOSE_ARRAY = 93;
1544
+ var COLON2 = 58;
1545
+ var REMOVE_BATCH_KEYS = /* @__PURE__ */ new Set(["action", "docIds", "options"]);
1546
+ var REMOVE_ACTION = "delete";
1547
+ function isWhitespace(byte) {
1548
+ return byte === 32 || byte === 9 || byte === 10 || byte === 13;
1549
+ }
1550
+ function nextByteAt(raw, from) {
1551
+ let at = from;
1552
+ while (at < raw.length && isWhitespace(raw[at])) at++;
1553
+ return at;
1554
+ }
1555
+ function plainStringOf(raw, open2, close) {
1556
+ const content = raw.subarray(open2 + 1, close);
1557
+ if (content.includes(BACKSLASH2)) return null;
1558
+ return new TextDecoder().decode(content);
1559
+ }
1560
+ function isRemoveOnlyBatch(raw) {
1561
+ const start = nextByteAt(raw, 0);
1562
+ if (raw[start] !== OPEN_OBJECT2) return false;
1563
+ let depth = 0;
1564
+ let action = null;
1565
+ for (let at = start; at < raw.length; at++) {
1566
+ const byte = raw[at];
1567
+ if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
1568
+ else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
1569
+ else if (byte === QUOTE2) {
1570
+ const close = closingQuoteOf(raw, at + 1);
1571
+ if (close >= raw.length) return false;
1572
+ const afterString = nextByteAt(raw, close + 1);
1573
+ if (depth === 1 && raw[afterString] === COLON2) {
1574
+ const key = plainStringOf(raw, at, close);
1575
+ if (key === null || !REMOVE_BATCH_KEYS.has(key)) return false;
1576
+ if (key === "action") {
1577
+ const value = nextByteAt(raw, afterString + 1);
1578
+ const valueClose = raw[value] === QUOTE2 ? closingQuoteOf(raw, value + 1) : raw.length;
1579
+ action = valueClose < raw.length ? plainStringOf(raw, value, valueClose) : null;
1580
+ if (action === null) return false;
1581
+ at = valueClose;
1582
+ continue;
1583
+ }
1584
+ at = afterString;
1585
+ continue;
1586
+ }
1587
+ at = close;
1588
+ }
1589
+ }
1590
+ return action === REMOVE_ACTION;
1591
+ }
1592
+ function batchBodyKindOf(prefix) {
1593
+ const scanned = prefix.length > BATCH_KIND_SCAN_BYTES ? prefix.subarray(0, BATCH_KIND_SCAN_BYTES) : prefix;
1594
+ const start = nextByteAt(scanned, 0);
1595
+ if (start >= scanned.length) return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
1596
+ if (scanned[start] !== OPEN_OBJECT2) return "write";
1597
+ let depth = 0;
1598
+ for (let at = start; at < scanned.length; at++) {
1599
+ const byte = scanned[at];
1600
+ if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
1601
+ else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
1602
+ else if (byte === QUOTE2) {
1603
+ const close = closingQuoteOf(scanned, at + 1);
1604
+ if (close >= scanned.length) break;
1605
+ const afterString = nextByteAt(scanned, close + 1);
1606
+ if (afterString >= scanned.length) break;
1607
+ if (depth === 1 && scanned[afterString] === COLON2) {
1608
+ const key = plainStringOf(scanned, at, close);
1609
+ if (key === null || !REMOVE_BATCH_KEYS.has(key)) return "write";
1610
+ if (key === "action") {
1611
+ const value = nextByteAt(scanned, afterString + 1);
1612
+ if (value >= scanned.length) break;
1613
+ if (scanned[value] !== QUOTE2) return "write";
1614
+ const valueClose = closingQuoteOf(scanned, value + 1);
1615
+ if (valueClose >= scanned.length) break;
1616
+ return plainStringOf(scanned, value, valueClose) === REMOVE_ACTION ? "read" : "write";
1617
+ }
1618
+ at = afterString;
1619
+ continue;
1620
+ }
1621
+ at = close;
1622
+ }
1623
+ }
1624
+ return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
1625
+ }
1626
+
1466
1627
  // src/server/routes.ts
1628
+ var writeBodyKind = () => "write";
1629
+ var readBodyKind = () => "read";
1467
1630
  function serverRoutes(limits) {
1468
1631
  const { maxBodyBytes, maxImportBytes } = limits;
1469
1632
  const probe = { maxBytes: 0, skipHooks: true };
@@ -1472,6 +1635,9 @@ function serverRoutes(limits) {
1472
1635
  const namedBody = { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes };
1473
1636
  const document = { paramCount: 2, maxBytes: 0 };
1474
1637
  const documentBody = { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes };
1638
+ const writeBody = { ...namedBody, bodyKind: writeBodyKind };
1639
+ const readBody2 = { ...namedBody, bodyKind: readBodyKind };
1640
+ const documentWriteBody = { ...documentBody, bodyKind: writeBodyKind };
1475
1641
  return [
1476
1642
  { method: "get", path: "/livez", handler: "livez", options: probe },
1477
1643
  { method: "get", path: "/readyz", handler: "readyz", options: probe },
@@ -1501,10 +1667,15 @@ function serverRoutes(limits) {
1501
1667
  { method: "post", path: "/indexes/:name/_open", handler: "openIndex", options: named },
1502
1668
  { method: "post", path: "/indexes/:name/_close", handler: "closeIndex", options: named },
1503
1669
  { method: "get", path: "/indexes/:name/count", handler: "count", options: named },
1504
- { method: "post", path: "/indexes/:name/documents", handler: "insert", options: namedBody },
1505
- { method: "post", path: "/indexes/:name/documents/_batch", handler: "batch", options: namedBody },
1506
- { method: "post", path: "/indexes/:name/documents/_list", handler: "list", options: namedBody },
1507
- { method: "post", path: "/indexes/:name/documents/_multi-get", handler: "multiGet", options: namedBody },
1670
+ { method: "post", path: "/indexes/:name/documents", handler: "insert", options: writeBody },
1671
+ {
1672
+ method: "post",
1673
+ path: "/indexes/:name/documents/_batch",
1674
+ handler: "batch",
1675
+ options: { ...namedBody, bodyKind: batchBodyKindOf }
1676
+ },
1677
+ { method: "post", path: "/indexes/:name/documents/_list", handler: "list", options: readBody2 },
1678
+ { method: "post", path: "/indexes/:name/documents/_multi-get", handler: "multiGet", options: readBody2 },
1508
1679
  {
1509
1680
  method: "post",
1510
1681
  path: "/indexes/:name/documents/_import",
@@ -1514,12 +1685,12 @@ function serverRoutes(limits) {
1514
1685
  { method: "post", path: "/indexes/:name/_wait-for-writes", handler: "waitForWrites", options: named },
1515
1686
  { method: "get", path: "/indexes/:name/documents/:id", handler: "get", options: document },
1516
1687
  { method: "get", path: "/indexes/:name/documents/:id/_exists", handler: "exists", options: document },
1517
- { method: "put", path: "/indexes/:name/documents/:id", handler: "put", options: documentBody },
1518
- { method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options: documentBody },
1688
+ { method: "put", path: "/indexes/:name/documents/:id", handler: "put", options: documentWriteBody },
1689
+ { method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options: documentWriteBody },
1519
1690
  { method: "del", path: "/indexes/:name/documents/:id", handler: "remove", options: document },
1520
- { method: "post", path: "/indexes/:name/search", handler: "search", options: namedBody },
1521
- { method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options: namedBody },
1522
- { method: "post", path: "/indexes/:name/suggest", handler: "suggest", options: namedBody },
1691
+ { method: "post", path: "/indexes/:name/search", handler: "search", options: readBody2 },
1692
+ { method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options: readBody2 },
1693
+ { method: "post", path: "/indexes/:name/suggest", handler: "suggest", options: readBody2 },
1523
1694
  { method: "post", path: "/indexes/:name/_checkpoint", handler: "checkpoint", options: named },
1524
1695
  { method: "post", path: "/indexes/:name/_rebuild-analysis", handler: "rebuildAnalysis", options: named },
1525
1696
  { method: "get", path: "/indexes/:name/snapshot", handler: "snapshot", options: named },
@@ -1656,4 +1827,4 @@ async function departureConfirmation(word) {
1656
1827
  if (waiting.async) await waiting.value;
1657
1828
  }
1658
1829
 
1659
- export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, closingQuoteOf, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
1830
+ export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, isRemoveOnlyBatch, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
@@ -1,5 +1,5 @@
1
1
  import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-OCB7TMUQ.mjs';
2
- import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-YIDSHBQS.mjs';
2
+ import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-4XWIW3F3.mjs';
3
3
  import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
4
4
  import { MAX_DOC_ID_LENGTH } from './chunk-LXWDQLTW.mjs';
5
5
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
@@ -82,6 +82,7 @@ var BUILD_CHUNK_SIZE = 100;
82
82
  var BUILD_TURN_HANDOVER_AFTER_MS = 50;
83
83
  var BUILD_RETRY_FIRST_DELAY_MS = 250;
84
84
  var BUILD_RETRY_MAX_DELAY_MS = 3e4;
85
+ var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
85
86
  var REBUILD_REMOVED_RATIO = 0.2;
86
87
  var DRIFT_CHECK_GROWTH_FACTOR = 2;
87
88
  var RECALIBRATION_DRIFT_RATIO = 2.5;
@@ -648,4 +649,4 @@ function recordCheckpoint(state, plan, written) {
648
649
  state.savedSignature = plan.signature;
649
650
  }
650
651
 
651
- export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
652
+ export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
@@ -1,6 +1,6 @@
1
- import { readMetadataEnvelope } from './chunk-SXVDBHYJ.mjs';
2
- import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-6S7YRUPQ.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-PKE2PCTS.mjs';
1
+ import { readMetadataEnvelope } from './chunk-JUAFWJON.mjs';
2
+ import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-27ZEY73B.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-LVNBY2TS.mjs';
4
4
 
5
5
  // src/persistence/durability/recovery.ts
6
6
  async function listPersistedIndexes(directory) {
@@ -0,0 +1,21 @@
1
+ import { V8_YOUNG_GENERATION_BYTES_BY_MAJOR } from './chunk-22YPZVMG.mjs';
2
+
3
+ // src/runtime/engine-threads.ts
4
+ var MEGABYTE_BYTES = 1048576;
5
+ var threadYoungGenerationBytes = 0;
6
+ function youngGenerationBytesOfThisV8() {
7
+ const major = Number(globalThis.process?.versions?.v8?.split(".")[0]);
8
+ return V8_YOUNG_GENERATION_BYTES_BY_MAJOR[major] ?? Math.max(...Object.values(V8_YOUNG_GENERATION_BYTES_BY_MAJOR));
9
+ }
10
+ function countEngineThread(thread, youngGenerationMb) {
11
+ const bytes = youngGenerationMb === void 0 ? youngGenerationBytesOfThisV8() : youngGenerationMb * MEGABYTE_BYTES;
12
+ threadYoungGenerationBytes += bytes;
13
+ thread.once("exit", () => {
14
+ threadYoungGenerationBytes -= bytes;
15
+ });
16
+ }
17
+ function engineThreadYoungGenerationBytes() {
18
+ return threadYoungGenerationBytes;
19
+ }
20
+
21
+ export { countEngineThread, engineThreadYoungGenerationBytes, youngGenerationBytesOfThisV8 };
@@ -1,4 +1,4 @@
1
- import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-7JOONQ24.mjs';
1
+ import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-5WEBVYOA.mjs';
2
2
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
3
3
  import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
4
4
  import { flattenSchema, partialFieldsOf } from './chunk-DG62H7OD.mjs';
@@ -1,4 +1,4 @@
1
- import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-YIDSHBQS.mjs';
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-4XWIW3F3.mjs';
2
2
  import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
4
4
  import { readSync, closeSync, openSync } from 'fs';
@@ -581,7 +581,8 @@ function openHandles(dimension, codeBits, blockBytes) {
581
581
  present: new Uint8Array(handles.present),
582
582
  partitions: new Int32Array(handles.partitions),
583
583
  diskFile: new Int32Array(handles.diskFile),
584
- diskOffset: new Uint32Array(handles.diskOffset)
584
+ diskOffset: new Uint32Array(handles.diskOffset),
585
+ longestDocIdBytes: 0
585
586
  };
586
587
  }
587
588
  function relayout(store) {
@@ -592,7 +593,10 @@ function claimSlot(store, docId, cold) {
592
593
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
594
  ensureOrdinal(store, ordinal, cold);
594
595
  store.diskFile[ordinal] = IN_MEMORY;
596
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
595
597
  appendDocId(store.handles, ordinal, docId);
598
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
599
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
596
600
  return ordinal;
597
601
  }
598
602
  function publishOrdinal(store, ordinal) {
@@ -668,6 +672,19 @@ function ensureOrdinal(store, ordinal, cold) {
668
672
  if (!cold) ensureFloatSlot(store, ordinal);
669
673
  ensureCodeSlot(store, ordinal);
670
674
  }
675
+ function bufferGrowthBytes(buffer, bytes) {
676
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
677
+ }
678
+ function growthBytesFor(store, vectors) {
679
+ const { handles } = store;
680
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
681
+ const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
682
+ if (last < slots) return 0;
683
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
684
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
685
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
686
+ return bytes;
687
+ }
671
688
  function writeVector(store, ordinal, vector) {
672
689
  const block = store.view.blockOf(ordinal);
673
690
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -1004,6 +1021,9 @@ function createVectorStore(options) {
1004
1021
  },
1005
1022
  memoryBytes() {
1006
1023
  return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1024
+ },
1025
+ growthBytesFor(vectors) {
1026
+ return open === null ? 0 : growthBytesFor(open, vectors);
1007
1027
  }
1008
1028
  };
1009
1029
  }