@delali/narsil 0.3.1-next.111 → 0.3.1-next.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-6YUUEMQ4.mjs → chunk-3YPLBK4C.mjs} +2 -2
- package/dist/browser/{chunk-5BQPBG3G.mjs → chunk-4FVSBSXY.mjs} +1 -1
- package/dist/browser/{chunk-HQOYKRMB.mjs → chunk-DC2SZO4D.mjs} +183 -44
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
- package/dist/browser/{durability-wiring-CYAJBYCO.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
- package/dist/browser/{restore-queue-RTZGJT4D.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
- package/dist/browser/{snapshot-BU6I25CW.mjs → snapshot-BE3F2PNC.mjs} +6 -5
- package/dist/browser/{write-RUPYHPBZ.mjs → write-MV7K7OCU.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-6S7YRUPQ.mjs → chunk-27ZEY73B.mjs} +4 -4
- package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
- package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
- package/dist/{chunk-R2MRMH6M.mjs → chunk-5QIRLJH4.mjs} +3 -3
- package/dist/{chunk-7JOONQ24.mjs → chunk-5WEBVYOA.mjs} +176 -38
- package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
- package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
- package/dist/{chunk-JBPYFYIU.mjs → chunk-DZXGNLK5.mjs} +3 -3
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-UGKO3RG2.mjs → chunk-GLZI636G.mjs} +1 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-SXVDBHYJ.mjs → chunk-JUAFWJON.mjs} +3 -3
- package/dist/{chunk-PKE2PCTS.mjs → chunk-LVNBY2TS.mjs} +1 -1
- package/dist/{chunk-YLE75NQ5.mjs → chunk-OABTAUCW.mjs} +2 -2
- package/dist/{chunk-YSR5L3XJ.mjs → chunk-OM5G2J6X.mjs} +1 -1
- package/dist/{chunk-IW6UR3AF.mjs → chunk-PABYSTKO.mjs} +13 -10
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-CUVTRLA4.mjs → chunk-QGVMEMCG.mjs} +4 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-QFAUKTCQ.mjs → direct-executor-R4HOSVPE.mjs} +6 -5
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +74 -37
- package/dist/{durability-wiring-NUHTYXHA.mjs → durability-wiring-XZI6355I.mjs} +15 -14
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +8 -7
- package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +12 -11
- package/dist/react.d.ts +6 -6
- package/dist/{restore-queue-QSLGXZJB.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +20 -62
- package/dist/{snapshot-IV3S2TMI.mjs → snapshot-PDUGHQ25.mjs} +16 -15
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/{thread-THLCKMEB.mjs → thread-S4SUCV4Y.mjs} +10 -7
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-VGRTOS7N.mjs → write-KVJ7YJJ6.mjs} +9 -8
- package/package.json +7 -7
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { readRecentMemoryFigures, readMemoryFigures,
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures, answerRefusalOf } from './chunk-HFJXEFFV.mjs';
|
|
2
|
+
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
2
3
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
4
|
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
5
|
import { loadNativeCore } from './chunk-VK45J47R.mjs';
|
|
5
6
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
6
7
|
import { MAX_SORT_FIELDS } from './chunk-E74YLAIG.mjs';
|
|
7
|
-
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-
|
|
8
|
+
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
|
|
8
9
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
9
10
|
import { randomUUID } from 'crypto';
|
|
10
11
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
@@ -301,6 +302,7 @@ var DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 0.05;
|
|
|
301
302
|
var MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 1;
|
|
302
303
|
var PARSE_HEAP_BLOCK_FRACTION = 0.9;
|
|
303
304
|
var PARSE_HEAP_READ_BLOCK_FRACTION = 0.95;
|
|
305
|
+
var BATCH_KIND_SCAN_BYTES = 65536;
|
|
304
306
|
var PARSE_WORST_HEAP_BYTES_PER_BODY_BYTE = 24;
|
|
305
307
|
var PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE = 2;
|
|
306
308
|
var PARSE_OBJECT_HEAP_BYTES = 72;
|
|
@@ -384,25 +386,34 @@ function scannedParseBytes(raw) {
|
|
|
384
386
|
}
|
|
385
387
|
var WRITE_PARSE_LINE = {
|
|
386
388
|
heap: Math.min(PARSE_HEAP_BLOCK_FRACTION, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
|
|
387
|
-
process: PARSE_HEAP_BLOCK_FRACTION
|
|
389
|
+
process: PARSE_HEAP_BLOCK_FRACTION,
|
|
390
|
+
holdsYoungReserve: true
|
|
388
391
|
};
|
|
389
|
-
var READ_PARSE_LINE = {
|
|
392
|
+
var READ_PARSE_LINE = {
|
|
393
|
+
heap: PARSE_HEAP_READ_BLOCK_FRACTION,
|
|
394
|
+
process: PARSE_HEAP_READ_BLOCK_FRACTION,
|
|
395
|
+
holdsYoungReserve: false
|
|
396
|
+
};
|
|
397
|
+
function processRoomIn(figures, line) {
|
|
398
|
+
const reserve = line.holdsYoungReserve ? figures.youngReserveBytes : 0;
|
|
399
|
+
return line.process * figures.processLimitBytes - figures.processBytes - reserve;
|
|
400
|
+
}
|
|
390
401
|
function roomIn(figures, line) {
|
|
391
402
|
return Math.min(
|
|
392
403
|
line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes,
|
|
393
|
-
|
|
404
|
+
processRoomIn(figures, line)
|
|
394
405
|
) - reservedHeapBytes();
|
|
395
406
|
}
|
|
396
|
-
function refusal(status,
|
|
407
|
+
function refusal(status, bodyBytes, parseBytes, figures, collectedGarbage, line) {
|
|
397
408
|
const blockBytes = Math.floor(line.heap * figures.heapLimitBytes);
|
|
398
|
-
const message = status === 413 ? `Parsing this ${
|
|
409
|
+
const message = status === 413 ? `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, more than the ${blockBytes} bytes that the server allows its heap even when idle. Send the request in smaller parts.` : `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, which with the ${figures.heapBytes} bytes already in use would pass the server's limit of ${blockBytes} bytes. Retry once the requests in progress finish.`;
|
|
399
410
|
return {
|
|
400
411
|
status,
|
|
401
412
|
code: status === 413 ? ServerErrorCodes.PAYLOAD_TOO_LARGE : ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
402
413
|
message,
|
|
403
414
|
details: {
|
|
404
415
|
reason: status === 413 ? PARSE_EXCEEDS_MEMORY_REASON : PARSE_WOULD_FILL_MEMORY_REASON,
|
|
405
|
-
bodyBytes
|
|
416
|
+
bodyBytes,
|
|
406
417
|
parseBytes,
|
|
407
418
|
heapBytes: figures.heapBytes,
|
|
408
419
|
youngHeapBytes: figures.youngHeapBytes,
|
|
@@ -424,7 +435,7 @@ function parseRefusalOf(raw, kind = "write") {
|
|
|
424
435
|
if (worstBytes <= roomIn(recent, line)) return admitted(worstBytes);
|
|
425
436
|
const parseBytes = Math.min(scannedParseBytes(raw), worstBytes);
|
|
426
437
|
if (parseBytes > line.heap * recent.heapLimitBytes) {
|
|
427
|
-
return refusal(413, raw, parseBytes, recent, false, line);
|
|
438
|
+
return refusal(413, raw.length, parseBytes, recent, false, line);
|
|
428
439
|
}
|
|
429
440
|
if (parseBytes <= roomIn(recent, line)) return admitted(parseBytes);
|
|
430
441
|
const figures = readMemoryFigures() ?? recent;
|
|
@@ -434,7 +445,31 @@ function parseRefusalOf(raw, kind = "write") {
|
|
|
434
445
|
const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
|
|
435
446
|
if (parseBytes <= roomIn(settled, line)) return admitted(parseBytes);
|
|
436
447
|
if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
|
|
437
|
-
return refusal(429, raw, parseBytes, settled, collectedGarbage, line);
|
|
448
|
+
return refusal(429, raw.length, parseBytes, settled, collectedGarbage, line);
|
|
449
|
+
}
|
|
450
|
+
function bodyFits(bodyBytes, figures, line) {
|
|
451
|
+
const reserved = reservedHeapBytes();
|
|
452
|
+
const heapRoom = line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes - reserved;
|
|
453
|
+
const processRoom = processRoomIn(figures, line) - reserved;
|
|
454
|
+
return bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE <= heapRoom && bodyBytes * (PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE + 1) <= processRoom;
|
|
455
|
+
}
|
|
456
|
+
function bodyRefusalOf(bodyBytes, kind) {
|
|
457
|
+
if (bodyBytes <= 0) return null;
|
|
458
|
+
const line = kind === "read" ? READ_PARSE_LINE : WRITE_PARSE_LINE;
|
|
459
|
+
const recent = readRecentMemoryFigures();
|
|
460
|
+
if (recent === null || bodyFits(bodyBytes, recent, line)) return null;
|
|
461
|
+
const leastParseBytes = bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE;
|
|
462
|
+
if (leastParseBytes > line.heap * recent.heapLimitBytes) {
|
|
463
|
+
return refusal(413, bodyBytes, leastParseBytes, recent, false, line);
|
|
464
|
+
}
|
|
465
|
+
const figures = readMemoryFigures() ?? recent;
|
|
466
|
+
if (bodyFits(bodyBytes, figures, line)) return null;
|
|
467
|
+
const collectedGarbage = collectGarbageBeforeRefusal();
|
|
468
|
+
const collectedYoung = !collectedGarbage && collectYoungGenerationBeforeRefusal();
|
|
469
|
+
const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
|
|
470
|
+
if (bodyFits(bodyBytes, settled, line)) return null;
|
|
471
|
+
if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
|
|
472
|
+
return refusal(429, bodyBytes, leastParseBytes, settled, collectedGarbage, line);
|
|
438
473
|
}
|
|
439
474
|
|
|
440
475
|
// src/json-encoding.ts
|
|
@@ -573,18 +608,18 @@ function sendRelayed(res, abort, status, headers, body) {
|
|
|
573
608
|
|
|
574
609
|
// src/server/handler-utils.ts
|
|
575
610
|
function refusesParse(ctx, raw, kind) {
|
|
576
|
-
const refusal2 = parseRefusalOf(raw, kind);
|
|
611
|
+
const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw, kind));
|
|
577
612
|
if (refusal2 === null) return false;
|
|
578
613
|
if (!ctx.abort.aborted) sendError(ctx.res, refusal2.status, refusal2.code, refusal2.message, refusal2.details);
|
|
579
614
|
return true;
|
|
580
615
|
}
|
|
581
616
|
function parseJson(ctx, kind = "write") {
|
|
582
617
|
const raw = ctx.rawBody;
|
|
618
|
+
if (refusesParse(ctx, raw, kind)) return void 0;
|
|
583
619
|
if (!raw || raw.length === 0) {
|
|
584
620
|
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
585
621
|
return void 0;
|
|
586
622
|
}
|
|
587
|
-
if (refusesParse(ctx, raw, kind)) return void 0;
|
|
588
623
|
try {
|
|
589
624
|
return JSON.parse(raw.toString("utf8"));
|
|
590
625
|
} catch {
|
|
@@ -594,11 +629,11 @@ function parseJson(ctx, kind = "write") {
|
|
|
594
629
|
}
|
|
595
630
|
async function parseWriteJson(ctx, writeBlock) {
|
|
596
631
|
const raw = ctx.rawBody;
|
|
597
|
-
if (
|
|
598
|
-
const refusal2 = parseRefusalOf(raw);
|
|
632
|
+
if (writeBlock === void 0 || ctx.bodyRefusal === null && (!raw || raw.length === 0)) return parseJson(ctx);
|
|
633
|
+
const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw));
|
|
599
634
|
if (refusal2 === null) {
|
|
600
635
|
try {
|
|
601
|
-
return JSON.parse(raw
|
|
636
|
+
return JSON.parse(raw?.toString("utf8") ?? "");
|
|
602
637
|
} catch {
|
|
603
638
|
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
604
639
|
return void 0;
|
|
@@ -611,8 +646,8 @@ async function parseWriteJson(ctx, writeBlock) {
|
|
|
611
646
|
}
|
|
612
647
|
function parseJsonOptional(ctx) {
|
|
613
648
|
const raw = ctx.rawBody;
|
|
614
|
-
if (!raw || raw.length === 0) return {};
|
|
615
649
|
if (refusesParse(ctx, raw, "write")) return void 0;
|
|
650
|
+
if (!raw || raw.length === 0) return {};
|
|
616
651
|
try {
|
|
617
652
|
return JSON.parse(raw.toString("utf8"));
|
|
618
653
|
} catch {
|
|
@@ -1104,6 +1139,111 @@ function createVersionHandler(deps) {
|
|
|
1104
1139
|
}
|
|
1105
1140
|
return { report };
|
|
1106
1141
|
}
|
|
1142
|
+
|
|
1143
|
+
// src/server/read-body.ts
|
|
1144
|
+
var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
|
|
1145
|
+
function bodyWithRoomFor(body, filled, needed, maxBytes) {
|
|
1146
|
+
if (needed <= body.length) return body;
|
|
1147
|
+
const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
|
|
1148
|
+
grown.set(body.subarray(0, filled));
|
|
1149
|
+
return grown;
|
|
1150
|
+
}
|
|
1151
|
+
function bodyOfExactly(body, filled) {
|
|
1152
|
+
if (filled === body.length) return body;
|
|
1153
|
+
const exact = Buffer.allocUnsafeSlow(filled);
|
|
1154
|
+
exact.set(body.subarray(0, filled));
|
|
1155
|
+
return exact;
|
|
1156
|
+
}
|
|
1157
|
+
var NO_BYTES = new Uint8Array(0);
|
|
1158
|
+
var RefusedBody = class extends Error {
|
|
1159
|
+
constructor(refusal2) {
|
|
1160
|
+
super(refusal2.message);
|
|
1161
|
+
this.refusal = refusal2;
|
|
1162
|
+
}
|
|
1163
|
+
};
|
|
1164
|
+
function refuseBodyBytes(res, refusal2) {
|
|
1165
|
+
sendError(
|
|
1166
|
+
res,
|
|
1167
|
+
429,
|
|
1168
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1169
|
+
`The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
|
|
1170
|
+
{ reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
function readBody(res, maxBytes, abort, hold, declaredBytes = null, kindOf = null) {
|
|
1174
|
+
return new Promise((resolve, reject) => {
|
|
1175
|
+
if (abort.aborted) {
|
|
1176
|
+
reject(new Error("aborted"));
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
let done = false;
|
|
1180
|
+
let total = 0;
|
|
1181
|
+
let filled = 0;
|
|
1182
|
+
let body = null;
|
|
1183
|
+
const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
|
|
1184
|
+
let kind = kindOf?.(NO_BYTES) ?? null;
|
|
1185
|
+
const memoryRefusal = kindOf === null ? null : bodyRefusalOf(reserved, kind ?? "read");
|
|
1186
|
+
if (memoryRefusal !== null) {
|
|
1187
|
+
res.onData(() => void 0);
|
|
1188
|
+
reject(new RefusedBody(memoryRefusal));
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
const refusal2 = hold.grow(reserved);
|
|
1192
|
+
if (refusal2 !== null) {
|
|
1193
|
+
refuseBodyBytes(res, refusal2);
|
|
1194
|
+
reject(new Error("request bytes in flight"));
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
abort.onAbort(() => {
|
|
1198
|
+
if (done) return;
|
|
1199
|
+
done = true;
|
|
1200
|
+
reject(new Error("aborted"));
|
|
1201
|
+
});
|
|
1202
|
+
res.onData((chunk, isLast) => {
|
|
1203
|
+
if (done || abort.aborted) return;
|
|
1204
|
+
total += chunk.byteLength;
|
|
1205
|
+
if (total > maxBytes) {
|
|
1206
|
+
done = true;
|
|
1207
|
+
if (!abort.aborted) {
|
|
1208
|
+
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1209
|
+
}
|
|
1210
|
+
reject(new Error("payload too large"));
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
|
|
1214
|
+
if (chunkRefusal !== null) {
|
|
1215
|
+
done = true;
|
|
1216
|
+
if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
|
|
1217
|
+
reject(new Error("request bytes in flight"));
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1220
|
+
const streamRefusal = kindOf !== null && declaredBytes === null ? bodyRefusalOf(total, kind ?? "read") : null;
|
|
1221
|
+
if (streamRefusal !== null) {
|
|
1222
|
+
done = true;
|
|
1223
|
+
reject(new RefusedBody(streamRefusal));
|
|
1224
|
+
return;
|
|
1225
|
+
}
|
|
1226
|
+
const arrived = new Uint8Array(chunk);
|
|
1227
|
+
const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
|
|
1228
|
+
body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
|
|
1229
|
+
body.set(arrived, filled);
|
|
1230
|
+
filled = total;
|
|
1231
|
+
if (kindOf !== null && kind === null) {
|
|
1232
|
+
kind = kindOf(body.subarray(0, filled));
|
|
1233
|
+
const kindRefusal = kind === null ? null : bodyRefusalOf(declaredBytes ?? total, kind);
|
|
1234
|
+
if (kindRefusal !== null) {
|
|
1235
|
+
done = true;
|
|
1236
|
+
reject(new RefusedBody(kindRefusal));
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
if (isLast) {
|
|
1241
|
+
done = true;
|
|
1242
|
+
resolve(bodyOfExactly(body, filled));
|
|
1243
|
+
}
|
|
1244
|
+
});
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1107
1247
|
var NotAdmittedError = class extends Error {
|
|
1108
1248
|
};
|
|
1109
1249
|
async function removeDirectory(directory) {
|
|
@@ -1187,7 +1327,6 @@ function spoolBody(res, maxBytes, abort, admitted2) {
|
|
|
1187
1327
|
}
|
|
1188
1328
|
|
|
1189
1329
|
// src/server/request.ts
|
|
1190
|
-
var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
|
|
1191
1330
|
function initAbortHandler(res) {
|
|
1192
1331
|
const listeners = [];
|
|
1193
1332
|
let aborted = false;
|
|
@@ -1208,79 +1347,6 @@ function initAbortHandler(res) {
|
|
|
1208
1347
|
function declaredBodyBytes(header) {
|
|
1209
1348
|
return /^\d{1,15}$/.test(header) ? Number(header) : null;
|
|
1210
1349
|
}
|
|
1211
|
-
function bodyWithRoomFor(body, filled, needed, maxBytes) {
|
|
1212
|
-
if (needed <= body.length) return body;
|
|
1213
|
-
const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
|
|
1214
|
-
grown.set(body.subarray(0, filled));
|
|
1215
|
-
return grown;
|
|
1216
|
-
}
|
|
1217
|
-
function bodyOfExactly(body, filled) {
|
|
1218
|
-
if (filled === body.length) return body;
|
|
1219
|
-
const exact = Buffer.allocUnsafeSlow(filled);
|
|
1220
|
-
exact.set(body.subarray(0, filled));
|
|
1221
|
-
return exact;
|
|
1222
|
-
}
|
|
1223
|
-
function refuseBodyBytes(res, refusal2) {
|
|
1224
|
-
sendError(
|
|
1225
|
-
res,
|
|
1226
|
-
429,
|
|
1227
|
-
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1228
|
-
`The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
|
|
1229
|
-
{ reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
|
|
1230
|
-
);
|
|
1231
|
-
}
|
|
1232
|
-
function readBody(res, maxBytes, abort, hold, declaredBytes = null) {
|
|
1233
|
-
return new Promise((resolve, reject) => {
|
|
1234
|
-
if (abort.aborted) {
|
|
1235
|
-
reject(new Error("aborted"));
|
|
1236
|
-
return;
|
|
1237
|
-
}
|
|
1238
|
-
let done = false;
|
|
1239
|
-
let total = 0;
|
|
1240
|
-
let filled = 0;
|
|
1241
|
-
let body = null;
|
|
1242
|
-
const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
|
|
1243
|
-
const refusal2 = hold.grow(reserved);
|
|
1244
|
-
if (refusal2 !== null) {
|
|
1245
|
-
refuseBodyBytes(res, refusal2);
|
|
1246
|
-
reject(new Error("request bytes in flight"));
|
|
1247
|
-
return;
|
|
1248
|
-
}
|
|
1249
|
-
abort.onAbort(() => {
|
|
1250
|
-
if (done) return;
|
|
1251
|
-
done = true;
|
|
1252
|
-
reject(new Error("aborted"));
|
|
1253
|
-
});
|
|
1254
|
-
res.onData((chunk, isLast) => {
|
|
1255
|
-
if (done || abort.aborted) return;
|
|
1256
|
-
total += chunk.byteLength;
|
|
1257
|
-
if (total > maxBytes) {
|
|
1258
|
-
done = true;
|
|
1259
|
-
if (!abort.aborted) {
|
|
1260
|
-
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1261
|
-
}
|
|
1262
|
-
reject(new Error("payload too large"));
|
|
1263
|
-
return;
|
|
1264
|
-
}
|
|
1265
|
-
const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
|
|
1266
|
-
if (chunkRefusal !== null) {
|
|
1267
|
-
done = true;
|
|
1268
|
-
if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
|
|
1269
|
-
reject(new Error("request bytes in flight"));
|
|
1270
|
-
return;
|
|
1271
|
-
}
|
|
1272
|
-
const arrived = new Uint8Array(chunk);
|
|
1273
|
-
const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
|
|
1274
|
-
body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
|
|
1275
|
-
body.set(arrived, filled);
|
|
1276
|
-
filled = total;
|
|
1277
|
-
if (isLast) {
|
|
1278
|
-
done = true;
|
|
1279
|
-
resolve(bodyOfExactly(body, filled));
|
|
1280
|
-
}
|
|
1281
|
-
});
|
|
1282
|
-
});
|
|
1283
|
-
}
|
|
1284
1350
|
function decodePathParameter(raw) {
|
|
1285
1351
|
if (!raw.includes("%")) return raw;
|
|
1286
1352
|
try {
|
|
@@ -1420,7 +1486,12 @@ function createRouteRunner(deps) {
|
|
|
1420
1486
|
}
|
|
1421
1487
|
const release = track?.(res, abort);
|
|
1422
1488
|
const bodyHold = bodyBudget.hold();
|
|
1423
|
-
|
|
1489
|
+
let bodyRefusal = null;
|
|
1490
|
+
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, bodyHold, declaredBytes, opts.bodyKind ?? null).catch((err) => {
|
|
1491
|
+
if (!(err instanceof RefusedBody)) throw err;
|
|
1492
|
+
bodyRefusal = err.refusal;
|
|
1493
|
+
return null;
|
|
1494
|
+
}) : Promise.resolve(null);
|
|
1424
1495
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
1425
1496
|
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
1426
1497
|
res,
|
|
@@ -1448,7 +1519,7 @@ function createRouteRunner(deps) {
|
|
|
1448
1519
|
return;
|
|
1449
1520
|
}
|
|
1450
1521
|
try {
|
|
1451
|
-
await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
|
|
1522
|
+
await handler({ res, params, query, contentType, rawBody, bodyRefusal, spooledBody, abort, hookContext });
|
|
1452
1523
|
} catch (err) {
|
|
1453
1524
|
if (!abort.aborted) {
|
|
1454
1525
|
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
@@ -1463,7 +1534,99 @@ function createRouteRunner(deps) {
|
|
|
1463
1534
|
};
|
|
1464
1535
|
}
|
|
1465
1536
|
|
|
1537
|
+
// src/server/batch-scan.ts
|
|
1538
|
+
var QUOTE2 = 34;
|
|
1539
|
+
var BACKSLASH2 = 92;
|
|
1540
|
+
var OPEN_OBJECT2 = 123;
|
|
1541
|
+
var CLOSE_OBJECT = 125;
|
|
1542
|
+
var OPEN_ARRAY2 = 91;
|
|
1543
|
+
var CLOSE_ARRAY = 93;
|
|
1544
|
+
var COLON2 = 58;
|
|
1545
|
+
var REMOVE_BATCH_KEYS = /* @__PURE__ */ new Set(["action", "docIds", "options"]);
|
|
1546
|
+
var REMOVE_ACTION = "delete";
|
|
1547
|
+
function isWhitespace(byte) {
|
|
1548
|
+
return byte === 32 || byte === 9 || byte === 10 || byte === 13;
|
|
1549
|
+
}
|
|
1550
|
+
function nextByteAt(raw, from) {
|
|
1551
|
+
let at = from;
|
|
1552
|
+
while (at < raw.length && isWhitespace(raw[at])) at++;
|
|
1553
|
+
return at;
|
|
1554
|
+
}
|
|
1555
|
+
function plainStringOf(raw, open2, close) {
|
|
1556
|
+
const content = raw.subarray(open2 + 1, close);
|
|
1557
|
+
if (content.includes(BACKSLASH2)) return null;
|
|
1558
|
+
return new TextDecoder().decode(content);
|
|
1559
|
+
}
|
|
1560
|
+
function isRemoveOnlyBatch(raw) {
|
|
1561
|
+
const start = nextByteAt(raw, 0);
|
|
1562
|
+
if (raw[start] !== OPEN_OBJECT2) return false;
|
|
1563
|
+
let depth = 0;
|
|
1564
|
+
let action = null;
|
|
1565
|
+
for (let at = start; at < raw.length; at++) {
|
|
1566
|
+
const byte = raw[at];
|
|
1567
|
+
if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
|
|
1568
|
+
else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
|
|
1569
|
+
else if (byte === QUOTE2) {
|
|
1570
|
+
const close = closingQuoteOf(raw, at + 1);
|
|
1571
|
+
if (close >= raw.length) return false;
|
|
1572
|
+
const afterString = nextByteAt(raw, close + 1);
|
|
1573
|
+
if (depth === 1 && raw[afterString] === COLON2) {
|
|
1574
|
+
const key = plainStringOf(raw, at, close);
|
|
1575
|
+
if (key === null || !REMOVE_BATCH_KEYS.has(key)) return false;
|
|
1576
|
+
if (key === "action") {
|
|
1577
|
+
const value = nextByteAt(raw, afterString + 1);
|
|
1578
|
+
const valueClose = raw[value] === QUOTE2 ? closingQuoteOf(raw, value + 1) : raw.length;
|
|
1579
|
+
action = valueClose < raw.length ? plainStringOf(raw, value, valueClose) : null;
|
|
1580
|
+
if (action === null) return false;
|
|
1581
|
+
at = valueClose;
|
|
1582
|
+
continue;
|
|
1583
|
+
}
|
|
1584
|
+
at = afterString;
|
|
1585
|
+
continue;
|
|
1586
|
+
}
|
|
1587
|
+
at = close;
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
return action === REMOVE_ACTION;
|
|
1591
|
+
}
|
|
1592
|
+
function batchBodyKindOf(prefix) {
|
|
1593
|
+
const scanned = prefix.length > BATCH_KIND_SCAN_BYTES ? prefix.subarray(0, BATCH_KIND_SCAN_BYTES) : prefix;
|
|
1594
|
+
const start = nextByteAt(scanned, 0);
|
|
1595
|
+
if (start >= scanned.length) return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
|
|
1596
|
+
if (scanned[start] !== OPEN_OBJECT2) return "write";
|
|
1597
|
+
let depth = 0;
|
|
1598
|
+
for (let at = start; at < scanned.length; at++) {
|
|
1599
|
+
const byte = scanned[at];
|
|
1600
|
+
if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
|
|
1601
|
+
else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
|
|
1602
|
+
else if (byte === QUOTE2) {
|
|
1603
|
+
const close = closingQuoteOf(scanned, at + 1);
|
|
1604
|
+
if (close >= scanned.length) break;
|
|
1605
|
+
const afterString = nextByteAt(scanned, close + 1);
|
|
1606
|
+
if (afterString >= scanned.length) break;
|
|
1607
|
+
if (depth === 1 && scanned[afterString] === COLON2) {
|
|
1608
|
+
const key = plainStringOf(scanned, at, close);
|
|
1609
|
+
if (key === null || !REMOVE_BATCH_KEYS.has(key)) return "write";
|
|
1610
|
+
if (key === "action") {
|
|
1611
|
+
const value = nextByteAt(scanned, afterString + 1);
|
|
1612
|
+
if (value >= scanned.length) break;
|
|
1613
|
+
if (scanned[value] !== QUOTE2) return "write";
|
|
1614
|
+
const valueClose = closingQuoteOf(scanned, value + 1);
|
|
1615
|
+
if (valueClose >= scanned.length) break;
|
|
1616
|
+
return plainStringOf(scanned, value, valueClose) === REMOVE_ACTION ? "read" : "write";
|
|
1617
|
+
}
|
|
1618
|
+
at = afterString;
|
|
1619
|
+
continue;
|
|
1620
|
+
}
|
|
1621
|
+
at = close;
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1466
1627
|
// src/server/routes.ts
|
|
1628
|
+
var writeBodyKind = () => "write";
|
|
1629
|
+
var readBodyKind = () => "read";
|
|
1467
1630
|
function serverRoutes(limits) {
|
|
1468
1631
|
const { maxBodyBytes, maxImportBytes } = limits;
|
|
1469
1632
|
const probe = { maxBytes: 0, skipHooks: true };
|
|
@@ -1472,6 +1635,9 @@ function serverRoutes(limits) {
|
|
|
1472
1635
|
const namedBody = { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes };
|
|
1473
1636
|
const document = { paramCount: 2, maxBytes: 0 };
|
|
1474
1637
|
const documentBody = { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes };
|
|
1638
|
+
const writeBody = { ...namedBody, bodyKind: writeBodyKind };
|
|
1639
|
+
const readBody2 = { ...namedBody, bodyKind: readBodyKind };
|
|
1640
|
+
const documentWriteBody = { ...documentBody, bodyKind: writeBodyKind };
|
|
1475
1641
|
return [
|
|
1476
1642
|
{ method: "get", path: "/livez", handler: "livez", options: probe },
|
|
1477
1643
|
{ method: "get", path: "/readyz", handler: "readyz", options: probe },
|
|
@@ -1501,10 +1667,15 @@ function serverRoutes(limits) {
|
|
|
1501
1667
|
{ method: "post", path: "/indexes/:name/_open", handler: "openIndex", options: named },
|
|
1502
1668
|
{ method: "post", path: "/indexes/:name/_close", handler: "closeIndex", options: named },
|
|
1503
1669
|
{ method: "get", path: "/indexes/:name/count", handler: "count", options: named },
|
|
1504
|
-
{ method: "post", path: "/indexes/:name/documents", handler: "insert", options:
|
|
1505
|
-
{
|
|
1506
|
-
|
|
1507
|
-
|
|
1670
|
+
{ method: "post", path: "/indexes/:name/documents", handler: "insert", options: writeBody },
|
|
1671
|
+
{
|
|
1672
|
+
method: "post",
|
|
1673
|
+
path: "/indexes/:name/documents/_batch",
|
|
1674
|
+
handler: "batch",
|
|
1675
|
+
options: { ...namedBody, bodyKind: batchBodyKindOf }
|
|
1676
|
+
},
|
|
1677
|
+
{ method: "post", path: "/indexes/:name/documents/_list", handler: "list", options: readBody2 },
|
|
1678
|
+
{ method: "post", path: "/indexes/:name/documents/_multi-get", handler: "multiGet", options: readBody2 },
|
|
1508
1679
|
{
|
|
1509
1680
|
method: "post",
|
|
1510
1681
|
path: "/indexes/:name/documents/_import",
|
|
@@ -1514,12 +1685,12 @@ function serverRoutes(limits) {
|
|
|
1514
1685
|
{ method: "post", path: "/indexes/:name/_wait-for-writes", handler: "waitForWrites", options: named },
|
|
1515
1686
|
{ method: "get", path: "/indexes/:name/documents/:id", handler: "get", options: document },
|
|
1516
1687
|
{ method: "get", path: "/indexes/:name/documents/:id/_exists", handler: "exists", options: document },
|
|
1517
|
-
{ method: "put", path: "/indexes/:name/documents/:id", handler: "put", options:
|
|
1518
|
-
{ method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options:
|
|
1688
|
+
{ method: "put", path: "/indexes/:name/documents/:id", handler: "put", options: documentWriteBody },
|
|
1689
|
+
{ method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options: documentWriteBody },
|
|
1519
1690
|
{ method: "del", path: "/indexes/:name/documents/:id", handler: "remove", options: document },
|
|
1520
|
-
{ method: "post", path: "/indexes/:name/search", handler: "search", options:
|
|
1521
|
-
{ method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options:
|
|
1522
|
-
{ method: "post", path: "/indexes/:name/suggest", handler: "suggest", options:
|
|
1691
|
+
{ method: "post", path: "/indexes/:name/search", handler: "search", options: readBody2 },
|
|
1692
|
+
{ method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options: readBody2 },
|
|
1693
|
+
{ method: "post", path: "/indexes/:name/suggest", handler: "suggest", options: readBody2 },
|
|
1523
1694
|
{ method: "post", path: "/indexes/:name/_checkpoint", handler: "checkpoint", options: named },
|
|
1524
1695
|
{ method: "post", path: "/indexes/:name/_rebuild-analysis", handler: "rebuildAnalysis", options: named },
|
|
1525
1696
|
{ method: "get", path: "/indexes/:name/snapshot", handler: "snapshot", options: named },
|
|
@@ -1656,4 +1827,4 @@ async function departureConfirmation(word) {
|
|
|
1656
1827
|
if (waiting.async) await waiting.value;
|
|
1657
1828
|
}
|
|
1658
1829
|
|
|
1659
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits,
|
|
1830
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, isRemoveOnlyBatch, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-OCB7TMUQ.mjs';
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-4XWIW3F3.mjs';
|
|
3
3
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
4
|
import { MAX_DOC_ID_LENGTH } from './chunk-LXWDQLTW.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
@@ -82,6 +82,7 @@ var BUILD_CHUNK_SIZE = 100;
|
|
|
82
82
|
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
83
83
|
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
84
84
|
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
85
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
85
86
|
var REBUILD_REMOVED_RATIO = 0.2;
|
|
86
87
|
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
87
88
|
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
@@ -648,4 +649,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
648
649
|
state.savedSignature = plan.signature;
|
|
649
650
|
}
|
|
650
651
|
|
|
651
|
-
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
652
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readMetadataEnvelope } from './chunk-
|
|
2
|
-
import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope } from './chunk-JUAFWJON.mjs';
|
|
2
|
+
import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-27ZEY73B.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-LVNBY2TS.mjs';
|
|
4
4
|
|
|
5
5
|
// src/persistence/durability/recovery.ts
|
|
6
6
|
async function listPersistedIndexes(directory) {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { V8_YOUNG_GENERATION_BYTES_BY_MAJOR } from './chunk-22YPZVMG.mjs';
|
|
2
|
+
|
|
3
|
+
// src/runtime/engine-threads.ts
|
|
4
|
+
var MEGABYTE_BYTES = 1048576;
|
|
5
|
+
var threadYoungGenerationBytes = 0;
|
|
6
|
+
function youngGenerationBytesOfThisV8() {
|
|
7
|
+
const major = Number(globalThis.process?.versions?.v8?.split(".")[0]);
|
|
8
|
+
return V8_YOUNG_GENERATION_BYTES_BY_MAJOR[major] ?? Math.max(...Object.values(V8_YOUNG_GENERATION_BYTES_BY_MAJOR));
|
|
9
|
+
}
|
|
10
|
+
function countEngineThread(thread, youngGenerationMb) {
|
|
11
|
+
const bytes = youngGenerationMb === void 0 ? youngGenerationBytesOfThisV8() : youngGenerationMb * MEGABYTE_BYTES;
|
|
12
|
+
threadYoungGenerationBytes += bytes;
|
|
13
|
+
thread.once("exit", () => {
|
|
14
|
+
threadYoungGenerationBytes -= bytes;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
function engineThreadYoungGenerationBytes() {
|
|
18
|
+
return threadYoungGenerationBytes;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export { countEngineThread, engineThreadYoungGenerationBytes, youngGenerationBytesOfThisV8 };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-5WEBVYOA.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
|
|
4
4
|
import { flattenSchema, partialFieldsOf } from './chunk-DG62H7OD.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles,
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-4XWIW3F3.mjs';
|
|
2
2
|
import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
4
4
|
import { readSync, closeSync, openSync } from 'fs';
|
|
@@ -581,7 +581,8 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
581
581
|
present: new Uint8Array(handles.present),
|
|
582
582
|
partitions: new Int32Array(handles.partitions),
|
|
583
583
|
diskFile: new Int32Array(handles.diskFile),
|
|
584
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
584
|
+
diskOffset: new Uint32Array(handles.diskOffset),
|
|
585
|
+
longestDocIdBytes: 0
|
|
585
586
|
};
|
|
586
587
|
}
|
|
587
588
|
function relayout(store) {
|
|
@@ -592,7 +593,10 @@ function claimSlot(store, docId, cold) {
|
|
|
592
593
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
593
594
|
ensureOrdinal(store, ordinal, cold);
|
|
594
595
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
596
|
+
const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
|
|
595
597
|
appendDocId(store.handles, ordinal, docId);
|
|
598
|
+
const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
|
|
599
|
+
if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
|
|
596
600
|
return ordinal;
|
|
597
601
|
}
|
|
598
602
|
function publishOrdinal(store, ordinal) {
|
|
@@ -668,6 +672,19 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
668
672
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
669
673
|
ensureCodeSlot(store, ordinal);
|
|
670
674
|
}
|
|
675
|
+
function bufferGrowthBytes(buffer, bytes) {
|
|
676
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
677
|
+
}
|
|
678
|
+
function growthBytesFor(store, vectors) {
|
|
679
|
+
const { handles } = store;
|
|
680
|
+
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
681
|
+
const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
|
|
682
|
+
if (last < slots) return 0;
|
|
683
|
+
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
684
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
685
|
+
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
686
|
+
return bytes;
|
|
687
|
+
}
|
|
671
688
|
function writeVector(store, ordinal, vector) {
|
|
672
689
|
const block = store.view.blockOf(ordinal);
|
|
673
690
|
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
@@ -1004,6 +1021,9 @@ function createVectorStore(options) {
|
|
|
1004
1021
|
},
|
|
1005
1022
|
memoryBytes() {
|
|
1006
1023
|
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
1024
|
+
},
|
|
1025
|
+
growthBytesFor(vectors) {
|
|
1026
|
+
return open === null ? 0 : growthBytesFor(open, vectors);
|
|
1007
1027
|
}
|
|
1008
1028
|
};
|
|
1009
1029
|
}
|