@delali/narsil 0.3.1-next.112 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-UR5MNN53.mjs → chunk-2TMWIATH.mjs} +108 -67
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-73U5DWOZ.mjs} +39 -4
- package/dist/browser/{chunk-O5IKTA27.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-G2PPZV3B.mjs} +127 -33
- package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-EBH6RXQO.mjs} +6 -5
- package/dist/browser/{write-IABOLE4N.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-4ZKBKPBV.mjs} +22 -2
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-3COL24MT.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-QI6JW4PM.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-3RRATITJ.mjs → chunk-BCSWODXP.mjs} +75 -55
- package/dist/{thread-WGBCVJII.mjs → chunk-DEXID4ZT.mjs} +301 -200
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-YIDSHBQS.mjs → chunk-HC5NURS6.mjs} +33 -4
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/{chunk-63HFWQEA.mjs → chunk-IE43SRL3.mjs} +276 -102
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-2IFU7SXG.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-7CI34JAP.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-PT7VRWS7.mjs → chunk-NX7KU77V.mjs} +3 -2
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-V7ISFBFC.mjs → chunk-Q62NDKEC.mjs} +5 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-BVOUSX3U.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-IBTKJIV7.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-IM76DLA7.mjs → chunk-YCFQ5754.mjs} +2 -2
- package/dist/{chunk-VLRT47GU.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-OUY4RPTU.mjs} +7 -6
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +79 -42
- package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-5V6DVHFK.mjs} +17 -16
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +10 -9
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WP4VVGV2.mjs → module-3IMTXNPW.mjs} +92 -29
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -13
- package/dist/react.d.ts +6 -6
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +110 -258
- package/dist/{snapshot-EU6NNB2P.mjs → snapshot-57WF53MK.mjs} +18 -17
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +19 -7
- package/dist/{write-CKRWS3P5.mjs → write-YEI7A5OR.mjs} +11 -10
- package/package.json +7 -7
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { readRecentMemoryFigures, readMemoryFigures,
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures, answerRefusalOf } from './chunk-HFJXEFFV.mjs';
|
|
2
|
+
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
2
3
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
4
|
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
5
|
import { loadNativeCore } from './chunk-VK45J47R.mjs';
|
|
5
6
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
6
7
|
import { MAX_SORT_FIELDS } from './chunk-E74YLAIG.mjs';
|
|
7
|
-
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-
|
|
8
|
+
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
|
|
8
9
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
9
10
|
import { randomUUID } from 'crypto';
|
|
10
11
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
@@ -301,6 +302,7 @@ var DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 0.05;
|
|
|
301
302
|
var MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 1;
|
|
302
303
|
var PARSE_HEAP_BLOCK_FRACTION = 0.9;
|
|
303
304
|
var PARSE_HEAP_READ_BLOCK_FRACTION = 0.95;
|
|
305
|
+
var BATCH_KIND_SCAN_BYTES = 65536;
|
|
304
306
|
var PARSE_WORST_HEAP_BYTES_PER_BODY_BYTE = 24;
|
|
305
307
|
var PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE = 2;
|
|
306
308
|
var PARSE_OBJECT_HEAP_BYTES = 72;
|
|
@@ -332,6 +334,9 @@ var DEFAULT_TASK_PAGE_SIZE = 20;
|
|
|
332
334
|
var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
|
|
333
335
|
var RUNNING_TASK_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
334
336
|
var TERMINAL_TASK_TTL_MS = 60 * 60 * 1e3;
|
|
337
|
+
var REQUEST_THREAD_MOVE_AFTER_MS = 10;
|
|
338
|
+
var SEARCH_CLOCK_SAMPLE_UNITS = 65536;
|
|
339
|
+
var MAX_LONG_QUERIES_IN_FLIGHT = 16;
|
|
335
340
|
|
|
336
341
|
// src/server/parse-heap.ts
|
|
337
342
|
var PARSE_WOULD_FILL_MEMORY_REASON = "parse_would_fill_memory";
|
|
@@ -384,25 +389,34 @@ function scannedParseBytes(raw) {
|
|
|
384
389
|
}
|
|
385
390
|
var WRITE_PARSE_LINE = {
|
|
386
391
|
heap: Math.min(PARSE_HEAP_BLOCK_FRACTION, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
|
|
387
|
-
process: PARSE_HEAP_BLOCK_FRACTION
|
|
392
|
+
process: PARSE_HEAP_BLOCK_FRACTION,
|
|
393
|
+
holdsYoungReserve: true
|
|
388
394
|
};
|
|
389
|
-
var READ_PARSE_LINE = {
|
|
395
|
+
var READ_PARSE_LINE = {
|
|
396
|
+
heap: PARSE_HEAP_READ_BLOCK_FRACTION,
|
|
397
|
+
process: PARSE_HEAP_READ_BLOCK_FRACTION,
|
|
398
|
+
holdsYoungReserve: false
|
|
399
|
+
};
|
|
400
|
+
function processRoomIn(figures, line) {
|
|
401
|
+
const reserve = line.holdsYoungReserve ? figures.youngReserveBytes : 0;
|
|
402
|
+
return line.process * figures.processLimitBytes - figures.processBytes - reserve;
|
|
403
|
+
}
|
|
390
404
|
function roomIn(figures, line) {
|
|
391
405
|
return Math.min(
|
|
392
406
|
line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes,
|
|
393
|
-
|
|
407
|
+
processRoomIn(figures, line)
|
|
394
408
|
) - reservedHeapBytes();
|
|
395
409
|
}
|
|
396
|
-
function refusal(status,
|
|
410
|
+
function refusal(status, bodyBytes, parseBytes, figures, collectedGarbage, line) {
|
|
397
411
|
const blockBytes = Math.floor(line.heap * figures.heapLimitBytes);
|
|
398
|
-
const message = status === 413 ? `Parsing this ${
|
|
412
|
+
const message = status === 413 ? `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, more than the ${blockBytes} bytes that the server allows its heap even when idle. Send the request in smaller parts.` : `Parsing this ${bodyBytes}-byte body can take up to ${parseBytes} bytes of memory, which with the ${figures.heapBytes} bytes already in use would pass the server's limit of ${blockBytes} bytes. Retry once the requests in progress finish.`;
|
|
399
413
|
return {
|
|
400
414
|
status,
|
|
401
415
|
code: status === 413 ? ServerErrorCodes.PAYLOAD_TOO_LARGE : ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
402
416
|
message,
|
|
403
417
|
details: {
|
|
404
418
|
reason: status === 413 ? PARSE_EXCEEDS_MEMORY_REASON : PARSE_WOULD_FILL_MEMORY_REASON,
|
|
405
|
-
bodyBytes
|
|
419
|
+
bodyBytes,
|
|
406
420
|
parseBytes,
|
|
407
421
|
heapBytes: figures.heapBytes,
|
|
408
422
|
youngHeapBytes: figures.youngHeapBytes,
|
|
@@ -424,7 +438,7 @@ function parseRefusalOf(raw, kind = "write") {
|
|
|
424
438
|
if (worstBytes <= roomIn(recent, line)) return admitted(worstBytes);
|
|
425
439
|
const parseBytes = Math.min(scannedParseBytes(raw), worstBytes);
|
|
426
440
|
if (parseBytes > line.heap * recent.heapLimitBytes) {
|
|
427
|
-
return refusal(413, raw, parseBytes, recent, false, line);
|
|
441
|
+
return refusal(413, raw.length, parseBytes, recent, false, line);
|
|
428
442
|
}
|
|
429
443
|
if (parseBytes <= roomIn(recent, line)) return admitted(parseBytes);
|
|
430
444
|
const figures = readMemoryFigures() ?? recent;
|
|
@@ -434,7 +448,31 @@ function parseRefusalOf(raw, kind = "write") {
|
|
|
434
448
|
const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
|
|
435
449
|
if (parseBytes <= roomIn(settled, line)) return admitted(parseBytes);
|
|
436
450
|
if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
|
|
437
|
-
return refusal(429, raw, parseBytes, settled, collectedGarbage, line);
|
|
451
|
+
return refusal(429, raw.length, parseBytes, settled, collectedGarbage, line);
|
|
452
|
+
}
|
|
453
|
+
function bodyFits(bodyBytes, figures, line) {
|
|
454
|
+
const reserved = reservedHeapBytes();
|
|
455
|
+
const heapRoom = line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes - reserved;
|
|
456
|
+
const processRoom = processRoomIn(figures, line) - reserved;
|
|
457
|
+
return bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE <= heapRoom && bodyBytes * (PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE + 1) <= processRoom;
|
|
458
|
+
}
|
|
459
|
+
function bodyRefusalOf(bodyBytes, kind) {
|
|
460
|
+
if (bodyBytes <= 0) return null;
|
|
461
|
+
const line = kind === "read" ? READ_PARSE_LINE : WRITE_PARSE_LINE;
|
|
462
|
+
const recent = readRecentMemoryFigures();
|
|
463
|
+
if (recent === null || bodyFits(bodyBytes, recent, line)) return null;
|
|
464
|
+
const leastParseBytes = bodyBytes * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE;
|
|
465
|
+
if (leastParseBytes > line.heap * recent.heapLimitBytes) {
|
|
466
|
+
return refusal(413, bodyBytes, leastParseBytes, recent, false, line);
|
|
467
|
+
}
|
|
468
|
+
const figures = readMemoryFigures() ?? recent;
|
|
469
|
+
if (bodyFits(bodyBytes, figures, line)) return null;
|
|
470
|
+
const collectedGarbage = collectGarbageBeforeRefusal();
|
|
471
|
+
const collectedYoung = !collectedGarbage && collectYoungGenerationBeforeRefusal();
|
|
472
|
+
const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
|
|
473
|
+
if (bodyFits(bodyBytes, settled, line)) return null;
|
|
474
|
+
if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
|
|
475
|
+
return refusal(429, bodyBytes, leastParseBytes, settled, collectedGarbage, line);
|
|
438
476
|
}
|
|
439
477
|
|
|
440
478
|
// src/json-encoding.ts
|
|
@@ -573,18 +611,18 @@ function sendRelayed(res, abort, status, headers, body) {
|
|
|
573
611
|
|
|
574
612
|
// src/server/handler-utils.ts
|
|
575
613
|
function refusesParse(ctx, raw, kind) {
|
|
576
|
-
const refusal2 = parseRefusalOf(raw, kind);
|
|
614
|
+
const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw, kind));
|
|
577
615
|
if (refusal2 === null) return false;
|
|
578
616
|
if (!ctx.abort.aborted) sendError(ctx.res, refusal2.status, refusal2.code, refusal2.message, refusal2.details);
|
|
579
617
|
return true;
|
|
580
618
|
}
|
|
581
619
|
function parseJson(ctx, kind = "write") {
|
|
582
620
|
const raw = ctx.rawBody;
|
|
621
|
+
if (refusesParse(ctx, raw, kind)) return void 0;
|
|
583
622
|
if (!raw || raw.length === 0) {
|
|
584
623
|
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
585
624
|
return void 0;
|
|
586
625
|
}
|
|
587
|
-
if (refusesParse(ctx, raw, kind)) return void 0;
|
|
588
626
|
try {
|
|
589
627
|
return JSON.parse(raw.toString("utf8"));
|
|
590
628
|
} catch {
|
|
@@ -594,11 +632,11 @@ function parseJson(ctx, kind = "write") {
|
|
|
594
632
|
}
|
|
595
633
|
async function parseWriteJson(ctx, writeBlock) {
|
|
596
634
|
const raw = ctx.rawBody;
|
|
597
|
-
if (
|
|
598
|
-
const refusal2 = parseRefusalOf(raw);
|
|
635
|
+
if (writeBlock === void 0 || ctx.bodyRefusal === null && (!raw || raw.length === 0)) return parseJson(ctx);
|
|
636
|
+
const refusal2 = ctx.bodyRefusal ?? (raw === null ? null : parseRefusalOf(raw));
|
|
599
637
|
if (refusal2 === null) {
|
|
600
638
|
try {
|
|
601
|
-
return JSON.parse(raw
|
|
639
|
+
return JSON.parse(raw?.toString("utf8") ?? "");
|
|
602
640
|
} catch {
|
|
603
641
|
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
604
642
|
return void 0;
|
|
@@ -611,8 +649,8 @@ async function parseWriteJson(ctx, writeBlock) {
|
|
|
611
649
|
}
|
|
612
650
|
function parseJsonOptional(ctx) {
|
|
613
651
|
const raw = ctx.rawBody;
|
|
614
|
-
if (!raw || raw.length === 0) return {};
|
|
615
652
|
if (refusesParse(ctx, raw, "write")) return void 0;
|
|
653
|
+
if (!raw || raw.length === 0) return {};
|
|
616
654
|
try {
|
|
617
655
|
return JSON.parse(raw.toString("utf8"));
|
|
618
656
|
} catch {
|
|
@@ -1104,6 +1142,111 @@ function createVersionHandler(deps) {
|
|
|
1104
1142
|
}
|
|
1105
1143
|
return { report };
|
|
1106
1144
|
}
|
|
1145
|
+
|
|
1146
|
+
// src/server/read-body.ts
|
|
1147
|
+
var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
|
|
1148
|
+
function bodyWithRoomFor(body, filled, needed, maxBytes) {
|
|
1149
|
+
if (needed <= body.length) return body;
|
|
1150
|
+
const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
|
|
1151
|
+
grown.set(body.subarray(0, filled));
|
|
1152
|
+
return grown;
|
|
1153
|
+
}
|
|
1154
|
+
function bodyOfExactly(body, filled) {
|
|
1155
|
+
if (filled === body.length) return body;
|
|
1156
|
+
const exact = Buffer.allocUnsafeSlow(filled);
|
|
1157
|
+
exact.set(body.subarray(0, filled));
|
|
1158
|
+
return exact;
|
|
1159
|
+
}
|
|
1160
|
+
var NO_BYTES = new Uint8Array(0);
|
|
1161
|
+
var RefusedBody = class extends Error {
|
|
1162
|
+
constructor(refusal2) {
|
|
1163
|
+
super(refusal2.message);
|
|
1164
|
+
this.refusal = refusal2;
|
|
1165
|
+
}
|
|
1166
|
+
};
|
|
1167
|
+
function refuseBodyBytes(res, refusal2) {
|
|
1168
|
+
sendError(
|
|
1169
|
+
res,
|
|
1170
|
+
429,
|
|
1171
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1172
|
+
`The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
|
|
1173
|
+
{ reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
1176
|
+
function readBody(res, maxBytes, abort, hold, declaredBytes = null, kindOf = null) {
|
|
1177
|
+
return new Promise((resolve, reject) => {
|
|
1178
|
+
if (abort.aborted) {
|
|
1179
|
+
reject(new Error("aborted"));
|
|
1180
|
+
return;
|
|
1181
|
+
}
|
|
1182
|
+
let done = false;
|
|
1183
|
+
let total = 0;
|
|
1184
|
+
let filled = 0;
|
|
1185
|
+
let body = null;
|
|
1186
|
+
const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
|
|
1187
|
+
let kind = kindOf?.(NO_BYTES) ?? null;
|
|
1188
|
+
const memoryRefusal = kindOf === null ? null : bodyRefusalOf(reserved, kind ?? "read");
|
|
1189
|
+
if (memoryRefusal !== null) {
|
|
1190
|
+
res.onData(() => void 0);
|
|
1191
|
+
reject(new RefusedBody(memoryRefusal));
|
|
1192
|
+
return;
|
|
1193
|
+
}
|
|
1194
|
+
const refusal2 = hold.grow(reserved);
|
|
1195
|
+
if (refusal2 !== null) {
|
|
1196
|
+
refuseBodyBytes(res, refusal2);
|
|
1197
|
+
reject(new Error("request bytes in flight"));
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1200
|
+
abort.onAbort(() => {
|
|
1201
|
+
if (done) return;
|
|
1202
|
+
done = true;
|
|
1203
|
+
reject(new Error("aborted"));
|
|
1204
|
+
});
|
|
1205
|
+
res.onData((chunk, isLast) => {
|
|
1206
|
+
if (done || abort.aborted) return;
|
|
1207
|
+
total += chunk.byteLength;
|
|
1208
|
+
if (total > maxBytes) {
|
|
1209
|
+
done = true;
|
|
1210
|
+
if (!abort.aborted) {
|
|
1211
|
+
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1212
|
+
}
|
|
1213
|
+
reject(new Error("payload too large"));
|
|
1214
|
+
return;
|
|
1215
|
+
}
|
|
1216
|
+
const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
|
|
1217
|
+
if (chunkRefusal !== null) {
|
|
1218
|
+
done = true;
|
|
1219
|
+
if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
|
|
1220
|
+
reject(new Error("request bytes in flight"));
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1223
|
+
const streamRefusal = kindOf !== null && declaredBytes === null ? bodyRefusalOf(total, kind ?? "read") : null;
|
|
1224
|
+
if (streamRefusal !== null) {
|
|
1225
|
+
done = true;
|
|
1226
|
+
reject(new RefusedBody(streamRefusal));
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
const arrived = new Uint8Array(chunk);
|
|
1230
|
+
const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
|
|
1231
|
+
body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
|
|
1232
|
+
body.set(arrived, filled);
|
|
1233
|
+
filled = total;
|
|
1234
|
+
if (kindOf !== null && kind === null) {
|
|
1235
|
+
kind = kindOf(body.subarray(0, filled));
|
|
1236
|
+
const kindRefusal = kind === null ? null : bodyRefusalOf(declaredBytes ?? total, kind);
|
|
1237
|
+
if (kindRefusal !== null) {
|
|
1238
|
+
done = true;
|
|
1239
|
+
reject(new RefusedBody(kindRefusal));
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
if (isLast) {
|
|
1244
|
+
done = true;
|
|
1245
|
+
resolve(bodyOfExactly(body, filled));
|
|
1246
|
+
}
|
|
1247
|
+
});
|
|
1248
|
+
});
|
|
1249
|
+
}
|
|
1107
1250
|
var NotAdmittedError = class extends Error {
|
|
1108
1251
|
};
|
|
1109
1252
|
async function removeDirectory(directory) {
|
|
@@ -1187,7 +1330,6 @@ function spoolBody(res, maxBytes, abort, admitted2) {
|
|
|
1187
1330
|
}
|
|
1188
1331
|
|
|
1189
1332
|
// src/server/request.ts
|
|
1190
|
-
var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
|
|
1191
1333
|
function initAbortHandler(res) {
|
|
1192
1334
|
const listeners = [];
|
|
1193
1335
|
let aborted = false;
|
|
@@ -1208,79 +1350,6 @@ function initAbortHandler(res) {
|
|
|
1208
1350
|
function declaredBodyBytes(header) {
|
|
1209
1351
|
return /^\d{1,15}$/.test(header) ? Number(header) : null;
|
|
1210
1352
|
}
|
|
1211
|
-
function bodyWithRoomFor(body, filled, needed, maxBytes) {
|
|
1212
|
-
if (needed <= body.length) return body;
|
|
1213
|
-
const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(needed, body.length * 2)));
|
|
1214
|
-
grown.set(body.subarray(0, filled));
|
|
1215
|
-
return grown;
|
|
1216
|
-
}
|
|
1217
|
-
function bodyOfExactly(body, filled) {
|
|
1218
|
-
if (filled === body.length) return body;
|
|
1219
|
-
const exact = Buffer.allocUnsafeSlow(filled);
|
|
1220
|
-
exact.set(body.subarray(0, filled));
|
|
1221
|
-
return exact;
|
|
1222
|
-
}
|
|
1223
|
-
function refuseBodyBytes(res, refusal2) {
|
|
1224
|
-
sendError(
|
|
1225
|
-
res,
|
|
1226
|
-
429,
|
|
1227
|
-
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1228
|
-
`The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
|
|
1229
|
-
{ reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
|
|
1230
|
-
);
|
|
1231
|
-
}
|
|
1232
|
-
function readBody(res, maxBytes, abort, hold, declaredBytes = null) {
|
|
1233
|
-
return new Promise((resolve, reject) => {
|
|
1234
|
-
if (abort.aborted) {
|
|
1235
|
-
reject(new Error("aborted"));
|
|
1236
|
-
return;
|
|
1237
|
-
}
|
|
1238
|
-
let done = false;
|
|
1239
|
-
let total = 0;
|
|
1240
|
-
let filled = 0;
|
|
1241
|
-
let body = null;
|
|
1242
|
-
const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
|
|
1243
|
-
const refusal2 = hold.grow(reserved);
|
|
1244
|
-
if (refusal2 !== null) {
|
|
1245
|
-
refuseBodyBytes(res, refusal2);
|
|
1246
|
-
reject(new Error("request bytes in flight"));
|
|
1247
|
-
return;
|
|
1248
|
-
}
|
|
1249
|
-
abort.onAbort(() => {
|
|
1250
|
-
if (done) return;
|
|
1251
|
-
done = true;
|
|
1252
|
-
reject(new Error("aborted"));
|
|
1253
|
-
});
|
|
1254
|
-
res.onData((chunk, isLast) => {
|
|
1255
|
-
if (done || abort.aborted) return;
|
|
1256
|
-
total += chunk.byteLength;
|
|
1257
|
-
if (total > maxBytes) {
|
|
1258
|
-
done = true;
|
|
1259
|
-
if (!abort.aborted) {
|
|
1260
|
-
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1261
|
-
}
|
|
1262
|
-
reject(new Error("payload too large"));
|
|
1263
|
-
return;
|
|
1264
|
-
}
|
|
1265
|
-
const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
|
|
1266
|
-
if (chunkRefusal !== null) {
|
|
1267
|
-
done = true;
|
|
1268
|
-
if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
|
|
1269
|
-
reject(new Error("request bytes in flight"));
|
|
1270
|
-
return;
|
|
1271
|
-
}
|
|
1272
|
-
const arrived = new Uint8Array(chunk);
|
|
1273
|
-
const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
|
|
1274
|
-
body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
|
|
1275
|
-
body.set(arrived, filled);
|
|
1276
|
-
filled = total;
|
|
1277
|
-
if (isLast) {
|
|
1278
|
-
done = true;
|
|
1279
|
-
resolve(bodyOfExactly(body, filled));
|
|
1280
|
-
}
|
|
1281
|
-
});
|
|
1282
|
-
});
|
|
1283
|
-
}
|
|
1284
1353
|
function decodePathParameter(raw) {
|
|
1285
1354
|
if (!raw.includes("%")) return raw;
|
|
1286
1355
|
try {
|
|
@@ -1420,7 +1489,12 @@ function createRouteRunner(deps) {
|
|
|
1420
1489
|
}
|
|
1421
1490
|
const release = track?.(res, abort);
|
|
1422
1491
|
const bodyHold = bodyBudget.hold();
|
|
1423
|
-
|
|
1492
|
+
let bodyRefusal = null;
|
|
1493
|
+
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, bodyHold, declaredBytes, opts.bodyKind ?? null).catch((err) => {
|
|
1494
|
+
if (!(err instanceof RefusedBody)) throw err;
|
|
1495
|
+
bodyRefusal = err.refusal;
|
|
1496
|
+
return null;
|
|
1497
|
+
}) : Promise.resolve(null);
|
|
1424
1498
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
1425
1499
|
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
1426
1500
|
res,
|
|
@@ -1448,7 +1522,7 @@ function createRouteRunner(deps) {
|
|
|
1448
1522
|
return;
|
|
1449
1523
|
}
|
|
1450
1524
|
try {
|
|
1451
|
-
await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
|
|
1525
|
+
await handler({ res, params, query, contentType, rawBody, bodyRefusal, spooledBody, abort, hookContext });
|
|
1452
1526
|
} catch (err) {
|
|
1453
1527
|
if (!abort.aborted) {
|
|
1454
1528
|
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
@@ -1463,7 +1537,99 @@ function createRouteRunner(deps) {
|
|
|
1463
1537
|
};
|
|
1464
1538
|
}
|
|
1465
1539
|
|
|
1540
|
+
// src/server/batch-scan.ts
|
|
1541
|
+
var QUOTE2 = 34;
|
|
1542
|
+
var BACKSLASH2 = 92;
|
|
1543
|
+
var OPEN_OBJECT2 = 123;
|
|
1544
|
+
var CLOSE_OBJECT = 125;
|
|
1545
|
+
var OPEN_ARRAY2 = 91;
|
|
1546
|
+
var CLOSE_ARRAY = 93;
|
|
1547
|
+
var COLON2 = 58;
|
|
1548
|
+
var REMOVE_BATCH_KEYS = /* @__PURE__ */ new Set(["action", "docIds", "options"]);
|
|
1549
|
+
var REMOVE_ACTION = "delete";
|
|
1550
|
+
function isWhitespace(byte) {
|
|
1551
|
+
return byte === 32 || byte === 9 || byte === 10 || byte === 13;
|
|
1552
|
+
}
|
|
1553
|
+
function nextByteAt(raw, from) {
|
|
1554
|
+
let at = from;
|
|
1555
|
+
while (at < raw.length && isWhitespace(raw[at])) at++;
|
|
1556
|
+
return at;
|
|
1557
|
+
}
|
|
1558
|
+
function plainStringOf(raw, open2, close) {
|
|
1559
|
+
const content = raw.subarray(open2 + 1, close);
|
|
1560
|
+
if (content.includes(BACKSLASH2)) return null;
|
|
1561
|
+
return new TextDecoder().decode(content);
|
|
1562
|
+
}
|
|
1563
|
+
function isRemoveOnlyBatch(raw) {
|
|
1564
|
+
const start = nextByteAt(raw, 0);
|
|
1565
|
+
if (raw[start] !== OPEN_OBJECT2) return false;
|
|
1566
|
+
let depth = 0;
|
|
1567
|
+
let action = null;
|
|
1568
|
+
for (let at = start; at < raw.length; at++) {
|
|
1569
|
+
const byte = raw[at];
|
|
1570
|
+
if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
|
|
1571
|
+
else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
|
|
1572
|
+
else if (byte === QUOTE2) {
|
|
1573
|
+
const close = closingQuoteOf(raw, at + 1);
|
|
1574
|
+
if (close >= raw.length) return false;
|
|
1575
|
+
const afterString = nextByteAt(raw, close + 1);
|
|
1576
|
+
if (depth === 1 && raw[afterString] === COLON2) {
|
|
1577
|
+
const key = plainStringOf(raw, at, close);
|
|
1578
|
+
if (key === null || !REMOVE_BATCH_KEYS.has(key)) return false;
|
|
1579
|
+
if (key === "action") {
|
|
1580
|
+
const value = nextByteAt(raw, afterString + 1);
|
|
1581
|
+
const valueClose = raw[value] === QUOTE2 ? closingQuoteOf(raw, value + 1) : raw.length;
|
|
1582
|
+
action = valueClose < raw.length ? plainStringOf(raw, value, valueClose) : null;
|
|
1583
|
+
if (action === null) return false;
|
|
1584
|
+
at = valueClose;
|
|
1585
|
+
continue;
|
|
1586
|
+
}
|
|
1587
|
+
at = afterString;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
at = close;
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
return action === REMOVE_ACTION;
|
|
1594
|
+
}
|
|
1595
|
+
function batchBodyKindOf(prefix) {
|
|
1596
|
+
const scanned = prefix.length > BATCH_KIND_SCAN_BYTES ? prefix.subarray(0, BATCH_KIND_SCAN_BYTES) : prefix;
|
|
1597
|
+
const start = nextByteAt(scanned, 0);
|
|
1598
|
+
if (start >= scanned.length) return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
|
|
1599
|
+
if (scanned[start] !== OPEN_OBJECT2) return "write";
|
|
1600
|
+
let depth = 0;
|
|
1601
|
+
for (let at = start; at < scanned.length; at++) {
|
|
1602
|
+
const byte = scanned[at];
|
|
1603
|
+
if (byte === OPEN_OBJECT2 || byte === OPEN_ARRAY2) depth++;
|
|
1604
|
+
else if (byte === CLOSE_OBJECT || byte === CLOSE_ARRAY) depth--;
|
|
1605
|
+
else if (byte === QUOTE2) {
|
|
1606
|
+
const close = closingQuoteOf(scanned, at + 1);
|
|
1607
|
+
if (close >= scanned.length) break;
|
|
1608
|
+
const afterString = nextByteAt(scanned, close + 1);
|
|
1609
|
+
if (afterString >= scanned.length) break;
|
|
1610
|
+
if (depth === 1 && scanned[afterString] === COLON2) {
|
|
1611
|
+
const key = plainStringOf(scanned, at, close);
|
|
1612
|
+
if (key === null || !REMOVE_BATCH_KEYS.has(key)) return "write";
|
|
1613
|
+
if (key === "action") {
|
|
1614
|
+
const value = nextByteAt(scanned, afterString + 1);
|
|
1615
|
+
if (value >= scanned.length) break;
|
|
1616
|
+
if (scanned[value] !== QUOTE2) return "write";
|
|
1617
|
+
const valueClose = closingQuoteOf(scanned, value + 1);
|
|
1618
|
+
if (valueClose >= scanned.length) break;
|
|
1619
|
+
return plainStringOf(scanned, value, valueClose) === REMOVE_ACTION ? "read" : "write";
|
|
1620
|
+
}
|
|
1621
|
+
at = afterString;
|
|
1622
|
+
continue;
|
|
1623
|
+
}
|
|
1624
|
+
at = close;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
return prefix.length > BATCH_KIND_SCAN_BYTES ? "read" : null;
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1466
1630
|
// src/server/routes.ts
|
|
1631
|
+
var writeBodyKind = () => "write";
|
|
1632
|
+
var readBodyKind = () => "read";
|
|
1467
1633
|
function serverRoutes(limits) {
|
|
1468
1634
|
const { maxBodyBytes, maxImportBytes } = limits;
|
|
1469
1635
|
const probe = { maxBytes: 0, skipHooks: true };
|
|
@@ -1472,6 +1638,9 @@ function serverRoutes(limits) {
|
|
|
1472
1638
|
const namedBody = { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes };
|
|
1473
1639
|
const document = { paramCount: 2, maxBytes: 0 };
|
|
1474
1640
|
const documentBody = { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes };
|
|
1641
|
+
const writeBody = { ...namedBody, bodyKind: writeBodyKind };
|
|
1642
|
+
const readBody2 = { ...namedBody, bodyKind: readBodyKind };
|
|
1643
|
+
const documentWriteBody = { ...documentBody, bodyKind: writeBodyKind };
|
|
1475
1644
|
return [
|
|
1476
1645
|
{ method: "get", path: "/livez", handler: "livez", options: probe },
|
|
1477
1646
|
{ method: "get", path: "/readyz", handler: "readyz", options: probe },
|
|
@@ -1501,10 +1670,15 @@ function serverRoutes(limits) {
|
|
|
1501
1670
|
{ method: "post", path: "/indexes/:name/_open", handler: "openIndex", options: named },
|
|
1502
1671
|
{ method: "post", path: "/indexes/:name/_close", handler: "closeIndex", options: named },
|
|
1503
1672
|
{ method: "get", path: "/indexes/:name/count", handler: "count", options: named },
|
|
1504
|
-
{ method: "post", path: "/indexes/:name/documents", handler: "insert", options:
|
|
1505
|
-
{
|
|
1506
|
-
|
|
1507
|
-
|
|
1673
|
+
{ method: "post", path: "/indexes/:name/documents", handler: "insert", options: writeBody },
|
|
1674
|
+
{
|
|
1675
|
+
method: "post",
|
|
1676
|
+
path: "/indexes/:name/documents/_batch",
|
|
1677
|
+
handler: "batch",
|
|
1678
|
+
options: { ...namedBody, bodyKind: batchBodyKindOf }
|
|
1679
|
+
},
|
|
1680
|
+
{ method: "post", path: "/indexes/:name/documents/_list", handler: "list", options: readBody2 },
|
|
1681
|
+
{ method: "post", path: "/indexes/:name/documents/_multi-get", handler: "multiGet", options: readBody2 },
|
|
1508
1682
|
{
|
|
1509
1683
|
method: "post",
|
|
1510
1684
|
path: "/indexes/:name/documents/_import",
|
|
@@ -1514,12 +1688,12 @@ function serverRoutes(limits) {
|
|
|
1514
1688
|
{ method: "post", path: "/indexes/:name/_wait-for-writes", handler: "waitForWrites", options: named },
|
|
1515
1689
|
{ method: "get", path: "/indexes/:name/documents/:id", handler: "get", options: document },
|
|
1516
1690
|
{ method: "get", path: "/indexes/:name/documents/:id/_exists", handler: "exists", options: document },
|
|
1517
|
-
{ method: "put", path: "/indexes/:name/documents/:id", handler: "put", options:
|
|
1518
|
-
{ method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options:
|
|
1691
|
+
{ method: "put", path: "/indexes/:name/documents/:id", handler: "put", options: documentWriteBody },
|
|
1692
|
+
{ method: "patch", path: "/indexes/:name/documents/:id", handler: "patch", options: documentWriteBody },
|
|
1519
1693
|
{ method: "del", path: "/indexes/:name/documents/:id", handler: "remove", options: document },
|
|
1520
|
-
{ method: "post", path: "/indexes/:name/search", handler: "search", options:
|
|
1521
|
-
{ method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options:
|
|
1522
|
-
{ method: "post", path: "/indexes/:name/suggest", handler: "suggest", options:
|
|
1694
|
+
{ method: "post", path: "/indexes/:name/search", handler: "search", options: readBody2 },
|
|
1695
|
+
{ method: "post", path: "/indexes/:name/search/preflight", handler: "preflight", options: readBody2 },
|
|
1696
|
+
{ method: "post", path: "/indexes/:name/suggest", handler: "suggest", options: readBody2 },
|
|
1523
1697
|
{ method: "post", path: "/indexes/:name/_checkpoint", handler: "checkpoint", options: named },
|
|
1524
1698
|
{ method: "post", path: "/indexes/:name/_rebuild-analysis", handler: "rebuildAnalysis", options: named },
|
|
1525
1699
|
{ method: "get", path: "/indexes/:name/snapshot", handler: "snapshot", options: named },
|
|
@@ -1656,4 +1830,4 @@ async function departureConfirmation(word) {
|
|
|
1656
1830
|
if (waiting.async) await waiting.value;
|
|
1657
1831
|
}
|
|
1658
1832
|
|
|
1659
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits,
|
|
1833
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MAX_LONG_QUERIES_IN_FLIGHT, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_CLOSE_AT_EXIT_MS, REQUEST_THREAD_DEPARTURE_DRAIN_MS, REQUEST_THREAD_MOVE_AFTER_MS, RUNNING_TASK_TTL_MS, SEARCH_CLOCK_SAMPLE_UNITS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closeEveryThreadAtExit, closeWhenTheProcessExits, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createCloseAtExitWords, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, isRemoveOnlyBatch, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondAnswer, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { collectYoungGeneration, collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
|
|
2
|
+
import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-22YPZVMG.mjs';
|
|
3
|
+
|
|
4
|
+
// src/runtime/thread-heap.ts
|
|
5
|
+
var refusalListeners = /* @__PURE__ */ new Set();
|
|
6
|
+
var reservedBytes = 0;
|
|
7
|
+
var admittedBytes = 0;
|
|
8
|
+
var lastYoungCollectionAt = Number.NEGATIVE_INFINITY;
|
|
9
|
+
function reservedHeapBytes() {
|
|
10
|
+
return reservedBytes;
|
|
11
|
+
}
|
|
12
|
+
function admittedHeapBytes() {
|
|
13
|
+
return admittedBytes;
|
|
14
|
+
}
|
|
15
|
+
function admitHeapBytes(bytes) {
|
|
16
|
+
admittedBytes += bytes;
|
|
17
|
+
}
|
|
18
|
+
function chargeHeapBytes(bytes) {
|
|
19
|
+
reservedBytes += bytes;
|
|
20
|
+
if (bytes > 0) admittedBytes += bytes;
|
|
21
|
+
}
|
|
22
|
+
function reserveHeapBytes(bytes) {
|
|
23
|
+
reservedBytes += bytes;
|
|
24
|
+
admittedBytes += bytes;
|
|
25
|
+
let released = false;
|
|
26
|
+
return {
|
|
27
|
+
release() {
|
|
28
|
+
if (released) return;
|
|
29
|
+
released = true;
|
|
30
|
+
reservedBytes -= bytes;
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function onRefusalAfterCollection(listener) {
|
|
35
|
+
refusalListeners.add(listener);
|
|
36
|
+
return () => {
|
|
37
|
+
refusalListeners.delete(listener);
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
function reportRefusalAfterCollection(figures) {
|
|
41
|
+
for (const listener of refusalListeners) listener(figures);
|
|
42
|
+
}
|
|
43
|
+
function refusalCollector() {
|
|
44
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
45
|
+
let intervalMs = REFUSAL_COLLECTION_MIN_INTERVAL_MS;
|
|
46
|
+
return () => {
|
|
47
|
+
const startedAt = performance.now();
|
|
48
|
+
if (startedAt - lastCollectionAt < intervalMs) return false;
|
|
49
|
+
lastCollectionAt = startedAt;
|
|
50
|
+
if (!collectAndReturnPages()) return false;
|
|
51
|
+
intervalMs = Math.max(
|
|
52
|
+
REFUSAL_COLLECTION_MIN_INTERVAL_MS,
|
|
53
|
+
(performance.now() - startedAt) * REFUSAL_COLLECTION_TIME_MULTIPLE
|
|
54
|
+
);
|
|
55
|
+
return true;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
var collectGarbageBeforeRefusal = refusalCollector();
|
|
59
|
+
var collectGarbageBeforeAnswerRefusal = refusalCollector();
|
|
60
|
+
function collectYoungGenerationBeforeRefusal() {
|
|
61
|
+
const startedAt = performance.now();
|
|
62
|
+
if (startedAt - lastYoungCollectionAt < REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS) return false;
|
|
63
|
+
lastYoungCollectionAt = startedAt;
|
|
64
|
+
return collectYoungGeneration();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export { admitHeapBytes, admittedHeapBytes, chargeHeapBytes, collectGarbageBeforeAnswerRefusal, collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, onRefusalAfterCollection, reportRefusalAfterCollection, reserveHeapBytes, reservedHeapBytes };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-7EWQZQDT.mjs';
|
|
2
2
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
3
3
|
import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
6
|
-
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-BCSWODXP.mjs';
|
|
5
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-Q62NDKEC.mjs';
|
|
6
|
+
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-N53HN5VA.mjs';
|
|
7
7
|
import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
|
|
8
8
|
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-DG62H7OD.mjs';
|
|
9
9
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-NX7KU77V.mjs';
|
|
2
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-YCSRB3IN.mjs';
|
|
3
3
|
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-OCB7TMUQ.mjs';
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-KGVC6KJ3.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-BCSWODXP.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
8
8
|
|