@delali/narsil 0.2.4-next.59 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -1,6 +1,65 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { nativeSearchCoreIsEnabled
|
|
3
|
-
import {
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures, collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-BOFBLJFZ.mjs';
|
|
2
|
+
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
|
+
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
|
+
import { loadNativeCore } from './chunk-JZVARCRZ.mjs';
|
|
5
|
+
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
6
|
+
import { MAX_SORT_FIELDS } from './chunk-E74YLAIG.mjs';
|
|
7
|
+
import { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-RFVUDM5E.mjs';
|
|
8
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-BWJVNZPR.mjs';
|
|
9
|
+
import { mkdtemp, open, rm } from 'fs/promises';
|
|
10
|
+
import { tmpdir } from 'os';
|
|
11
|
+
import path from 'path';
|
|
12
|
+
|
|
13
|
+
// src/server/body-budget.ts
|
|
14
|
+
function createBodyBudgetBuffer(slotCount) {
|
|
15
|
+
return new SharedArrayBuffer(slotCount * BigInt64Array.BYTES_PER_ELEMENT);
|
|
16
|
+
}
|
|
17
|
+
function clearBodyBudgetSlot(buffer, slot) {
|
|
18
|
+
const counters = new BigInt64Array(buffer);
|
|
19
|
+
if (slot >= 0 && slot < counters.length) Atomics.store(counters, slot, 0n);
|
|
20
|
+
}
|
|
21
|
+
function createSharedBodyBudget(buffer, limitBytes, slot) {
|
|
22
|
+
const counters = new BigInt64Array(buffer);
|
|
23
|
+
function inFlight() {
|
|
24
|
+
let total = 0n;
|
|
25
|
+
for (let at = 0; at < counters.length; at++) total += Atomics.load(counters, at);
|
|
26
|
+
return Number(total);
|
|
27
|
+
}
|
|
28
|
+
function take(bytes) {
|
|
29
|
+
Atomics.add(counters, slot, BigInt(bytes));
|
|
30
|
+
}
|
|
31
|
+
function give(bytes) {
|
|
32
|
+
if (Atomics.load(counters, slot) >= BigInt(bytes)) Atomics.sub(counters, slot, BigInt(bytes));
|
|
33
|
+
else Atomics.store(counters, slot, 0n);
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
hold() {
|
|
37
|
+
let held = 0;
|
|
38
|
+
return {
|
|
39
|
+
get bytes() {
|
|
40
|
+
return held;
|
|
41
|
+
},
|
|
42
|
+
grow(bytes) {
|
|
43
|
+
if (bytes <= 0) return null;
|
|
44
|
+
take(bytes);
|
|
45
|
+
const total = inFlight();
|
|
46
|
+
const others = total - held - bytes;
|
|
47
|
+
if (others > 0 && total > limitBytes) {
|
|
48
|
+
give(bytes);
|
|
49
|
+
return { requestBytes: held + bytes, inFlightBytes: others, limitBytes };
|
|
50
|
+
}
|
|
51
|
+
held += bytes;
|
|
52
|
+
return null;
|
|
53
|
+
},
|
|
54
|
+
releaseAll() {
|
|
55
|
+
if (held === 0) return;
|
|
56
|
+
give(held);
|
|
57
|
+
held = 0;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
4
63
|
|
|
5
64
|
// src/server/capabilities.ts
|
|
6
65
|
var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
|
|
@@ -8,12 +67,16 @@ var TASK_CANCEL_CAPABILITY = "tasks.cancel";
|
|
|
8
67
|
var TASK_FILTER_CAPABILITY = "tasks.filter";
|
|
9
68
|
var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
10
69
|
var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
70
|
+
var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
71
|
+
var SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
11
72
|
var SERVER_CAPABILITIES = [
|
|
12
73
|
ASYNC_IMPORT_CAPABILITY,
|
|
13
74
|
TASK_CANCEL_CAPABILITY,
|
|
14
75
|
TASK_FILTER_CAPABILITY,
|
|
15
76
|
REBUILD_ANALYSIS_CAPABILITY,
|
|
16
|
-
INDEX_LIFECYCLE_CAPABILITY
|
|
77
|
+
INDEX_LIFECYCLE_CAPABILITY,
|
|
78
|
+
DOCUMENT_EXPORT_CAPABILITY,
|
|
79
|
+
SYNONYMS_CAPABILITY
|
|
17
80
|
];
|
|
18
81
|
|
|
19
82
|
// src/server/concurrency-gate.ts
|
|
@@ -113,11 +176,13 @@ var STATUS_BY_CODE = {
|
|
|
113
176
|
SEARCH_INVALID_MODE: 400,
|
|
114
177
|
SEARCH_INVALID_CURSOR: 400,
|
|
115
178
|
SEARCH_RESULT_WINDOW_EXCEEDED: 400,
|
|
179
|
+
SEARCH_WORK_CAP_EXCEEDED: 400,
|
|
116
180
|
LANGUAGE_NOT_SUPPORTED: 400,
|
|
117
181
|
CONFIG_INVALID: 400,
|
|
118
182
|
EMBEDDING_DIMENSION_MISMATCH: 400,
|
|
119
183
|
EMBEDDING_NO_SOURCE: 400,
|
|
120
184
|
EMBEDDING_CONFIG_INVALID: 400,
|
|
185
|
+
RERANK_CONFIG_INVALID: 400,
|
|
121
186
|
INDEX_NOT_FOUND: 404,
|
|
122
187
|
DOC_NOT_FOUND: 404,
|
|
123
188
|
INDEX_ALREADY_EXISTS: 409,
|
|
@@ -127,6 +192,7 @@ var STATUS_BY_CODE = {
|
|
|
127
192
|
DOC_ALREADY_EXISTS: 409,
|
|
128
193
|
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
129
194
|
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
195
|
+
RESTORE_MEMORY_EXCEEDED: 507,
|
|
130
196
|
WORKER_BUSY: 429,
|
|
131
197
|
PARTITION_CORRUPTED: 503,
|
|
132
198
|
WORKER_CRASHED: 503,
|
|
@@ -139,11 +205,13 @@ var STATUS_BY_CODE = {
|
|
|
139
205
|
PERSISTENCE_WAL_CORRUPT: 503,
|
|
140
206
|
PERSISTENCE_FSYNC_FAILED: 503,
|
|
141
207
|
EMBEDDING_FAILED: 503,
|
|
208
|
+
RERANK_FAILED: 503,
|
|
142
209
|
QUERY_ROUTING_FAILED: 503,
|
|
143
210
|
QUERY_PARTIAL_FAILURE: 503,
|
|
144
211
|
QUERY_NODE_TIMEOUT: 503,
|
|
145
212
|
QUERY_NO_ACTIVE_REPLICA: 503,
|
|
146
213
|
CLUSTER_OPERATION_UNSUPPORTED: 501,
|
|
214
|
+
CLUSTER_WRITE_BLOCKED: 429,
|
|
147
215
|
REPLICATION_ENTRY_INVALID: 503,
|
|
148
216
|
REPLICATION_INSYNC_REMOVAL_FAILED: 503,
|
|
149
217
|
REPLICATION_ROLLBACK_FAILED: 503,
|
|
@@ -164,13 +232,15 @@ var STATUS_BY_CODE = {
|
|
|
164
232
|
NODE_NOT_READY: 503,
|
|
165
233
|
NODE_NOT_CONTROLLER: 503,
|
|
166
234
|
NODE_BOOTSTRAP_FAILED: 503,
|
|
235
|
+
NODE_VERSION_INCOMPATIBLE: 409,
|
|
167
236
|
COORDINATOR_DEPENDENCY_MISSING: 500,
|
|
237
|
+
COORDINATOR_AUTH_FAILED: 502,
|
|
238
|
+
COORDINATOR_UNAVAILABLE: 503,
|
|
168
239
|
SNAPSHOT_SYNC_UNAUTHORIZED: 403,
|
|
169
240
|
SNAPSHOT_SYNC_REQUEST_INVALID: 400,
|
|
170
241
|
SNAPSHOT_SYNC_HEADER_INVALID: 400,
|
|
171
242
|
SNAPSHOT_SYNC_FRAME_INVALID: 400,
|
|
172
243
|
SNAPSHOT_SYNC_INDEX_NOT_FOUND: 404,
|
|
173
|
-
SNAPSHOT_SYNC_TOO_LARGE: 413,
|
|
174
244
|
SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: 413,
|
|
175
245
|
SNAPSHOT_SYNC_ABORTED: 409,
|
|
176
246
|
SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: 503,
|
|
@@ -210,26 +280,33 @@ function toHttpError(err) {
|
|
|
210
280
|
};
|
|
211
281
|
}
|
|
212
282
|
|
|
213
|
-
// src/json-encoding.ts
|
|
214
|
-
function toPlainNumbers(key, value) {
|
|
215
|
-
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
216
|
-
return Array.from(value);
|
|
217
|
-
}
|
|
218
|
-
return value;
|
|
219
|
-
}
|
|
220
|
-
function encodeJson(value) {
|
|
221
|
-
return JSON.stringify(value, toPlainNumbers);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
283
|
// src/server/constants.ts
|
|
225
284
|
var BYTES_PER_MB = 1024 * 1024;
|
|
226
285
|
var DEFAULT_MAX_BODY_BYTES = 16 * BYTES_PER_MB;
|
|
227
286
|
var DEFAULT_MAX_IMPORT_BYTES = 100 * BYTES_PER_MB;
|
|
287
|
+
var DEFAULT_MAX_RESTORE_BYTES = 64 * 1024 * BYTES_PER_MB;
|
|
288
|
+
var SPOOL_PAUSE_ABOVE_BYTES = 8 * BYTES_PER_MB;
|
|
289
|
+
var SNAPSHOT_ARCHIVE_CONTENT_TYPE = "application/x-tar";
|
|
290
|
+
var DOCUMENT_EXPORT_CONTENT_TYPE = "application/x-ndjson";
|
|
291
|
+
var RELAY_STREAM_CHUNKS_IN_FLIGHT = 4;
|
|
228
292
|
var DEFAULT_MAX_LINE_BYTES = 4 * BYTES_PER_MB;
|
|
229
293
|
var DEFAULT_IMPORT_BATCH_SIZE = 1e3;
|
|
230
294
|
var IMPORT_FLUSHES_IN_FLIGHT = 4;
|
|
231
295
|
var DEFAULT_MAX_CONCURRENT_REQUESTS = 0;
|
|
296
|
+
var DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 0.05;
|
|
297
|
+
var MAX_BODY_BYTES_IN_FLIGHT_FRACTION = 1;
|
|
298
|
+
var PARSE_HEAP_BLOCK_FRACTION = 0.9;
|
|
299
|
+
var PARSE_HEAP_READ_BLOCK_FRACTION = 0.95;
|
|
300
|
+
var PARSE_WORST_HEAP_BYTES_PER_BODY_BYTE = 24;
|
|
301
|
+
var PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE = 2;
|
|
302
|
+
var PARSE_OBJECT_HEAP_BYTES = 72;
|
|
303
|
+
var PARSE_ARRAY_HEAP_BYTES = 48;
|
|
304
|
+
var PARSE_PROPERTY_HEAP_BYTES = 112;
|
|
305
|
+
var PARSE_STRING_HEAP_BYTES = 48;
|
|
306
|
+
var PARSE_STRING_HEAP_BYTES_PER_BYTE = 2;
|
|
307
|
+
var PARSE_SCALAR_HEAP_BYTES = 16;
|
|
232
308
|
var DEFAULT_MAX_RESULT_WINDOW = 1e4;
|
|
309
|
+
var DEFAULT_MAX_RERANK_WINDOW = 100;
|
|
233
310
|
var DEFAULT_MAX_FETCH_DOCUMENTS = 1e4;
|
|
234
311
|
var DEFAULT_MAX_IMPORT_ERRORS = 100;
|
|
235
312
|
var DEFAULT_MAX_TASK_PAGE_SIZE = 1e3;
|
|
@@ -239,12 +316,128 @@ var MIN_UNBOUNDED_SERVER_LIMIT = 0;
|
|
|
239
316
|
var RESPONSE_STREAM_THRESHOLD_BYTES = 64 * 1024;
|
|
240
317
|
var MAX_INDEX_TOUCH_INTERVAL_MS = 1e3;
|
|
241
318
|
var INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT = 4;
|
|
319
|
+
var REQUEST_THREAD_DEPARTURE_DRAIN_MS = 100;
|
|
242
320
|
var DEFAULT_MAX_RETAINED_TASKS = 1e3;
|
|
243
321
|
var DEFAULT_TASK_PAGE_SIZE = 20;
|
|
244
322
|
var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
|
|
245
323
|
var RUNNING_TASK_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
246
324
|
var TERMINAL_TASK_TTL_MS = 60 * 60 * 1e3;
|
|
247
325
|
|
|
326
|
+
// src/server/parse-heap.ts
|
|
327
|
+
var PARSE_WOULD_FILL_MEMORY_REASON = "parse_would_fill_memory";
|
|
328
|
+
var PARSE_EXCEEDS_MEMORY_REASON = "parse_exceeds_memory";
|
|
329
|
+
var QUOTE = 34;
|
|
330
|
+
var BACKSLASH = 92;
|
|
331
|
+
var OPEN_OBJECT = 123;
|
|
332
|
+
var OPEN_ARRAY = 91;
|
|
333
|
+
var COLON = 58;
|
|
334
|
+
function isScalarByte(byte) {
|
|
335
|
+
return byte >= 48 && byte <= 57 || byte >= 97 && byte <= 122 || byte >= 65 && byte <= 90 || byte === 45 || byte === 43 || byte === 46;
|
|
336
|
+
}
|
|
337
|
+
function closingQuoteOf(raw, from) {
|
|
338
|
+
let at = raw.indexOf(QUOTE, from);
|
|
339
|
+
while (at !== -1) {
|
|
340
|
+
let backslashes = 0;
|
|
341
|
+
while (at - 1 - backslashes >= from && raw[at - 1 - backslashes] === BACKSLASH) backslashes++;
|
|
342
|
+
if (backslashes % 2 === 0) return at;
|
|
343
|
+
at = raw.indexOf(QUOTE, at + 1);
|
|
344
|
+
}
|
|
345
|
+
return raw.length;
|
|
346
|
+
}
|
|
347
|
+
function scannedParseBytes(raw) {
|
|
348
|
+
let objects = 0;
|
|
349
|
+
let arrays = 0;
|
|
350
|
+
let properties = 0;
|
|
351
|
+
let strings = 0;
|
|
352
|
+
let stringBytes = 0;
|
|
353
|
+
let scalars = 0;
|
|
354
|
+
let inScalar = false;
|
|
355
|
+
for (let at = 0; at < raw.length; at++) {
|
|
356
|
+
const byte = raw[at];
|
|
357
|
+
if (byte === QUOTE) {
|
|
358
|
+
const end = closingQuoteOf(raw, at + 1);
|
|
359
|
+
strings++;
|
|
360
|
+
stringBytes += end - at - 1;
|
|
361
|
+
at = end;
|
|
362
|
+
inScalar = false;
|
|
363
|
+
} else if (isScalarByte(byte)) {
|
|
364
|
+
if (!inScalar) scalars++;
|
|
365
|
+
inScalar = true;
|
|
366
|
+
} else {
|
|
367
|
+
inScalar = false;
|
|
368
|
+
if (byte === OPEN_OBJECT) objects++;
|
|
369
|
+
else if (byte === OPEN_ARRAY) arrays++;
|
|
370
|
+
else if (byte === COLON) properties++;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return raw.length * PARSE_TEXT_HEAP_BYTES_PER_BODY_BYTE + objects * PARSE_OBJECT_HEAP_BYTES + arrays * PARSE_ARRAY_HEAP_BYTES + properties * PARSE_PROPERTY_HEAP_BYTES + strings * PARSE_STRING_HEAP_BYTES + stringBytes * PARSE_STRING_HEAP_BYTES_PER_BYTE + scalars * PARSE_SCALAR_HEAP_BYTES;
|
|
374
|
+
}
|
|
375
|
+
var WRITE_PARSE_LINE = {
|
|
376
|
+
heap: Math.min(PARSE_HEAP_BLOCK_FRACTION, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
|
|
377
|
+
process: PARSE_HEAP_BLOCK_FRACTION
|
|
378
|
+
};
|
|
379
|
+
var READ_PARSE_LINE = { heap: PARSE_HEAP_READ_BLOCK_FRACTION, process: PARSE_HEAP_READ_BLOCK_FRACTION };
|
|
380
|
+
function roomIn(figures, line) {
|
|
381
|
+
return Math.min(
|
|
382
|
+
line.heap * figures.heapLimitBytes - figures.heapBytes - figures.youngHeapBytes,
|
|
383
|
+
line.process * figures.processLimitBytes - figures.processBytes
|
|
384
|
+
) - reservedHeapBytes();
|
|
385
|
+
}
|
|
386
|
+
function refusal(status, raw, parseBytes, figures, collectedGarbage, line) {
|
|
387
|
+
const blockBytes = Math.floor(line.heap * figures.heapLimitBytes);
|
|
388
|
+
const message = status === 413 ? `Parsing this ${raw.length}-byte body can take up to ${parseBytes} bytes of memory, more than the ${blockBytes} bytes that the server allows its heap even when idle. Send the request in smaller parts.` : `Parsing this ${raw.length}-byte body can take up to ${parseBytes} bytes of memory, which with the ${figures.heapBytes} bytes already in use would pass the server's limit of ${blockBytes} bytes. Retry once the requests in progress finish.`;
|
|
389
|
+
return {
|
|
390
|
+
status,
|
|
391
|
+
code: status === 413 ? ServerErrorCodes.PAYLOAD_TOO_LARGE : ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
392
|
+
message,
|
|
393
|
+
details: {
|
|
394
|
+
reason: status === 413 ? PARSE_EXCEEDS_MEMORY_REASON : PARSE_WOULD_FILL_MEMORY_REASON,
|
|
395
|
+
bodyBytes: raw.length,
|
|
396
|
+
parseBytes,
|
|
397
|
+
heapBytes: figures.heapBytes,
|
|
398
|
+
youngHeapBytes: figures.youngHeapBytes,
|
|
399
|
+
reservedBytes: reservedHeapBytes(),
|
|
400
|
+
blockBytes,
|
|
401
|
+
collectedGarbage
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
function admitted(parseBytes) {
|
|
406
|
+
admitHeapBytes(parseBytes);
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
function parseRefusalOf(raw, kind = "write") {
|
|
410
|
+
const line = kind === "read" ? READ_PARSE_LINE : WRITE_PARSE_LINE;
|
|
411
|
+
const recent = readRecentMemoryFigures();
|
|
412
|
+
if (recent === null) return null;
|
|
413
|
+
const worstBytes = raw.length * PARSE_WORST_HEAP_BYTES_PER_BODY_BYTE;
|
|
414
|
+
if (worstBytes <= roomIn(recent, line)) return admitted(worstBytes);
|
|
415
|
+
const parseBytes = Math.min(scannedParseBytes(raw), worstBytes);
|
|
416
|
+
if (parseBytes > line.heap * recent.heapLimitBytes) {
|
|
417
|
+
return refusal(413, raw, parseBytes, recent, false, line);
|
|
418
|
+
}
|
|
419
|
+
if (parseBytes <= roomIn(recent, line)) return admitted(parseBytes);
|
|
420
|
+
const figures = readMemoryFigures() ?? recent;
|
|
421
|
+
if (parseBytes <= roomIn(figures, line)) return admitted(parseBytes);
|
|
422
|
+
const collectedGarbage = collectGarbageBeforeRefusal();
|
|
423
|
+
const collectedYoung = !collectedGarbage && collectYoungGenerationBeforeRefusal();
|
|
424
|
+
const settled = collectedGarbage || collectedYoung ? readMemoryFigures() ?? figures : figures;
|
|
425
|
+
if (parseBytes <= roomIn(settled, line)) return admitted(parseBytes);
|
|
426
|
+
if (collectedGarbage && kind === "write") reportRefusalAfterCollection(settled);
|
|
427
|
+
return refusal(429, raw, parseBytes, settled, collectedGarbage, line);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// src/json-encoding.ts
|
|
431
|
+
function toPlainNumbers(key, value) {
|
|
432
|
+
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
433
|
+
return Array.from(value);
|
|
434
|
+
}
|
|
435
|
+
return value;
|
|
436
|
+
}
|
|
437
|
+
function encodeJson(value) {
|
|
438
|
+
return JSON.stringify(value, toPlainNumbers);
|
|
439
|
+
}
|
|
440
|
+
|
|
248
441
|
// src/server/response.ts
|
|
249
442
|
var STATUS_TEXT = {
|
|
250
443
|
200: "200 OK",
|
|
@@ -299,8 +492,59 @@ function sendError(res, status, code, message, details) {
|
|
|
299
492
|
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
300
493
|
});
|
|
301
494
|
}
|
|
302
|
-
function
|
|
303
|
-
streamBody(res, abort,
|
|
495
|
+
function sendText(res, text, contentType, abort) {
|
|
496
|
+
streamBody(res, abort, Buffer.from(text, "utf8"), 200, [["Content-Type", contentType]]);
|
|
497
|
+
}
|
|
498
|
+
async function sendStream(res, abort, source, headers) {
|
|
499
|
+
const reader = source.getReader();
|
|
500
|
+
let wake = null;
|
|
501
|
+
abort.onAbort(() => wake?.());
|
|
502
|
+
const waitUntilWritable = () => new Promise((resolve) => {
|
|
503
|
+
wake = () => {
|
|
504
|
+
wake = null;
|
|
505
|
+
resolve();
|
|
506
|
+
};
|
|
507
|
+
res.onWritable(() => {
|
|
508
|
+
wake?.();
|
|
509
|
+
return true;
|
|
510
|
+
});
|
|
511
|
+
});
|
|
512
|
+
let started = false;
|
|
513
|
+
const begin = () => {
|
|
514
|
+
if (started) return;
|
|
515
|
+
started = true;
|
|
516
|
+
res.writeStatus(statusLine(200));
|
|
517
|
+
for (const [key, value] of headers) res.writeHeader(key, value);
|
|
518
|
+
};
|
|
519
|
+
try {
|
|
520
|
+
for (; ; ) {
|
|
521
|
+
const step = await reader.read();
|
|
522
|
+
if (abort.aborted) {
|
|
523
|
+
await reader.cancel().catch(() => void 0);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
if (step.done) {
|
|
527
|
+
res.cork(() => {
|
|
528
|
+
begin();
|
|
529
|
+
res.end();
|
|
530
|
+
});
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
let writable = true;
|
|
534
|
+
res.cork(() => {
|
|
535
|
+
begin();
|
|
536
|
+
writable = res.write(step.value);
|
|
537
|
+
});
|
|
538
|
+
if (!writable && !abort.aborted) await waitUntilWritable();
|
|
539
|
+
}
|
|
540
|
+
} catch (err) {
|
|
541
|
+
await reader.cancel().catch(() => void 0);
|
|
542
|
+
if (abort.aborted) return;
|
|
543
|
+
if (!started) throw err;
|
|
544
|
+
res.cork(() => {
|
|
545
|
+
res.close();
|
|
546
|
+
});
|
|
547
|
+
}
|
|
304
548
|
}
|
|
305
549
|
function sendRelayed(res, abort, status, headers, body) {
|
|
306
550
|
if (abort.aborted) return;
|
|
@@ -316,12 +560,19 @@ function sendRelayed(res, abort, status, headers, body) {
|
|
|
316
560
|
}
|
|
317
561
|
|
|
318
562
|
// src/server/handler-utils.ts
|
|
319
|
-
function
|
|
563
|
+
function refusesParse(ctx, raw, kind) {
|
|
564
|
+
const refusal2 = parseRefusalOf(raw, kind);
|
|
565
|
+
if (refusal2 === null) return false;
|
|
566
|
+
if (!ctx.abort.aborted) sendError(ctx.res, refusal2.status, refusal2.code, refusal2.message, refusal2.details);
|
|
567
|
+
return true;
|
|
568
|
+
}
|
|
569
|
+
function parseJson(ctx, kind = "write") {
|
|
320
570
|
const raw = ctx.rawBody;
|
|
321
571
|
if (!raw || raw.length === 0) {
|
|
322
572
|
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
323
573
|
return void 0;
|
|
324
574
|
}
|
|
575
|
+
if (refusesParse(ctx, raw, kind)) return void 0;
|
|
325
576
|
try {
|
|
326
577
|
return JSON.parse(raw.toString("utf8"));
|
|
327
578
|
} catch {
|
|
@@ -329,9 +580,27 @@ function parseJson(ctx) {
|
|
|
329
580
|
return void 0;
|
|
330
581
|
}
|
|
331
582
|
}
|
|
583
|
+
async function parseWriteJson(ctx, writeBlock) {
|
|
584
|
+
const raw = ctx.rawBody;
|
|
585
|
+
if (!raw || raw.length === 0 || writeBlock === void 0) return parseJson(ctx);
|
|
586
|
+
const refusal2 = parseRefusalOf(raw);
|
|
587
|
+
if (refusal2 === null) {
|
|
588
|
+
try {
|
|
589
|
+
return JSON.parse(raw.toString("utf8"));
|
|
590
|
+
} catch {
|
|
591
|
+
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
592
|
+
return void 0;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
const block = refusal2.status === 429 ? await writeBlock(ctx.params[0]).catch(() => null) : null;
|
|
596
|
+
if (block !== null) respondError(ctx, block);
|
|
597
|
+
else if (!ctx.abort.aborted) sendError(ctx.res, refusal2.status, refusal2.code, refusal2.message, refusal2.details);
|
|
598
|
+
return void 0;
|
|
599
|
+
}
|
|
332
600
|
function parseJsonOptional(ctx) {
|
|
333
601
|
const raw = ctx.rawBody;
|
|
334
602
|
if (!raw || raw.length === 0) return {};
|
|
603
|
+
if (refusesParse(ctx, raw, "write")) return void 0;
|
|
335
604
|
try {
|
|
336
605
|
return JSON.parse(raw.toString("utf8"));
|
|
337
606
|
} catch {
|
|
@@ -365,6 +634,43 @@ function serializeBatchResult(result) {
|
|
|
365
634
|
};
|
|
366
635
|
}
|
|
367
636
|
|
|
637
|
+
// src/server/regex-policy.ts
|
|
638
|
+
function isRecord(value) {
|
|
639
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
640
|
+
}
|
|
641
|
+
function regexFieldIn(expression, depth) {
|
|
642
|
+
if (!isRecord(expression) || depth > MAX_FILTER_DEPTH) return null;
|
|
643
|
+
if (isRecord(expression.fields)) {
|
|
644
|
+
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
645
|
+
if (isRecord(filter) && Object.hasOwn(filter, "regex")) return fieldPath;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
for (const nested of [expression.and, expression.or]) {
|
|
649
|
+
if (!Array.isArray(nested)) continue;
|
|
650
|
+
for (const child of nested) {
|
|
651
|
+
const found = regexFieldIn(child, depth + 1);
|
|
652
|
+
if (found !== null) return found;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
return regexFieldIn(expression.not, depth + 1);
|
|
656
|
+
}
|
|
657
|
+
function refusedRegex(filters, allowRegex) {
|
|
658
|
+
if (allowRegex) return null;
|
|
659
|
+
const fieldPath = regexFieldIn(filters, 0);
|
|
660
|
+
if (fieldPath === null) return null;
|
|
661
|
+
return new NarsilError(
|
|
662
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
663
|
+
`This server refuses every "regex" test, because its operator set the allowRegex server option to false. Test field "${fieldPath}" with "contains" or "wildcard"`,
|
|
664
|
+
{ fieldPath, operator: "regex", option: "allowRegex" }
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
function refuseRegex(ctx, filters, allowRegex) {
|
|
668
|
+
const refused = refusedRegex(filters, allowRegex);
|
|
669
|
+
if (refused === null) return false;
|
|
670
|
+
respondError(ctx, refused);
|
|
671
|
+
return true;
|
|
672
|
+
}
|
|
673
|
+
|
|
368
674
|
// src/server/validation.ts
|
|
369
675
|
function isPlainObject(value) {
|
|
370
676
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -378,7 +684,28 @@ function overWindow(field, value, maxWindow) {
|
|
|
378
684
|
}
|
|
379
685
|
return null;
|
|
380
686
|
}
|
|
381
|
-
function
|
|
687
|
+
function overRerankWindow(rerank, maxRerankWindow) {
|
|
688
|
+
if (!isPlainObject(rerank)) return null;
|
|
689
|
+
const window = rerank.window;
|
|
690
|
+
if (typeof window === "number" && window > maxRerankWindow) {
|
|
691
|
+
return {
|
|
692
|
+
message: `Field "rerank.window" exceeds this server's maximum rerank window of ${maxRerankWindow}`,
|
|
693
|
+
details: { value: window, limit: maxRerankWindow }
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
return null;
|
|
697
|
+
}
|
|
698
|
+
function withServerRerankWindow(params, limits) {
|
|
699
|
+
const rerank = params.rerank;
|
|
700
|
+
if (rerank === void 0 || !isPlainObject(rerank) || rerank.window !== void 0) return params;
|
|
701
|
+
const depth = params.vector === void 0 ? void 0 : params.hybrid?.depth;
|
|
702
|
+
const fuses = params.term !== void 0 || params.mode === "hybrid";
|
|
703
|
+
const window = fuses && typeof depth === "number" && Number.isInteger(depth) ? Math.min(limits.maxRerankWindow, depth) : limits.maxRerankWindow;
|
|
704
|
+
if (window >= DEFAULT_RERANK_WINDOW) return params;
|
|
705
|
+
return { ...params, rerank: { ...rerank, window } };
|
|
706
|
+
}
|
|
707
|
+
function validateQuery(params, limits) {
|
|
708
|
+
const maxWindow = limits.maxResultWindow;
|
|
382
709
|
if (params.group && typeof params.group === "object" && "reduce" in params.group) {
|
|
383
710
|
return {
|
|
384
711
|
message: 'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
|
|
@@ -386,6 +713,8 @@ function validateQuery(params, maxWindow) {
|
|
|
386
713
|
}
|
|
387
714
|
const rowBound = overWindow("limit", params.limit, maxWindow) ?? overWindow("offset", params.offset, maxWindow);
|
|
388
715
|
if (rowBound) return rowBound;
|
|
716
|
+
const reranked = overRerankWindow(params.rerank, limits.maxRerankWindow);
|
|
717
|
+
if (reranked) return reranked;
|
|
389
718
|
if (params.group && typeof params.group === "object") {
|
|
390
719
|
const grouped = overWindow("group.maxPerGroup", params.group.maxPerGroup, maxWindow) ?? overWindow("group.limit", params.group.limit, maxWindow);
|
|
391
720
|
if (grouped) return grouped;
|
|
@@ -495,7 +824,14 @@ function validateRebalance(body) {
|
|
|
495
824
|
}
|
|
496
825
|
return null;
|
|
497
826
|
}
|
|
498
|
-
var TASK_TYPES = [
|
|
827
|
+
var TASK_TYPES = [
|
|
828
|
+
"optimizeVectors",
|
|
829
|
+
"rebalance",
|
|
830
|
+
"restore",
|
|
831
|
+
"import",
|
|
832
|
+
"importExport",
|
|
833
|
+
"rebuildAnalysis"
|
|
834
|
+
];
|
|
499
835
|
var TASK_STATUSES = ["running", "succeeded", "failed", "cancelled"];
|
|
500
836
|
function parseCountParam(raw, field, minimum) {
|
|
501
837
|
if (raw === null) return minimum;
|
|
@@ -551,7 +887,7 @@ function parseTaskListQuery(params, maxPageSize) {
|
|
|
551
887
|
|
|
552
888
|
// src/server/handlers/document-reads.ts
|
|
553
889
|
function createDocumentReadHandlers(deps) {
|
|
554
|
-
const { engine, limits } = deps;
|
|
890
|
+
const { engine, limits, allowRegex } = deps;
|
|
555
891
|
async function get(ctx) {
|
|
556
892
|
try {
|
|
557
893
|
const document = await engine.get(ctx.params[0], ctx.params[1]);
|
|
@@ -579,7 +915,7 @@ function createDocumentReadHandlers(deps) {
|
|
|
579
915
|
}
|
|
580
916
|
}
|
|
581
917
|
async function multiGet(ctx) {
|
|
582
|
-
const body = parseJson(ctx);
|
|
918
|
+
const body = parseJson(ctx, "read");
|
|
583
919
|
if (!body) return;
|
|
584
920
|
const failure = validateMultiGet(body, limits.maxFetchDocuments);
|
|
585
921
|
if (failure) {
|
|
@@ -596,13 +932,14 @@ function createDocumentReadHandlers(deps) {
|
|
|
596
932
|
}
|
|
597
933
|
}
|
|
598
934
|
async function list(ctx) {
|
|
599
|
-
const params = parseJson(ctx);
|
|
935
|
+
const params = parseJson(ctx, "read");
|
|
600
936
|
if (!params) return;
|
|
601
937
|
const failure = validateList(params, limits.maxFetchDocuments);
|
|
602
938
|
if (failure) {
|
|
603
939
|
rejectInvalid(ctx, failure);
|
|
604
940
|
return;
|
|
605
941
|
}
|
|
942
|
+
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
606
943
|
try {
|
|
607
944
|
respondJson(ctx, await engine.listDocuments(ctx.params[0], params));
|
|
608
945
|
} catch (err) {
|
|
@@ -630,8 +967,90 @@ function createVersionHandler(deps) {
|
|
|
630
967
|
}
|
|
631
968
|
return { report };
|
|
632
969
|
}
|
|
970
|
+
var NotAdmittedError = class extends Error {
|
|
971
|
+
};
|
|
972
|
+
async function removeDirectory(directory) {
|
|
973
|
+
if (directory !== null) await rm(directory, { recursive: true, force: true });
|
|
974
|
+
}
|
|
975
|
+
function spoolBody(res, maxBytes, abort, admitted2) {
|
|
976
|
+
return new Promise((resolve, reject) => {
|
|
977
|
+
const state = {
|
|
978
|
+
directory: null,
|
|
979
|
+
handle: null,
|
|
980
|
+
chain: Promise.resolve(),
|
|
981
|
+
queued: 0,
|
|
982
|
+
total: 0,
|
|
983
|
+
paused: true,
|
|
984
|
+
done: false
|
|
985
|
+
};
|
|
986
|
+
const fail = (error, answer) => {
|
|
987
|
+
if (state.done) return;
|
|
988
|
+
state.done = true;
|
|
989
|
+
if (answer && !abort.aborted) {
|
|
990
|
+
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "The server could not store the request body");
|
|
991
|
+
}
|
|
992
|
+
void state.chain.catch(() => void 0).then(async () => {
|
|
993
|
+
await state.handle?.close().catch(() => void 0);
|
|
994
|
+
await removeDirectory(state.directory).catch(() => void 0);
|
|
995
|
+
}).finally(() => reject(error));
|
|
996
|
+
};
|
|
997
|
+
const failFromChain = (error) => {
|
|
998
|
+
if (error instanceof NotAdmittedError) fail(error, false);
|
|
999
|
+
else fail(error instanceof Error ? error : new Error(String(error)), true);
|
|
1000
|
+
};
|
|
1001
|
+
res.pause();
|
|
1002
|
+
state.chain = (async () => {
|
|
1003
|
+
if (!await admitted2) throw new NotAdmittedError("the request was not admitted");
|
|
1004
|
+
state.directory = await mkdtemp(path.join(tmpdir(), "narsil-spool-"));
|
|
1005
|
+
state.handle = await open(path.join(state.directory, "body"), "w");
|
|
1006
|
+
if (!abort.aborted && !state.done && state.queued <= SPOOL_PAUSE_ABOVE_BYTES) {
|
|
1007
|
+
state.paused = false;
|
|
1008
|
+
res.resume();
|
|
1009
|
+
}
|
|
1010
|
+
})();
|
|
1011
|
+
state.chain.catch(failFromChain);
|
|
1012
|
+
abort.onAbort(() => fail(new Error("aborted"), false));
|
|
1013
|
+
res.onData((chunk, isLast) => {
|
|
1014
|
+
if (state.done || abort.aborted) return;
|
|
1015
|
+
state.total += chunk.byteLength;
|
|
1016
|
+
if (maxBytes > 0 && state.total > maxBytes) {
|
|
1017
|
+
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1018
|
+
fail(new Error("payload too large"), false);
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
const copy = new Uint8Array(chunk.slice(0));
|
|
1022
|
+
state.queued += copy.byteLength;
|
|
1023
|
+
if (!state.paused && state.queued > SPOOL_PAUSE_ABOVE_BYTES) {
|
|
1024
|
+
state.paused = true;
|
|
1025
|
+
res.pause();
|
|
1026
|
+
}
|
|
1027
|
+
state.chain = state.chain.then(async () => {
|
|
1028
|
+
if (state.handle === null) throw new Error("the spool file is not open");
|
|
1029
|
+
await state.handle.write(copy);
|
|
1030
|
+
state.queued -= copy.byteLength;
|
|
1031
|
+
if (state.paused && state.queued <= SPOOL_PAUSE_ABOVE_BYTES && !abort.aborted && !state.done) {
|
|
1032
|
+
state.paused = false;
|
|
1033
|
+
res.resume();
|
|
1034
|
+
}
|
|
1035
|
+
});
|
|
1036
|
+
state.chain.catch(failFromChain);
|
|
1037
|
+
if (!isLast) return;
|
|
1038
|
+
state.chain.then(async () => {
|
|
1039
|
+
if (state.done) return;
|
|
1040
|
+
state.done = true;
|
|
1041
|
+
await state.handle?.close();
|
|
1042
|
+
resolve({
|
|
1043
|
+
path: path.join(state.directory ?? "", "body"),
|
|
1044
|
+
bytes: state.total,
|
|
1045
|
+
remove: () => removeDirectory(state.directory)
|
|
1046
|
+
});
|
|
1047
|
+
}, failFromChain);
|
|
1048
|
+
});
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
633
1051
|
|
|
634
1052
|
// src/server/request.ts
|
|
1053
|
+
var REQUEST_BYTES_IN_FLIGHT_REASON = "request_bytes_in_flight";
|
|
635
1054
|
function initAbortHandler(res) {
|
|
636
1055
|
const listeners = [];
|
|
637
1056
|
let aborted = false;
|
|
@@ -664,7 +1083,16 @@ function bodyOfExactly(body, filled) {
|
|
|
664
1083
|
exact.set(body.subarray(0, filled));
|
|
665
1084
|
return exact;
|
|
666
1085
|
}
|
|
667
|
-
function
|
|
1086
|
+
function refuseBodyBytes(res, refusal2) {
|
|
1087
|
+
sendError(
|
|
1088
|
+
res,
|
|
1089
|
+
429,
|
|
1090
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1091
|
+
`The server is holding ${refusal2.inFlightBytes} bytes of other request bodies, and this body's ${refusal2.requestBytes} bytes would take the total past its limit of ${refusal2.limitBytes} bytes. Retry once the requests in progress finish.`,
|
|
1092
|
+
{ reason: REQUEST_BYTES_IN_FLIGHT_REASON, ...refusal2 }
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
function readBody(res, maxBytes, abort, hold, declaredBytes = null) {
|
|
668
1096
|
return new Promise((resolve, reject) => {
|
|
669
1097
|
if (abort.aborted) {
|
|
670
1098
|
reject(new Error("aborted"));
|
|
@@ -675,6 +1103,12 @@ function readBody(res, maxBytes, abort, declaredBytes = null) {
|
|
|
675
1103
|
let filled = 0;
|
|
676
1104
|
let body = null;
|
|
677
1105
|
const reserved = declaredBytes !== null && declaredBytes <= maxBytes ? declaredBytes : 0;
|
|
1106
|
+
const refusal2 = hold.grow(reserved);
|
|
1107
|
+
if (refusal2 !== null) {
|
|
1108
|
+
refuseBodyBytes(res, refusal2);
|
|
1109
|
+
reject(new Error("request bytes in flight"));
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
678
1112
|
abort.onAbort(() => {
|
|
679
1113
|
if (done) return;
|
|
680
1114
|
done = true;
|
|
@@ -691,6 +1125,13 @@ function readBody(res, maxBytes, abort, declaredBytes = null) {
|
|
|
691
1125
|
reject(new Error("payload too large"));
|
|
692
1126
|
return;
|
|
693
1127
|
}
|
|
1128
|
+
const chunkRefusal = total > hold.bytes ? hold.grow(total - hold.bytes) : null;
|
|
1129
|
+
if (chunkRefusal !== null) {
|
|
1130
|
+
done = true;
|
|
1131
|
+
if (!abort.aborted) refuseBodyBytes(res, chunkRefusal);
|
|
1132
|
+
reject(new Error("request bytes in flight"));
|
|
1133
|
+
return;
|
|
1134
|
+
}
|
|
694
1135
|
const arrived = new Uint8Array(chunk);
|
|
695
1136
|
const firstSize = isLast ? arrived.length : Math.max(reserved, arrived.length);
|
|
696
1137
|
body = bodyWithRoomFor(body ?? Buffer.allocUnsafeSlow(firstSize), filled, total, maxBytes);
|
|
@@ -727,8 +1168,21 @@ function sinkWritingAfterStatus(res, headers) {
|
|
|
727
1168
|
res.writeHeader(key, value);
|
|
728
1169
|
return sink;
|
|
729
1170
|
},
|
|
730
|
-
end(body) {
|
|
731
|
-
res.end(body);
|
|
1171
|
+
end(body, closeConnection) {
|
|
1172
|
+
res.end(body, closeConnection);
|
|
1173
|
+
return sink;
|
|
1174
|
+
},
|
|
1175
|
+
write(chunk) {
|
|
1176
|
+
return res.write(chunk);
|
|
1177
|
+
},
|
|
1178
|
+
pause() {
|
|
1179
|
+
res.pause();
|
|
1180
|
+
},
|
|
1181
|
+
resume() {
|
|
1182
|
+
res.resume();
|
|
1183
|
+
},
|
|
1184
|
+
close() {
|
|
1185
|
+
res.close();
|
|
732
1186
|
return sink;
|
|
733
1187
|
},
|
|
734
1188
|
endWithoutBody() {
|
|
@@ -782,7 +1236,7 @@ function sendDenial(res, authorization) {
|
|
|
782
1236
|
sendError(res, authorization.denial.status, authorization.denial.code, authorization.denial.message);
|
|
783
1237
|
}
|
|
784
1238
|
function createRouteRunner(deps) {
|
|
785
|
-
const { authorize, gate, corsHeaders } = deps;
|
|
1239
|
+
const { authorize, gate, bodyBudget, corsHeaders, track } = deps;
|
|
786
1240
|
return (handler, opts) => {
|
|
787
1241
|
const paramCount = opts.paramCount ?? 0;
|
|
788
1242
|
const needsBody = opts.needsBody ?? false;
|
|
@@ -827,20 +1281,37 @@ function createRouteRunner(deps) {
|
|
|
827
1281
|
);
|
|
828
1282
|
return;
|
|
829
1283
|
}
|
|
830
|
-
const
|
|
1284
|
+
const release = track?.(res, abort);
|
|
1285
|
+
const bodyHold = bodyBudget.hold();
|
|
1286
|
+
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, bodyHold, declaredBytes) : Promise.resolve(null);
|
|
831
1287
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
832
|
-
|
|
833
|
-
|
|
1288
|
+
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
1289
|
+
res,
|
|
1290
|
+
opts.maxBytes,
|
|
1291
|
+
abort,
|
|
1292
|
+
hookPromise.then((authorization) => authorization.allowed)
|
|
1293
|
+
).catch(() => null) : Promise.resolve(null);
|
|
1294
|
+
const running = Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
1295
|
+
const discardSpool = () => {
|
|
1296
|
+
void spooledBody?.remove().catch(() => void 0);
|
|
1297
|
+
};
|
|
1298
|
+
if (abort.aborted) {
|
|
1299
|
+
discardSpool();
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
834
1302
|
if (!authorization.allowed) {
|
|
1303
|
+
discardSpool();
|
|
835
1304
|
sendDenial(res, authorization);
|
|
836
1305
|
return;
|
|
837
1306
|
}
|
|
1307
|
+
if (opts.spoolsBody === true && spooledBody === null) return;
|
|
838
1308
|
if (gated && !gate.tryAcquire()) {
|
|
1309
|
+
discardSpool();
|
|
839
1310
|
sendError(res, 429, ServerErrorCodes.TOO_MANY_REQUESTS, "The server is at capacity; retry shortly");
|
|
840
1311
|
return;
|
|
841
1312
|
}
|
|
842
1313
|
try {
|
|
843
|
-
await handler({ res, params, query, contentType, rawBody, abort, hookContext });
|
|
1314
|
+
await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
|
|
844
1315
|
} catch (err) {
|
|
845
1316
|
if (!abort.aborted) {
|
|
846
1317
|
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
@@ -849,7 +1320,8 @@ function createRouteRunner(deps) {
|
|
|
849
1320
|
if (gated) gate.release();
|
|
850
1321
|
}
|
|
851
1322
|
}).catch(() => {
|
|
852
|
-
});
|
|
1323
|
+
}).finally(() => bodyHold.releaseAll());
|
|
1324
|
+
if (release !== void 0) void running.then(release);
|
|
853
1325
|
};
|
|
854
1326
|
};
|
|
855
1327
|
}
|
|
@@ -870,6 +1342,8 @@ function serverRoutes(limits) {
|
|
|
870
1342
|
{ method: "get", path: "/version", handler: "version", options: probe },
|
|
871
1343
|
{ method: "get", path: "/capabilities", handler: "capabilities", options: probe },
|
|
872
1344
|
{ method: "get", path: "/cluster", handler: "clusterTopology", options: bare },
|
|
1345
|
+
{ method: "get", path: "/cluster/health", handler: "clusterHealth", options: bare },
|
|
1346
|
+
{ method: "get", path: "/metrics", handler: "metrics", options: bare },
|
|
873
1347
|
{ method: "get", path: "/stats/memory", handler: "memory", options: bare },
|
|
874
1348
|
{ method: "get", path: "/tasks", handler: "listTasks", options: bare },
|
|
875
1349
|
{ method: "get", path: "/tasks/:id", handler: "getTask", options: named },
|
|
@@ -880,6 +1354,12 @@ function serverRoutes(limits) {
|
|
|
880
1354
|
{ method: "get", path: "/indexes/:name/stats", handler: "indexStats", options: named },
|
|
881
1355
|
{ method: "get", path: "/indexes/:name/partitions", handler: "indexPartitions", options: named },
|
|
882
1356
|
{ method: "get", path: "/indexes/:name/cluster", handler: "clusterAllocation", options: named },
|
|
1357
|
+
{
|
|
1358
|
+
method: "put",
|
|
1359
|
+
path: "/indexes/:name/replication-factor",
|
|
1360
|
+
handler: "clusterReplicationFactor",
|
|
1361
|
+
options: namedBody
|
|
1362
|
+
},
|
|
883
1363
|
{ method: "post", path: "/indexes/:name/_clear", handler: "clearIndex", options: named },
|
|
884
1364
|
{ method: "post", path: "/indexes/:name/_open", handler: "openIndex", options: named },
|
|
885
1365
|
{ method: "post", path: "/indexes/:name/_close", handler: "closeIndex", options: named },
|
|
@@ -911,11 +1391,20 @@ function serverRoutes(limits) {
|
|
|
911
1391
|
{ method: "post", path: "/indexes/:name/vectors/_optimize", handler: "optimize", options: namedBody },
|
|
912
1392
|
{ method: "post", path: "/indexes/:name/_rebalance", handler: "rebalance", options: namedBody },
|
|
913
1393
|
{ method: "post", path: "/indexes/:name/partition-config", handler: "partitionConfig", options: namedBody },
|
|
1394
|
+
{ method: "get", path: "/indexes/:name/synonyms", handler: "readSynonyms", options: named },
|
|
1395
|
+
{ method: "post", path: "/indexes/:name/synonyms", handler: "replaceSynonyms", options: namedBody },
|
|
914
1396
|
{
|
|
915
1397
|
method: "post",
|
|
916
1398
|
path: "/indexes/:name/restore",
|
|
917
1399
|
handler: "restore",
|
|
918
|
-
options: { paramCount: 1,
|
|
1400
|
+
options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
|
|
1401
|
+
},
|
|
1402
|
+
{ method: "get", path: "/indexes/:name/export", handler: "exportDocuments", options: named },
|
|
1403
|
+
{
|
|
1404
|
+
method: "post",
|
|
1405
|
+
path: "/indexes/:name/import",
|
|
1406
|
+
handler: "importExport",
|
|
1407
|
+
options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
|
|
919
1408
|
}
|
|
920
1409
|
];
|
|
921
1410
|
}
|
|
@@ -974,4 +1463,26 @@ function childAppOf(app) {
|
|
|
974
1463
|
return candidate;
|
|
975
1464
|
}
|
|
976
1465
|
|
|
977
|
-
|
|
1466
|
+
// src/server/request-threads/departure.ts
|
|
1467
|
+
var DEPARTURE_WORD = 0;
|
|
1468
|
+
var AWAITING_DEPARTURE = 0;
|
|
1469
|
+
var DEPARTURE_CONFIRMED = 1;
|
|
1470
|
+
function createDepartureWord() {
|
|
1471
|
+
return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
1472
|
+
}
|
|
1473
|
+
function confirmDeparture(word) {
|
|
1474
|
+
Atomics.store(word, DEPARTURE_WORD, DEPARTURE_CONFIRMED);
|
|
1475
|
+
Atomics.notify(word, DEPARTURE_WORD);
|
|
1476
|
+
}
|
|
1477
|
+
function departureConfirmed(word) {
|
|
1478
|
+
return Atomics.load(word, DEPARTURE_WORD) === DEPARTURE_CONFIRMED;
|
|
1479
|
+
}
|
|
1480
|
+
function awaitDepartureConfirmation(word) {
|
|
1481
|
+
while (!departureConfirmed(word)) Atomics.wait(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1482
|
+
}
|
|
1483
|
+
async function departureConfirmation(word) {
|
|
1484
|
+
const waiting = Atomics.waitAsync(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1485
|
+
if (waiting.async) await waiting.value;
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearBodyBudgetSlot, clearGateSlot, closingQuoteOf, confirmDeparture, corsHeaderReader, createBodyBudgetBuffer, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedBodyBudget, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseRefusalOf, parseTaskListQuery, parseWriteJson, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, sendText, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|