@delali/narsil 0.3.1-next.71 → 0.3.1-next.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-CYeAWYxQ.d.ts} +19 -4
- package/dist/{chunk-3JGWULDF.mjs → chunk-2WLD2NFW.mjs} +5 -5
- package/dist/{chunk-LMO5TJUS.mjs → chunk-3YKWSO54.mjs} +642 -196
- package/dist/{chunk-HWQLTLNG.mjs → chunk-4MQ5F4IL.mjs} +1 -1
- package/dist/{chunk-FVLQ63B7.mjs → chunk-AFKH4ZK3.mjs} +30 -6
- package/dist/{chunk-653BNMAF.mjs → chunk-AK3BLGCI.mjs} +195 -25
- package/dist/{chunk-IOR2K26O.mjs → chunk-CVFAFZSN.mjs} +8000 -4391
- package/dist/{chunk-USKFU7J5.mjs → chunk-I64LXKO3.mjs} +1 -1
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-2STVUTNA.mjs → chunk-LPBWCHBE.mjs} +194 -13
- package/dist/{chunk-YNVIUGF5.mjs → chunk-PU43WDY4.mjs} +4 -2
- package/dist/{chunk-M76V3OGJ.mjs → chunk-THMSFOJO.mjs} +1289 -1405
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-WVDFG45M.mjs} +93 -95
- package/dist/chunk-XW25X6AU.mjs +614 -0
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-ZULSEZCE.mjs} +68 -28
- package/dist/{client-COek5GXK.d.ts → client-D698E5ge.d.ts} +77 -26
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +45 -22
- package/dist/{config-rBpbgTw8.d.ts → config-Bv8etwUZ.d.ts} +11 -4
- package/dist/{config-Dfanyqqt.d.ts → config-QXU3HMpq.d.ts} +26 -2
- package/dist/direct-executor-5SRDFZAS.mjs +13 -0
- package/dist/distribution/coordinator/etcd.d.ts +29 -16
- package/dist/distribution/coordinator/etcd.mjs +163 -76
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +22 -2
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +6 -3
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +40 -16
- package/dist/distribution.d.ts +9 -9
- package/dist/distribution.mjs +682 -388
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-igk0aMpQ.d.ts → engine-D_OzKZ-6.d.ts} +97 -188
- package/dist/index.browser.mjs +11671 -7427
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +13 -14
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-B4jMmygw.d.ts} +279 -27
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -17
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -16
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -1
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +213 -34
- package/dist/{tasks-BZt_zes6.d.ts → tasks-DpgCpW1x.d.ts} +3 -3
- package/dist/{thread-JEVAYETR.mjs → thread-3LNVA43V.mjs} +94 -27
- package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-GIVAGD3Q.mjs +0 -25
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
|
@@ -31,6 +31,7 @@ var ErrorCodes = {
|
|
|
31
31
|
SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
|
|
32
32
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
33
33
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
34
|
+
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
34
35
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
35
36
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
36
37
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
@@ -71,7 +72,6 @@ var ErrorCodes = {
|
|
|
71
72
|
SNAPSHOT_SYNC_UNAUTHORIZED: "SNAPSHOT_SYNC_UNAUTHORIZED",
|
|
72
73
|
SNAPSHOT_SYNC_REQUEST_INVALID: "SNAPSHOT_SYNC_REQUEST_INVALID",
|
|
73
74
|
SNAPSHOT_SYNC_INDEX_NOT_FOUND: "SNAPSHOT_SYNC_INDEX_NOT_FOUND",
|
|
74
|
-
SNAPSHOT_SYNC_TOO_LARGE: "SNAPSHOT_SYNC_TOO_LARGE",
|
|
75
75
|
SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: "SNAPSHOT_SYNC_CAPACITY_EXHAUSTED",
|
|
76
76
|
SNAPSHOT_SYNC_SNAPSHOT_FAILED: "SNAPSHOT_SYNC_SNAPSHOT_FAILED",
|
|
77
77
|
SNAPSHOT_SYNC_DECODE_FAILED: "SNAPSHOT_SYNC_DECODE_FAILED",
|
|
@@ -15,6 +15,31 @@ function exceedsCodePoints(value, maximum) {
|
|
|
15
15
|
}
|
|
16
16
|
return false;
|
|
17
17
|
}
|
|
18
|
+
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
19
|
+
function holdsUnpairedSurrogate(value) {
|
|
20
|
+
if (!SURROGATE_UNIT.test(value)) return false;
|
|
21
|
+
for (let i = 0; i < value.length; i++) {
|
|
22
|
+
const unit = value.charCodeAt(i);
|
|
23
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
24
|
+
if (unit >= 56320) return true;
|
|
25
|
+
const low = value.charCodeAt(i + 1);
|
|
26
|
+
if (!(low >= 56320 && low < 57344)) return true;
|
|
27
|
+
i++;
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
function codePointCount(value) {
|
|
32
|
+
let counted = 0;
|
|
33
|
+
for (let i = 0; i < value.length; i++) {
|
|
34
|
+
const unit = value.charCodeAt(i);
|
|
35
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
36
|
+
const low = value.charCodeAt(i + 1);
|
|
37
|
+
if (low >= 56320 && low < 57344) i++;
|
|
38
|
+
}
|
|
39
|
+
counted++;
|
|
40
|
+
}
|
|
41
|
+
return counted;
|
|
42
|
+
}
|
|
18
43
|
function adjustUtf16Unit(unit) {
|
|
19
44
|
if (unit < 55296) return unit;
|
|
20
45
|
if (unit < 57344) return unit + 8192;
|
|
@@ -65,6 +90,10 @@ function multiFoldTable() {
|
|
|
65
90
|
|
|
66
91
|
// src/core/ordering/fold-compare.ts
|
|
67
92
|
var EXHAUSTED = -1;
|
|
93
|
+
function foldOf(codePoint, single, multi) {
|
|
94
|
+
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
95
|
+
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
96
|
+
}
|
|
68
97
|
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
69
98
|
if (cursor.pending !== null) {
|
|
70
99
|
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
@@ -78,9 +107,6 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
78
107
|
if (cursor.index >= text.length) return EXHAUSTED;
|
|
79
108
|
let cp = text.charCodeAt(cursor.index);
|
|
80
109
|
cursor.index++;
|
|
81
|
-
if (cp < 128) {
|
|
82
|
-
return cp >= 65 && cp <= 90 ? cp + 32 : cp;
|
|
83
|
-
}
|
|
84
110
|
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
85
111
|
const low = text.charCodeAt(cursor.index);
|
|
86
112
|
if (low >= 56320 && low < 57344) {
|
|
@@ -88,15 +114,13 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
88
114
|
cursor.index++;
|
|
89
115
|
}
|
|
90
116
|
}
|
|
91
|
-
const
|
|
92
|
-
if (
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
if (expansion.length > 1) {
|
|
96
|
-
cursor.pending = expansion;
|
|
117
|
+
const folded = foldOf(cp, single, multi);
|
|
118
|
+
if (typeof folded === "number") return folded;
|
|
119
|
+
if (folded.length > 1) {
|
|
120
|
+
cursor.pending = folded;
|
|
97
121
|
cursor.pendingIndex = 1;
|
|
98
122
|
}
|
|
99
|
-
return
|
|
123
|
+
return folded[0];
|
|
100
124
|
}
|
|
101
125
|
function compareCaseFolded(a, b) {
|
|
102
126
|
const shorter = a.length < b.length ? a.length : b.length;
|
|
@@ -123,6 +147,28 @@ function compareSortStrings(a, b) {
|
|
|
123
147
|
if (folded !== 0) return folded;
|
|
124
148
|
return compareCodePoints(a, b);
|
|
125
149
|
}
|
|
150
|
+
function appendFoldedCodePoint(codePoint, out) {
|
|
151
|
+
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
152
|
+
if (typeof folded === "number") {
|
|
153
|
+
out.push(folded);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
157
|
+
}
|
|
158
|
+
function appendFoldedCodePoints(text, out) {
|
|
159
|
+
const single = singleFoldTable();
|
|
160
|
+
const multi = multiFoldTable();
|
|
161
|
+
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
162
|
+
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
163
|
+
out.push(codePoint);
|
|
164
|
+
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function foldedCodePointCount(text) {
|
|
168
|
+
const folded = [];
|
|
169
|
+
appendFoldedCodePoints(text, folded);
|
|
170
|
+
return folded.length;
|
|
171
|
+
}
|
|
126
172
|
|
|
127
173
|
// src/core/ordering/constants.ts
|
|
128
174
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
@@ -244,4 +290,4 @@ function compareSortValues(aValues, bValues, directions) {
|
|
|
244
290
|
return 0;
|
|
245
291
|
}
|
|
246
292
|
|
|
247
|
-
export { SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, exceedsCodePoints, isSortMode, readSortField, sortModeOf, toComparableSortValue, toReducedSortValue, truncateSortString };
|
|
293
|
+
export { SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, appendFoldedCodePoint, codePointCount, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, exceedsCodePoints, foldedCodePointCount, holdsUnpairedSurrogate, isSortMode, readSortField, sortModeOf, toComparableSortValue, toReducedSortValue, truncateSortString };
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { MAX_SORT_FIELDS } from './chunk-
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
3
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_SORT_FIELDS } from './chunk-AK3BLGCI.mjs';
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-PU43WDY4.mjs';
|
|
3
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
4
|
+
import { mkdtemp, open, rm } from 'fs/promises';
|
|
5
|
+
import { tmpdir } from 'os';
|
|
6
|
+
import path from 'path';
|
|
4
7
|
|
|
5
8
|
// src/server/capabilities.ts
|
|
6
9
|
var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
|
|
@@ -8,12 +11,14 @@ var TASK_CANCEL_CAPABILITY = "tasks.cancel";
|
|
|
8
11
|
var TASK_FILTER_CAPABILITY = "tasks.filter";
|
|
9
12
|
var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
10
13
|
var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
14
|
+
var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
11
15
|
var SERVER_CAPABILITIES = [
|
|
12
16
|
ASYNC_IMPORT_CAPABILITY,
|
|
13
17
|
TASK_CANCEL_CAPABILITY,
|
|
14
18
|
TASK_FILTER_CAPABILITY,
|
|
15
19
|
REBUILD_ANALYSIS_CAPABILITY,
|
|
16
|
-
INDEX_LIFECYCLE_CAPABILITY
|
|
20
|
+
INDEX_LIFECYCLE_CAPABILITY,
|
|
21
|
+
DOCUMENT_EXPORT_CAPABILITY
|
|
17
22
|
];
|
|
18
23
|
|
|
19
24
|
// src/server/concurrency-gate.ts
|
|
@@ -113,6 +118,7 @@ var STATUS_BY_CODE = {
|
|
|
113
118
|
SEARCH_INVALID_MODE: 400,
|
|
114
119
|
SEARCH_INVALID_CURSOR: 400,
|
|
115
120
|
SEARCH_RESULT_WINDOW_EXCEEDED: 400,
|
|
121
|
+
SEARCH_WORK_CAP_EXCEEDED: 400,
|
|
116
122
|
LANGUAGE_NOT_SUPPORTED: 400,
|
|
117
123
|
CONFIG_INVALID: 400,
|
|
118
124
|
EMBEDDING_DIMENSION_MISMATCH: 400,
|
|
@@ -170,7 +176,6 @@ var STATUS_BY_CODE = {
|
|
|
170
176
|
SNAPSHOT_SYNC_HEADER_INVALID: 400,
|
|
171
177
|
SNAPSHOT_SYNC_FRAME_INVALID: 400,
|
|
172
178
|
SNAPSHOT_SYNC_INDEX_NOT_FOUND: 404,
|
|
173
|
-
SNAPSHOT_SYNC_TOO_LARGE: 413,
|
|
174
179
|
SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: 413,
|
|
175
180
|
SNAPSHOT_SYNC_ABORTED: 409,
|
|
176
181
|
SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: 503,
|
|
@@ -225,6 +230,11 @@ function encodeJson(value) {
|
|
|
225
230
|
var BYTES_PER_MB = 1024 * 1024;
|
|
226
231
|
var DEFAULT_MAX_BODY_BYTES = 16 * BYTES_PER_MB;
|
|
227
232
|
var DEFAULT_MAX_IMPORT_BYTES = 100 * BYTES_PER_MB;
|
|
233
|
+
var DEFAULT_MAX_RESTORE_BYTES = 64 * 1024 * BYTES_PER_MB;
|
|
234
|
+
var SPOOL_PAUSE_ABOVE_BYTES = 8 * BYTES_PER_MB;
|
|
235
|
+
var SNAPSHOT_ARCHIVE_CONTENT_TYPE = "application/x-tar";
|
|
236
|
+
var DOCUMENT_EXPORT_CONTENT_TYPE = "application/x-ndjson";
|
|
237
|
+
var RELAY_STREAM_CHUNKS_IN_FLIGHT = 4;
|
|
228
238
|
var DEFAULT_MAX_LINE_BYTES = 4 * BYTES_PER_MB;
|
|
229
239
|
var DEFAULT_IMPORT_BATCH_SIZE = 1e3;
|
|
230
240
|
var IMPORT_FLUSHES_IN_FLIGHT = 4;
|
|
@@ -299,8 +309,56 @@ function sendError(res, status, code, message, details) {
|
|
|
299
309
|
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
300
310
|
});
|
|
301
311
|
}
|
|
302
|
-
function
|
|
303
|
-
|
|
312
|
+
async function sendStream(res, abort, source, headers) {
|
|
313
|
+
const reader = source.getReader();
|
|
314
|
+
let wake = null;
|
|
315
|
+
abort.onAbort(() => wake?.());
|
|
316
|
+
const waitUntilWritable = () => new Promise((resolve) => {
|
|
317
|
+
wake = () => {
|
|
318
|
+
wake = null;
|
|
319
|
+
resolve();
|
|
320
|
+
};
|
|
321
|
+
res.onWritable(() => {
|
|
322
|
+
wake?.();
|
|
323
|
+
return true;
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
let started = false;
|
|
327
|
+
const begin = () => {
|
|
328
|
+
if (started) return;
|
|
329
|
+
started = true;
|
|
330
|
+
res.writeStatus(statusLine(200));
|
|
331
|
+
for (const [key, value] of headers) res.writeHeader(key, value);
|
|
332
|
+
};
|
|
333
|
+
try {
|
|
334
|
+
for (; ; ) {
|
|
335
|
+
const step = await reader.read();
|
|
336
|
+
if (abort.aborted) {
|
|
337
|
+
await reader.cancel().catch(() => void 0);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
if (step.done) {
|
|
341
|
+
res.cork(() => {
|
|
342
|
+
begin();
|
|
343
|
+
res.end();
|
|
344
|
+
});
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
let writable = true;
|
|
348
|
+
res.cork(() => {
|
|
349
|
+
begin();
|
|
350
|
+
writable = res.write(step.value);
|
|
351
|
+
});
|
|
352
|
+
if (!writable && !abort.aborted) await waitUntilWritable();
|
|
353
|
+
}
|
|
354
|
+
} catch (err) {
|
|
355
|
+
await reader.cancel().catch(() => void 0);
|
|
356
|
+
if (abort.aborted) return;
|
|
357
|
+
if (!started) throw err;
|
|
358
|
+
res.cork(() => {
|
|
359
|
+
res.close();
|
|
360
|
+
});
|
|
361
|
+
}
|
|
304
362
|
}
|
|
305
363
|
function sendRelayed(res, abort, status, headers, body) {
|
|
306
364
|
if (abort.aborted) return;
|
|
@@ -495,7 +553,14 @@ function validateRebalance(body) {
|
|
|
495
553
|
}
|
|
496
554
|
return null;
|
|
497
555
|
}
|
|
498
|
-
var TASK_TYPES = [
|
|
556
|
+
var TASK_TYPES = [
|
|
557
|
+
"optimizeVectors",
|
|
558
|
+
"rebalance",
|
|
559
|
+
"restore",
|
|
560
|
+
"import",
|
|
561
|
+
"importExport",
|
|
562
|
+
"rebuildAnalysis"
|
|
563
|
+
];
|
|
499
564
|
var TASK_STATUSES = ["running", "succeeded", "failed", "cancelled"];
|
|
500
565
|
function parseCountParam(raw, field, minimum) {
|
|
501
566
|
if (raw === null) return minimum;
|
|
@@ -630,6 +695,87 @@ function createVersionHandler(deps) {
|
|
|
630
695
|
}
|
|
631
696
|
return { report };
|
|
632
697
|
}
|
|
698
|
+
var NotAdmittedError = class extends Error {
|
|
699
|
+
};
|
|
700
|
+
async function removeDirectory(directory) {
|
|
701
|
+
if (directory !== null) await rm(directory, { recursive: true, force: true });
|
|
702
|
+
}
|
|
703
|
+
function spoolBody(res, maxBytes, abort, admitted) {
|
|
704
|
+
return new Promise((resolve, reject) => {
|
|
705
|
+
const state = {
|
|
706
|
+
directory: null,
|
|
707
|
+
handle: null,
|
|
708
|
+
chain: Promise.resolve(),
|
|
709
|
+
queued: 0,
|
|
710
|
+
total: 0,
|
|
711
|
+
paused: true,
|
|
712
|
+
done: false
|
|
713
|
+
};
|
|
714
|
+
const fail = (error, answer) => {
|
|
715
|
+
if (state.done) return;
|
|
716
|
+
state.done = true;
|
|
717
|
+
if (answer && !abort.aborted) {
|
|
718
|
+
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "The server could not store the request body");
|
|
719
|
+
}
|
|
720
|
+
void state.chain.catch(() => void 0).then(async () => {
|
|
721
|
+
await state.handle?.close().catch(() => void 0);
|
|
722
|
+
await removeDirectory(state.directory).catch(() => void 0);
|
|
723
|
+
}).finally(() => reject(error));
|
|
724
|
+
};
|
|
725
|
+
const failFromChain = (error) => {
|
|
726
|
+
if (error instanceof NotAdmittedError) fail(error, false);
|
|
727
|
+
else fail(error instanceof Error ? error : new Error(String(error)), true);
|
|
728
|
+
};
|
|
729
|
+
res.pause();
|
|
730
|
+
state.chain = (async () => {
|
|
731
|
+
if (!await admitted) throw new NotAdmittedError("the request was not admitted");
|
|
732
|
+
state.directory = await mkdtemp(path.join(tmpdir(), "narsil-spool-"));
|
|
733
|
+
state.handle = await open(path.join(state.directory, "body"), "w");
|
|
734
|
+
if (!abort.aborted && !state.done && state.queued <= SPOOL_PAUSE_ABOVE_BYTES) {
|
|
735
|
+
state.paused = false;
|
|
736
|
+
res.resume();
|
|
737
|
+
}
|
|
738
|
+
})();
|
|
739
|
+
state.chain.catch(failFromChain);
|
|
740
|
+
abort.onAbort(() => fail(new Error("aborted"), false));
|
|
741
|
+
res.onData((chunk, isLast) => {
|
|
742
|
+
if (state.done || abort.aborted) return;
|
|
743
|
+
state.total += chunk.byteLength;
|
|
744
|
+
if (maxBytes > 0 && state.total > maxBytes) {
|
|
745
|
+
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
746
|
+
fail(new Error("payload too large"), false);
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
const copy = new Uint8Array(chunk.slice(0));
|
|
750
|
+
state.queued += copy.byteLength;
|
|
751
|
+
if (!state.paused && state.queued > SPOOL_PAUSE_ABOVE_BYTES) {
|
|
752
|
+
state.paused = true;
|
|
753
|
+
res.pause();
|
|
754
|
+
}
|
|
755
|
+
state.chain = state.chain.then(async () => {
|
|
756
|
+
if (state.handle === null) throw new Error("the spool file is not open");
|
|
757
|
+
await state.handle.write(copy);
|
|
758
|
+
state.queued -= copy.byteLength;
|
|
759
|
+
if (state.paused && state.queued <= SPOOL_PAUSE_ABOVE_BYTES && !abort.aborted && !state.done) {
|
|
760
|
+
state.paused = false;
|
|
761
|
+
res.resume();
|
|
762
|
+
}
|
|
763
|
+
});
|
|
764
|
+
state.chain.catch(failFromChain);
|
|
765
|
+
if (!isLast) return;
|
|
766
|
+
state.chain.then(async () => {
|
|
767
|
+
if (state.done) return;
|
|
768
|
+
state.done = true;
|
|
769
|
+
await state.handle?.close();
|
|
770
|
+
resolve({
|
|
771
|
+
path: path.join(state.directory ?? "", "body"),
|
|
772
|
+
bytes: state.total,
|
|
773
|
+
remove: () => removeDirectory(state.directory)
|
|
774
|
+
});
|
|
775
|
+
}, failFromChain);
|
|
776
|
+
});
|
|
777
|
+
});
|
|
778
|
+
}
|
|
633
779
|
|
|
634
780
|
// src/server/request.ts
|
|
635
781
|
function initAbortHandler(res) {
|
|
@@ -731,6 +877,19 @@ function sinkWritingAfterStatus(res, headers) {
|
|
|
731
877
|
res.end(body);
|
|
732
878
|
return sink;
|
|
733
879
|
},
|
|
880
|
+
write(chunk) {
|
|
881
|
+
return res.write(chunk);
|
|
882
|
+
},
|
|
883
|
+
pause() {
|
|
884
|
+
res.pause();
|
|
885
|
+
},
|
|
886
|
+
resume() {
|
|
887
|
+
res.resume();
|
|
888
|
+
},
|
|
889
|
+
close() {
|
|
890
|
+
res.close();
|
|
891
|
+
return sink;
|
|
892
|
+
},
|
|
734
893
|
endWithoutBody() {
|
|
735
894
|
res.endWithoutBody();
|
|
736
895
|
return sink;
|
|
@@ -829,18 +988,33 @@ function createRouteRunner(deps) {
|
|
|
829
988
|
}
|
|
830
989
|
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
|
|
831
990
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
832
|
-
|
|
833
|
-
|
|
991
|
+
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
992
|
+
res,
|
|
993
|
+
opts.maxBytes,
|
|
994
|
+
abort,
|
|
995
|
+
hookPromise.then((authorization) => authorization.allowed)
|
|
996
|
+
).catch(() => null) : Promise.resolve(null);
|
|
997
|
+
Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
998
|
+
const discardSpool = () => {
|
|
999
|
+
void spooledBody?.remove().catch(() => void 0);
|
|
1000
|
+
};
|
|
1001
|
+
if (abort.aborted) {
|
|
1002
|
+
discardSpool();
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
834
1005
|
if (!authorization.allowed) {
|
|
1006
|
+
discardSpool();
|
|
835
1007
|
sendDenial(res, authorization);
|
|
836
1008
|
return;
|
|
837
1009
|
}
|
|
1010
|
+
if (opts.spoolsBody === true && spooledBody === null) return;
|
|
838
1011
|
if (gated && !gate.tryAcquire()) {
|
|
1012
|
+
discardSpool();
|
|
839
1013
|
sendError(res, 429, ServerErrorCodes.TOO_MANY_REQUESTS, "The server is at capacity; retry shortly");
|
|
840
1014
|
return;
|
|
841
1015
|
}
|
|
842
1016
|
try {
|
|
843
|
-
await handler({ res, params, query, contentType, rawBody, abort, hookContext });
|
|
1017
|
+
await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
|
|
844
1018
|
} catch (err) {
|
|
845
1019
|
if (!abort.aborted) {
|
|
846
1020
|
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
@@ -915,7 +1089,14 @@ function serverRoutes(limits) {
|
|
|
915
1089
|
method: "post",
|
|
916
1090
|
path: "/indexes/:name/restore",
|
|
917
1091
|
handler: "restore",
|
|
918
|
-
options: { paramCount: 1,
|
|
1092
|
+
options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
|
|
1093
|
+
},
|
|
1094
|
+
{ method: "get", path: "/indexes/:name/export", handler: "exportDocuments", options: named },
|
|
1095
|
+
{
|
|
1096
|
+
method: "post",
|
|
1097
|
+
path: "/indexes/:name/import",
|
|
1098
|
+
handler: "importExport",
|
|
1099
|
+
options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
|
|
919
1100
|
}
|
|
920
1101
|
];
|
|
921
1102
|
}
|
|
@@ -974,4 +1155,4 @@ function childAppOf(app) {
|
|
|
974
1155
|
return candidate;
|
|
975
1156
|
}
|
|
976
1157
|
|
|
977
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson,
|
|
1158
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
2
2
|
import { existsSync } from 'fs';
|
|
3
3
|
import { createRequire } from 'module';
|
|
4
4
|
import { join, dirname } from 'path';
|
|
@@ -6,6 +6,8 @@ import { join, dirname } from 'path';
|
|
|
6
6
|
// src/vector/hnsw/constants.ts
|
|
7
7
|
var MAX_LAYER_CAP = 32;
|
|
8
8
|
var MAX_M = 512;
|
|
9
|
+
var MIN_M = 2;
|
|
10
|
+
var EARLIER_RELEASE_MIN_M = 1;
|
|
9
11
|
var INITIAL_UPPER_SLOTS = 64;
|
|
10
12
|
var INITIAL_LIST_CAPACITY = 64;
|
|
11
13
|
var PLACEMENT_ROOM_REQUESTS = 8;
|
|
@@ -136,4 +138,4 @@ function loadNativeCore() {
|
|
|
136
138
|
return loaded;
|
|
137
139
|
}
|
|
138
140
|
|
|
139
|
-
export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, PLACEMENT_ROOM_REQUESTS, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback, requireSearchBackendSettings, stopUsingTheNativeSearchCore };
|
|
141
|
+
export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, EARLIER_RELEASE_MIN_M, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, MIN_M, PLACEMENT_ROOM_REQUESTS, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback, requireSearchBackendSettings, stopUsingTheNativeSearchCore };
|