@delali/narsil 0.2.4-next.48 → 0.2.4-next.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/indexeddb.mjs +15 -10
- package/dist/{options-BC68VPoE.d.ts → capabilities-JrVQOwi4.d.ts} +55 -47
- package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
- package/dist/chunk-AV4IJKD2.mjs +35 -0
- package/dist/{chunk-EV45L35L.mjs → chunk-BEIK3EOZ.mjs} +845 -2407
- package/dist/chunk-CBLS46IE.mjs +10 -0
- package/dist/chunk-CMVG2UEF.mjs +2038 -0
- package/dist/chunk-CV5H7XL7.mjs +876 -0
- package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
- package/dist/{chunk-XWICBGYT.mjs → chunk-MHE7YWAU.mjs} +3 -5
- package/dist/{chunk-UZTT2BKQ.mjs → chunk-ND34I3SV.mjs} +1202 -1881
- package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
- package/dist/chunk-ORBLD3EL.mjs +6 -0
- package/dist/chunk-PIWKZPP7.mjs +64 -0
- package/dist/{chunk-4COAUBLJ.mjs → chunk-PMBHH44R.mjs} +124 -8
- package/dist/chunk-RP23PWMV.mjs +11 -0
- package/dist/chunk-UNEEFTEU.mjs +7 -0
- package/dist/{chunk-WTONLDU3.mjs → chunk-WYJXSLER.mjs} +9 -19
- package/dist/{chunk-DFZDV5FV.mjs → chunk-X6YZO4MQ.mjs} +19 -15
- package/dist/chunk-XE6XR2OE.mjs +23 -0
- package/dist/chunk-YEO6G2NG.mjs +6 -0
- package/dist/chunk-ZHUQKZBE.mjs +1292 -0
- package/dist/{client-C5BkUqB8.d.ts → client-CBOVHdjp.d.ts} +31 -7
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +250 -232
- package/dist/{config-B-ZI-ft5.d.ts → config-Czh2YNDi.d.ts} +2 -2
- package/dist/{config-BZRExglL.d.ts → config-OON7IJ_B.d.ts} +50 -13
- package/dist/direct-executor-OREXU72C.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -3
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution/transport/grpc.mjs +7 -9
- package/dist/distribution/transport/in-memory.mjs +2 -1
- package/dist/distribution/transport/tcp.mjs +3 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +142 -101
- package/dist/embeddings/openai.mjs +5 -3
- package/dist/{engine-Cjr1uBLa.d.ts → engine-9PtFY50q.d.ts} +125 -69
- package/dist/index.browser.mjs +4689 -3330
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +14 -8
- package/dist/invalidation/broadcast-channel.mjs +2 -1
- package/dist/invalidation/filesystem.mjs +3 -4
- package/dist/{search-DDHjfXhJ.d.ts → memory-Bzd5xcEW.d.ts} +89 -63
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -8
- package/dist/react.d.ts +4 -4
- package/dist/react.mjs +7 -5
- package/dist/{schema-B7juWg8Q.d.ts → schema-BqRx9t4E.d.ts} +22 -3
- package/dist/server.d.ts +47 -42
- package/dist/server.mjs +338 -821
- package/dist/thread-PZLEO6JL.mjs +310 -0
- package/dist/{types-DsSSjY4d.d.ts → types-DXPxaXPO.d.ts} +1 -1
- package/dist/vector/hnsw-build-worker.mjs +3 -4
- package/dist/vector/search-worker.mjs +4 -123
- package/dist/workers/entry.mjs +47 -3
- package/package.json +5 -1
- package/dist/direct-executor-EOTFGXTO.mjs +0 -8
package/dist/server.mjs
CHANGED
|
@@ -1,475 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsWriter, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, parseTaskListQuery, parseJsonOptional, validateRebalance, sendBinary, badRequest, SERVER_CAPABILITIES } from './chunk-CV5H7XL7.mjs';
|
|
2
|
+
export { ASYNC_IMPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-CV5H7XL7.mjs';
|
|
3
|
+
import { validateIndexName } from './chunk-MHE7YWAU.mjs';
|
|
4
|
+
import { engineCoreOf, resolveVectorText } from './chunk-PIWKZPP7.mjs';
|
|
5
|
+
import './chunk-XE6XR2OE.mjs';
|
|
6
|
+
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
7
|
+
import './chunk-CBLS46IE.mjs';
|
|
8
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
|
|
3
9
|
export { ServerErrorCodes } from './chunk-P4ELXVJQ.mjs';
|
|
4
10
|
import { randomUUID } from 'crypto';
|
|
5
|
-
|
|
6
|
-
// src/server/capabilities.ts
|
|
7
|
-
var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
|
|
8
|
-
var TASK_CANCEL_CAPABILITY = "tasks.cancel";
|
|
9
|
-
var TASK_FILTER_CAPABILITY = "tasks.filter";
|
|
10
|
-
var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
11
|
-
var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
12
|
-
var SERVER_CAPABILITIES = [
|
|
13
|
-
ASYNC_IMPORT_CAPABILITY,
|
|
14
|
-
TASK_CANCEL_CAPABILITY,
|
|
15
|
-
TASK_FILTER_CAPABILITY,
|
|
16
|
-
REBUILD_ANALYSIS_CAPABILITY,
|
|
17
|
-
INDEX_LIFECYCLE_CAPABILITY
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
// src/server/cors.ts
|
|
21
|
-
function resolveCors(cors) {
|
|
22
|
-
if (!cors) return null;
|
|
23
|
-
if (cors === true) {
|
|
24
|
-
return { origin: "*", methods: "GET, POST, PUT, PATCH, DELETE, OPTIONS", headers: "Content-Type, Authorization" };
|
|
25
|
-
}
|
|
26
|
-
return {
|
|
27
|
-
origin: cors.origin ?? "*",
|
|
28
|
-
methods: cors.methods?.join(", ") ?? "GET, POST, PUT, PATCH, DELETE, OPTIONS",
|
|
29
|
-
headers: cors.headers?.join(", ") ?? "Content-Type, Authorization"
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
function matchOrigin(cors, requestOrigin) {
|
|
33
|
-
if (cors.origin === "*") return "*";
|
|
34
|
-
if (typeof cors.origin === "string") return cors.origin;
|
|
35
|
-
return cors.origin.includes(requestOrigin) ? requestOrigin : null;
|
|
36
|
-
}
|
|
37
|
-
function writeCorsOrigin(res, cors, requestOrigin) {
|
|
38
|
-
const allowed = matchOrigin(cors, requestOrigin);
|
|
39
|
-
if (!allowed) return;
|
|
40
|
-
res.writeHeader("Access-Control-Allow-Origin", allowed);
|
|
41
|
-
if (allowed !== "*") res.writeHeader("Vary", "Origin");
|
|
42
|
-
}
|
|
43
|
-
function corsWriter(cors) {
|
|
44
|
-
return (res, req) => writeCorsOrigin(res, cors, req.getHeader("origin"));
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// src/server/errors.ts
|
|
48
|
-
var STATUS_BY_CODE = {
|
|
49
|
-
INVALID_REQUEST: 400,
|
|
50
|
-
INVALID_JSON: 400,
|
|
51
|
-
EMPTY_BODY: 400,
|
|
52
|
-
PAYLOAD_TOO_LARGE: 413,
|
|
53
|
-
NOT_FOUND: 404,
|
|
54
|
-
TASK_NOT_FOUND: 404,
|
|
55
|
-
TASK_NOT_CANCELLABLE: 409,
|
|
56
|
-
TASK_OWNED_BY_ANOTHER_INSTANCE: 409,
|
|
57
|
-
TASK_INTERRUPTED: 503,
|
|
58
|
-
TOO_MANY_REQUESTS: 429,
|
|
59
|
-
HOOK_ERROR: 500,
|
|
60
|
-
INTERNAL_ERROR: 500,
|
|
61
|
-
ENVELOPE_INVALID_MAGIC: 400,
|
|
62
|
-
ENVELOPE_VERSION_MISMATCH: 400,
|
|
63
|
-
SCHEMA_INVALID_TYPE: 400,
|
|
64
|
-
SCHEMA_DEPTH_EXCEEDED: 400,
|
|
65
|
-
SCHEMA_INVALID_VECTOR_DIMENSION: 400,
|
|
66
|
-
SCHEMA_INVALID_GEOPOINT: 400,
|
|
67
|
-
DOC_VALIDATION_FAILED: 400,
|
|
68
|
-
DOC_MISSING_REQUIRED_FIELD: 400,
|
|
69
|
-
SEARCH_INVALID_FIELD: 400,
|
|
70
|
-
VECTOR_DIMENSION_MISMATCH: 400,
|
|
71
|
-
SEARCH_INVALID_FILTER: 400,
|
|
72
|
-
SEARCH_INVALID_MODE: 400,
|
|
73
|
-
SEARCH_INVALID_CURSOR: 400,
|
|
74
|
-
SEARCH_RESULT_WINDOW_EXCEEDED: 400,
|
|
75
|
-
LANGUAGE_NOT_SUPPORTED: 400,
|
|
76
|
-
CONFIG_INVALID: 400,
|
|
77
|
-
EMBEDDING_DIMENSION_MISMATCH: 400,
|
|
78
|
-
EMBEDDING_NO_SOURCE: 400,
|
|
79
|
-
EMBEDDING_CONFIG_INVALID: 400,
|
|
80
|
-
INDEX_NOT_FOUND: 404,
|
|
81
|
-
DOC_NOT_FOUND: 404,
|
|
82
|
-
INDEX_ALREADY_EXISTS: 409,
|
|
83
|
-
INDEX_ORPHANED: 409,
|
|
84
|
-
INDEX_REOPEN_CAPACITY_EXHAUSTED: 503,
|
|
85
|
-
DOC_ALREADY_EXISTS: 409,
|
|
86
|
-
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
87
|
-
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
88
|
-
WORKER_BUSY: 429,
|
|
89
|
-
PARTITION_CORRUPTED: 503,
|
|
90
|
-
WORKER_CRASHED: 503,
|
|
91
|
-
WORKER_TIMEOUT: 503,
|
|
92
|
-
PERSISTENCE_SAVE_FAILED: 503,
|
|
93
|
-
PERSISTENCE_LOAD_FAILED: 503,
|
|
94
|
-
PERSISTENCE_DELETE_FAILED: 503,
|
|
95
|
-
PERSISTENCE_CRC_MISMATCH: 503,
|
|
96
|
-
PERSISTENCE_WAL_CORRUPT: 503,
|
|
97
|
-
PERSISTENCE_FSYNC_FAILED: 503,
|
|
98
|
-
EMBEDDING_FAILED: 503,
|
|
99
|
-
QUERY_ROUTING_FAILED: 503,
|
|
100
|
-
QUERY_PARTIAL_FAILURE: 503,
|
|
101
|
-
QUERY_NODE_TIMEOUT: 503,
|
|
102
|
-
QUERY_NO_ACTIVE_REPLICA: 503,
|
|
103
|
-
CLUSTER_OPERATION_UNSUPPORTED: 501,
|
|
104
|
-
REPLICATION_ENTRY_INVALID: 503,
|
|
105
|
-
REPLICATION_INSYNC_REMOVAL_FAILED: 503,
|
|
106
|
-
REPLICATION_ROLLBACK_FAILED: 503,
|
|
107
|
-
REPLICATION_LOG_FULL: 503,
|
|
108
|
-
REPLICATION_ENTRY_CORRUPT: 503,
|
|
109
|
-
REPLICATION_SNAPSHOT_CORRUPT: 503,
|
|
110
|
-
REPLICATION_TERM_MISMATCH: 503,
|
|
111
|
-
REPLICATION_SYNC_FAILED: 503,
|
|
112
|
-
PARTITION_NOT_PRIMARY: 503,
|
|
113
|
-
PARTITION_UNASSIGNED: 503,
|
|
114
|
-
INSUFFICIENT_REPLICAS: 503,
|
|
115
|
-
TRANSPORT_DEPENDENCY_MISSING: 500,
|
|
116
|
-
ALLOCATION_INVALID_CONFIG: 400,
|
|
117
|
-
ALLOCATION_NO_DATA_NODES: 503,
|
|
118
|
-
ALLOCATION_FAILED: 503,
|
|
119
|
-
NODE_ALREADY_JOINED: 409,
|
|
120
|
-
NODE_NOT_JOINED: 503,
|
|
121
|
-
NODE_NOT_READY: 503,
|
|
122
|
-
NODE_NOT_CONTROLLER: 503,
|
|
123
|
-
NODE_BOOTSTRAP_FAILED: 503,
|
|
124
|
-
COORDINATOR_DEPENDENCY_MISSING: 500,
|
|
125
|
-
SNAPSHOT_SYNC_UNAUTHORIZED: 403,
|
|
126
|
-
SNAPSHOT_SYNC_REQUEST_INVALID: 400,
|
|
127
|
-
SNAPSHOT_SYNC_HEADER_INVALID: 400,
|
|
128
|
-
SNAPSHOT_SYNC_FRAME_INVALID: 400,
|
|
129
|
-
SNAPSHOT_SYNC_INDEX_NOT_FOUND: 404,
|
|
130
|
-
SNAPSHOT_SYNC_TOO_LARGE: 413,
|
|
131
|
-
SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: 413,
|
|
132
|
-
SNAPSHOT_SYNC_ABORTED: 409,
|
|
133
|
-
SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: 503,
|
|
134
|
-
SNAPSHOT_SYNC_SNAPSHOT_FAILED: 503,
|
|
135
|
-
SNAPSHOT_SYNC_DECODE_FAILED: 503,
|
|
136
|
-
SNAPSHOT_SYNC_HEADER_MISMATCH: 503,
|
|
137
|
-
SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER: 503,
|
|
138
|
-
SNAPSHOT_SYNC_CHUNK_OVERFLOW: 503,
|
|
139
|
-
SNAPSHOT_SYNC_CHUNK_MISSING: 503,
|
|
140
|
-
SNAPSHOT_SYNC_END_MISSING: 503,
|
|
141
|
-
SNAPSHOT_SYNC_CHECKSUM_MISMATCH: 503,
|
|
142
|
-
SNAPSHOT_SYNC_PRIMARY_ERROR: 503,
|
|
143
|
-
SNAPSHOT_SYNC_NO_TARGETS: 503,
|
|
144
|
-
SNAPSHOT_SYNC_TRANSPORT_FAILED: 503,
|
|
145
|
-
SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE: 503,
|
|
146
|
-
SNAPSHOT_SYNC_RESTORE_FAILED: 503,
|
|
147
|
-
SNAPSHOT_SYNC_RESTORE_CLEANUP_FAILED: 503,
|
|
148
|
-
SNAPSHOT_SYNC_TIMEOUT: 503,
|
|
149
|
-
SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE: 503,
|
|
150
|
-
SNAPSHOT_SYNC_NOT_ASSIGNED: 503,
|
|
151
|
-
CONTROLLER_METADATA_INVALID: 500
|
|
152
|
-
};
|
|
153
|
-
function httpStatusForNarsilError(code) {
|
|
154
|
-
return STATUS_BY_CODE[code] ?? 500;
|
|
155
|
-
}
|
|
156
|
-
function serializeNarsilError(err) {
|
|
157
|
-
const hasDetails = err.details && Object.keys(err.details).length > 0;
|
|
158
|
-
return hasDetails ? { code: err.code, message: err.message, details: err.details } : { code: err.code, message: err.message };
|
|
159
|
-
}
|
|
160
|
-
function toHttpError(err) {
|
|
161
|
-
if (err instanceof NarsilError) {
|
|
162
|
-
return { status: httpStatusForNarsilError(err.code), body: serializeNarsilError(err) };
|
|
163
|
-
}
|
|
164
|
-
return {
|
|
165
|
-
status: 500,
|
|
166
|
-
body: { code: ServerErrorCodes.INTERNAL_ERROR, message: "An unexpected error occurred" }
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
// src/json-encoding.ts
|
|
171
|
-
function toPlainNumbers(key, value) {
|
|
172
|
-
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
173
|
-
return Array.from(value);
|
|
174
|
-
}
|
|
175
|
-
return value;
|
|
176
|
-
}
|
|
177
|
-
function encodeJson(value) {
|
|
178
|
-
return JSON.stringify(value, toPlainNumbers);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// src/server/response.ts
|
|
182
|
-
var STATUS_TEXT = {
|
|
183
|
-
200: "200 OK",
|
|
184
|
-
201: "201 Created",
|
|
185
|
-
202: "202 Accepted",
|
|
186
|
-
204: "204 No Content",
|
|
187
|
-
400: "400 Bad Request",
|
|
188
|
-
401: "401 Unauthorized",
|
|
189
|
-
403: "403 Forbidden",
|
|
190
|
-
404: "404 Not Found",
|
|
191
|
-
405: "405 Method Not Allowed",
|
|
192
|
-
409: "409 Conflict",
|
|
193
|
-
413: "413 Payload Too Large",
|
|
194
|
-
415: "415 Unsupported Media Type",
|
|
195
|
-
429: "429 Too Many Requests",
|
|
196
|
-
500: "500 Internal Server Error",
|
|
197
|
-
503: "503 Service Unavailable"
|
|
198
|
-
};
|
|
199
|
-
var STREAM_THRESHOLD = 64 * 1024;
|
|
200
|
-
function statusLine(status) {
|
|
201
|
-
return STATUS_TEXT[status] ?? `${status}`;
|
|
202
|
-
}
|
|
203
|
-
function streamBody(res, abort, body, status, contentType) {
|
|
204
|
-
if (abort.aborted) return;
|
|
205
|
-
const totalSize = body.byteLength;
|
|
206
|
-
res.cork(() => {
|
|
207
|
-
res.writeStatus(statusLine(status)).writeHeader("Content-Type", contentType);
|
|
208
|
-
const startOffset = res.getWriteOffset();
|
|
209
|
-
const [, done] = res.tryEnd(body, totalSize);
|
|
210
|
-
if (done) return;
|
|
211
|
-
res.onWritable((offset) => {
|
|
212
|
-
if (abort.aborted) return true;
|
|
213
|
-
const [retryOk] = res.tryEnd(body.subarray(offset - startOffset), totalSize);
|
|
214
|
-
return retryOk;
|
|
215
|
-
});
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
function sendJson(res, data, status = 200, abort) {
|
|
219
|
-
const payload = encodeJson(data);
|
|
220
|
-
if (abort && payload !== void 0 && Buffer.byteLength(payload) >= STREAM_THRESHOLD) {
|
|
221
|
-
streamBody(res, abort, Buffer.from(payload, "utf8"), status, "application/json");
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
res.cork(() => {
|
|
225
|
-
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
226
|
-
});
|
|
227
|
-
}
|
|
228
|
-
function sendError(res, status, code, message, details) {
|
|
229
|
-
const body = { error: details ? { code, message, details } : { code, message } };
|
|
230
|
-
const payload = encodeJson(body);
|
|
231
|
-
res.cork(() => {
|
|
232
|
-
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
|
-
function sendBinary(res, data, abort, status = 200) {
|
|
236
|
-
streamBody(res, abort, data, status, "application/octet-stream");
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
// src/server/handler-utils.ts
|
|
240
|
-
function parseJson(ctx) {
|
|
241
|
-
const raw = ctx.rawBody;
|
|
242
|
-
if (!raw || raw.length === 0) {
|
|
243
|
-
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
244
|
-
return void 0;
|
|
245
|
-
}
|
|
246
|
-
try {
|
|
247
|
-
return JSON.parse(raw.toString("utf8"));
|
|
248
|
-
} catch {
|
|
249
|
-
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
250
|
-
return void 0;
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
function parseJsonOptional(ctx) {
|
|
254
|
-
const raw = ctx.rawBody;
|
|
255
|
-
if (!raw || raw.length === 0) return {};
|
|
256
|
-
try {
|
|
257
|
-
return JSON.parse(raw.toString("utf8"));
|
|
258
|
-
} catch {
|
|
259
|
-
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
260
|
-
return void 0;
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
function respondError(ctx, err) {
|
|
264
|
-
if (ctx.abort.aborted) return;
|
|
265
|
-
const { status, body } = toHttpError(err);
|
|
266
|
-
sendError(ctx.res, status, body.code, body.message, body.details);
|
|
267
|
-
}
|
|
268
|
-
function respondJson(ctx, data, status = 200) {
|
|
269
|
-
if (ctx.abort.aborted) return;
|
|
270
|
-
sendJson(ctx.res, data, status, ctx.abort);
|
|
271
|
-
}
|
|
272
|
-
function badRequest(res, message, details) {
|
|
273
|
-
sendError(res, 400, ServerErrorCodes.INVALID_REQUEST, message, details);
|
|
274
|
-
}
|
|
275
|
-
function rejectInvalid(ctx, failure) {
|
|
276
|
-
if (ctx.abort.aborted) return;
|
|
277
|
-
badRequest(ctx.res, failure.message, failure.details);
|
|
278
|
-
}
|
|
279
|
-
function serializeBatchResult(result) {
|
|
280
|
-
return {
|
|
281
|
-
succeeded: result.succeeded,
|
|
282
|
-
failed: result.failed.map((f) => ({
|
|
283
|
-
docId: f.docId,
|
|
284
|
-
error: f.error instanceof NarsilError ? serializeNarsilError(f.error) : { code: "INTERNAL_ERROR", message: String(f.error) }
|
|
285
|
-
}))
|
|
286
|
-
};
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
// src/server/validation.ts
|
|
290
|
-
var MAX_LIST_SORT_FIELDS = 8;
|
|
291
|
-
function isPlainObject(value) {
|
|
292
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
293
|
-
}
|
|
294
|
-
function overWindow(field, value, maxWindow) {
|
|
295
|
-
if (typeof value === "number" && value > maxWindow) {
|
|
296
|
-
return {
|
|
297
|
-
message: `Field "${field}" exceeds the maximum result window of ${maxWindow}`,
|
|
298
|
-
details: { value, limit: maxWindow }
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
return null;
|
|
302
|
-
}
|
|
303
|
-
function validateQuery(params, maxWindow) {
|
|
304
|
-
if (params.group && typeof params.group === "object" && "reduce" in params.group) {
|
|
305
|
-
return {
|
|
306
|
-
message: 'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
const rowBound = overWindow("limit", params.limit, maxWindow) ?? overWindow("offset", params.offset, maxWindow);
|
|
310
|
-
if (rowBound) return rowBound;
|
|
311
|
-
if (params.group && typeof params.group === "object") {
|
|
312
|
-
const grouped = overWindow("group.maxPerGroup", params.group.maxPerGroup, maxWindow) ?? overWindow("group.limit", params.group.limit, maxWindow);
|
|
313
|
-
if (grouped) return grouped;
|
|
314
|
-
}
|
|
315
|
-
if (params.facets && typeof params.facets === "object") {
|
|
316
|
-
for (const [field, facet] of Object.entries(params.facets)) {
|
|
317
|
-
if (facet && typeof facet === "object") {
|
|
318
|
-
const faceted = overWindow(`facets.${field}.limit`, facet.limit, maxWindow);
|
|
319
|
-
if (faceted) return faceted;
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
return null;
|
|
324
|
-
}
|
|
325
|
-
function validateSuggest(params) {
|
|
326
|
-
if (typeof params.prefix !== "string") {
|
|
327
|
-
return { message: 'Field "prefix" is required and must be a string' };
|
|
328
|
-
}
|
|
329
|
-
return null;
|
|
330
|
-
}
|
|
331
|
-
function validateDocumentBody(body) {
|
|
332
|
-
if (!isPlainObject(body.document)) {
|
|
333
|
-
return { message: 'Field "document" is required and must be an object' };
|
|
334
|
-
}
|
|
335
|
-
return null;
|
|
336
|
-
}
|
|
337
|
-
function validateMultiGet(body, maxFetch) {
|
|
338
|
-
if (!Array.isArray(body.docIds)) {
|
|
339
|
-
return { message: 'Field "docIds" is required and must be an array of strings' };
|
|
340
|
-
}
|
|
341
|
-
if (body.docIds.length > maxFetch) {
|
|
342
|
-
return {
|
|
343
|
-
message: `Field "docIds" exceeds the maximum of ${maxFetch} ids per request`,
|
|
344
|
-
details: { count: body.docIds.length, limit: maxFetch }
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
return null;
|
|
348
|
-
}
|
|
349
|
-
function validateList(params, maxFetch) {
|
|
350
|
-
if (params.cursor !== void 0 && typeof params.cursor !== "string") {
|
|
351
|
-
return { message: 'Field "cursor" must be a string' };
|
|
352
|
-
}
|
|
353
|
-
if (params.limit !== void 0 && typeof params.limit !== "number") {
|
|
354
|
-
return { message: 'Field "limit" must be a number' };
|
|
355
|
-
}
|
|
356
|
-
if (typeof params.limit === "number" && params.limit > maxFetch) {
|
|
357
|
-
return {
|
|
358
|
-
message: `Field "limit" exceeds the maximum of ${maxFetch} documents per request`,
|
|
359
|
-
details: { value: params.limit, limit: maxFetch }
|
|
360
|
-
};
|
|
361
|
-
}
|
|
362
|
-
return validateListSort(params.sort);
|
|
363
|
-
}
|
|
364
|
-
function validateListSort(sort) {
|
|
365
|
-
if (sort === void 0) return null;
|
|
366
|
-
if (typeof sort !== "object" || sort === null) {
|
|
367
|
-
return { message: 'Field "sort" must be an object keyed by field name, or a list of fields' };
|
|
368
|
-
}
|
|
369
|
-
const entries = Array.isArray(sort) ? sort.map(
|
|
370
|
-
(entry) => typeof entry === "object" && entry !== null ? [entry.field, entry.direction] : [entry, void 0]
|
|
371
|
-
) : Object.entries(sort);
|
|
372
|
-
if (entries.length > MAX_LIST_SORT_FIELDS) {
|
|
373
|
-
return {
|
|
374
|
-
message: `Field "sort" exceeds the maximum of ${MAX_LIST_SORT_FIELDS} fields`,
|
|
375
|
-
details: { count: entries.length, limit: MAX_LIST_SORT_FIELDS }
|
|
376
|
-
};
|
|
377
|
-
}
|
|
378
|
-
for (const [field, direction] of entries) {
|
|
379
|
-
if (typeof field !== "string" || field.length === 0) {
|
|
380
|
-
return { message: 'Every entry of "sort" must name a field', details: { value: field } };
|
|
381
|
-
}
|
|
382
|
-
if (direction !== "asc" && direction !== "desc") {
|
|
383
|
-
return { message: `Field "sort.${field}" must be "asc" or "desc"`, details: { value: direction } };
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
return null;
|
|
387
|
-
}
|
|
388
|
-
function validateBatch(body) {
|
|
389
|
-
const action = body.action ?? "insert";
|
|
390
|
-
if (action === "insert") {
|
|
391
|
-
if (!Array.isArray(body.documents)) return { message: 'Batch insert requires a "documents" array' };
|
|
392
|
-
return null;
|
|
393
|
-
}
|
|
394
|
-
if (action === "update") {
|
|
395
|
-
if (!Array.isArray(body.updates)) return { message: 'Batch update requires an "updates" array' };
|
|
396
|
-
return null;
|
|
397
|
-
}
|
|
398
|
-
if (action === "delete") {
|
|
399
|
-
if (!Array.isArray(body.docIds)) return { message: 'Batch delete requires a "docIds" array' };
|
|
400
|
-
return null;
|
|
401
|
-
}
|
|
402
|
-
return { message: 'Field "action" must be one of "insert", "update", or "delete"' };
|
|
403
|
-
}
|
|
404
|
-
function validateCreateIndex(body) {
|
|
405
|
-
if (typeof body.name !== "string" || body.name.length === 0) {
|
|
406
|
-
return { message: 'Field "name" is required and must be a non-empty string' };
|
|
407
|
-
}
|
|
408
|
-
if (!isPlainObject(body.config)) {
|
|
409
|
-
return { message: 'Field "config" is required and must be an object' };
|
|
410
|
-
}
|
|
411
|
-
return null;
|
|
412
|
-
}
|
|
413
|
-
function validateRebalance(body) {
|
|
414
|
-
const target = body.targetPartitionCount;
|
|
415
|
-
if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
|
|
416
|
-
return { message: 'Field "targetPartitionCount" is required and must be a positive integer' };
|
|
417
|
-
}
|
|
418
|
-
return null;
|
|
419
|
-
}
|
|
420
|
-
var TASK_TYPES = ["optimizeVectors", "rebalance", "restore", "import", "rebuildAnalysis"];
|
|
421
|
-
var TASK_STATUSES = ["queued", "running", "succeeded", "failed", "cancelled"];
|
|
422
|
-
function parseCountParam(raw, field, minimum) {
|
|
423
|
-
if (raw === null) return minimum;
|
|
424
|
-
const value = Number(raw);
|
|
425
|
-
if (!Number.isInteger(value) || value < minimum) {
|
|
426
|
-
return { message: `Query parameter "${field}" must be an integer of ${minimum} or more`, details: { value: raw } };
|
|
427
|
-
}
|
|
428
|
-
return value;
|
|
429
|
-
}
|
|
430
|
-
function parseEnumParam(raw, field, allowed) {
|
|
431
|
-
if (raw === null) return void 0;
|
|
432
|
-
const requested = raw.split(",").map((entry) => entry.trim());
|
|
433
|
-
const unknown = requested.filter((entry) => !allowed.includes(entry));
|
|
434
|
-
if (requested.length === 0 || unknown.length > 0) {
|
|
435
|
-
return {
|
|
436
|
-
message: `Query parameter "${field}" accepts only ${allowed.join(", ")}`,
|
|
437
|
-
details: { unknown }
|
|
438
|
-
};
|
|
439
|
-
}
|
|
440
|
-
return requested;
|
|
441
|
-
}
|
|
442
|
-
function isFailure(value) {
|
|
443
|
-
return isPlainObject(value) && typeof value.message === "string";
|
|
444
|
-
}
|
|
445
|
-
function parseTaskListQuery(params, maxPageSize) {
|
|
446
|
-
const type = parseEnumParam(params.get("type"), "type", TASK_TYPES);
|
|
447
|
-
if (isFailure(type)) return { failure: type };
|
|
448
|
-
const status = parseEnumParam(params.get("status"), "status", TASK_STATUSES);
|
|
449
|
-
if (isFailure(status)) return { failure: status };
|
|
450
|
-
const from = parseCountParam(params.get("from"), "from", 0);
|
|
451
|
-
if (isFailure(from)) return { failure: from };
|
|
452
|
-
const limit = parseCountParam(params.get("limit"), "limit", 1);
|
|
453
|
-
if (isFailure(limit)) return { failure: limit };
|
|
454
|
-
if (params.get("limit") !== null && limit > maxPageSize) {
|
|
455
|
-
return {
|
|
456
|
-
failure: {
|
|
457
|
-
message: `Query parameter "limit" exceeds the maximum page size of ${maxPageSize}`,
|
|
458
|
-
details: { value: limit, limit: maxPageSize }
|
|
459
|
-
}
|
|
460
|
-
};
|
|
461
|
-
}
|
|
462
|
-
const indexName = params.get("indexName");
|
|
463
|
-
return {
|
|
464
|
-
query: {
|
|
465
|
-
...indexName === null ? {} : { indexName },
|
|
466
|
-
...type === void 0 ? {} : { type },
|
|
467
|
-
...status === void 0 ? {} : { status },
|
|
468
|
-
from,
|
|
469
|
-
...params.get("limit") === null ? {} : { limit }
|
|
470
|
-
}
|
|
471
|
-
};
|
|
472
|
-
}
|
|
11
|
+
import { MessageChannel } from 'worker_threads';
|
|
473
12
|
|
|
474
13
|
// src/server/handlers/admin.ts
|
|
475
14
|
function createAdminHandlers(deps) {
|
|
@@ -727,7 +266,8 @@ function createClusterHandlers(deps) {
|
|
|
727
266
|
|
|
728
267
|
// src/server/handlers/documents.ts
|
|
729
268
|
function createDocumentHandlers(deps) {
|
|
730
|
-
const { engine
|
|
269
|
+
const { engine } = deps;
|
|
270
|
+
const reads = createDocumentReadHandlers(deps);
|
|
731
271
|
async function insert(ctx) {
|
|
732
272
|
const body = parseJson(ctx);
|
|
733
273
|
if (!body) return;
|
|
@@ -743,25 +283,6 @@ function createDocumentHandlers(deps) {
|
|
|
743
283
|
respondError(ctx, err);
|
|
744
284
|
}
|
|
745
285
|
}
|
|
746
|
-
async function get(ctx) {
|
|
747
|
-
try {
|
|
748
|
-
const document = await engine.get(ctx.params[0], ctx.params[1]);
|
|
749
|
-
if (document === void 0) {
|
|
750
|
-
respondError(ctx, indexDocNotFound(ctx.params[1]));
|
|
751
|
-
return;
|
|
752
|
-
}
|
|
753
|
-
respondJson(ctx, { document });
|
|
754
|
-
} catch (err) {
|
|
755
|
-
respondError(ctx, err);
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
async function exists(ctx) {
|
|
759
|
-
try {
|
|
760
|
-
respondJson(ctx, { exists: await engine.has(ctx.params[0], ctx.params[1]) });
|
|
761
|
-
} catch (err) {
|
|
762
|
-
respondError(ctx, err);
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
286
|
async function put(ctx) {
|
|
766
287
|
const body = parseJson(ctx);
|
|
767
288
|
if (!body) return;
|
|
@@ -773,10 +294,10 @@ function createDocumentHandlers(deps) {
|
|
|
773
294
|
const [name, id] = ctx.params;
|
|
774
295
|
try {
|
|
775
296
|
if (await engine.has(name, id)) {
|
|
776
|
-
await engine.update(name, id, body.document);
|
|
297
|
+
await engine.update(name, id, body.document, body.options);
|
|
777
298
|
respondJson(ctx, { id, created: false });
|
|
778
299
|
} else {
|
|
779
|
-
const created = await engine.insert(name, body.document, id);
|
|
300
|
+
const created = await engine.insert(name, body.document, id, body.options);
|
|
780
301
|
respondJson(ctx, { id: created, created: true }, 201);
|
|
781
302
|
}
|
|
782
303
|
} catch (err) {
|
|
@@ -792,7 +313,7 @@ function createDocumentHandlers(deps) {
|
|
|
792
313
|
return;
|
|
793
314
|
}
|
|
794
315
|
try {
|
|
795
|
-
await engine.update(ctx.params[0], ctx.params[1], body.document);
|
|
316
|
+
await engine.update(ctx.params[0], ctx.params[1], body.document, body.options);
|
|
796
317
|
respondJson(ctx, { id: ctx.params[1] });
|
|
797
318
|
} catch (err) {
|
|
798
319
|
respondError(ctx, err);
|
|
@@ -800,46 +321,16 @@ function createDocumentHandlers(deps) {
|
|
|
800
321
|
}
|
|
801
322
|
async function remove(ctx) {
|
|
802
323
|
try {
|
|
803
|
-
await engine.remove(ctx.params[0], ctx.params[1]);
|
|
324
|
+
await engine.remove(ctx.params[0], ctx.params[1], { wait: ctx.query.get("wait") === "true" });
|
|
804
325
|
respondJson(ctx, { id: ctx.params[1], removed: true });
|
|
805
326
|
} catch (err) {
|
|
806
327
|
respondError(ctx, err);
|
|
807
328
|
}
|
|
808
329
|
}
|
|
809
|
-
async function
|
|
810
|
-
try {
|
|
811
|
-
respondJson(ctx, { count: await engine.countDocuments(ctx.params[0]) });
|
|
812
|
-
} catch (err) {
|
|
813
|
-
respondError(ctx, err);
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
async function multiGet(ctx) {
|
|
817
|
-
const body = parseJson(ctx);
|
|
818
|
-
if (!body) return;
|
|
819
|
-
const failure = validateMultiGet(body, limits.maxFetchDocuments);
|
|
820
|
-
if (failure) {
|
|
821
|
-
rejectInvalid(ctx, failure);
|
|
822
|
-
return;
|
|
823
|
-
}
|
|
330
|
+
async function waitForWrites(ctx) {
|
|
824
331
|
try {
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
for (const [id, doc] of found) documents[id] = doc;
|
|
828
|
-
respondJson(ctx, { documents });
|
|
829
|
-
} catch (err) {
|
|
830
|
-
respondError(ctx, err);
|
|
831
|
-
}
|
|
832
|
-
}
|
|
833
|
-
async function list(ctx) {
|
|
834
|
-
const params = parseJson(ctx);
|
|
835
|
-
if (!params) return;
|
|
836
|
-
const failure = validateList(params, limits.maxFetchDocuments);
|
|
837
|
-
if (failure) {
|
|
838
|
-
rejectInvalid(ctx, failure);
|
|
839
|
-
return;
|
|
840
|
-
}
|
|
841
|
-
try {
|
|
842
|
-
respondJson(ctx, await engine.listDocuments(ctx.params[0], params));
|
|
332
|
+
await engine.waitForWrites(ctx.params[0]);
|
|
333
|
+
respondJson(ctx, { ok: true });
|
|
843
334
|
} catch (err) {
|
|
844
335
|
respondError(ctx, err);
|
|
845
336
|
}
|
|
@@ -855,9 +346,12 @@ function createDocumentHandlers(deps) {
|
|
|
855
346
|
const action = body.action ?? "insert";
|
|
856
347
|
try {
|
|
857
348
|
if (action === "update") {
|
|
858
|
-
respondJson(
|
|
349
|
+
respondJson(
|
|
350
|
+
ctx,
|
|
351
|
+
serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [], body.options))
|
|
352
|
+
);
|
|
859
353
|
} else if (action === "delete") {
|
|
860
|
-
respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [])));
|
|
354
|
+
respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [], body.options)));
|
|
861
355
|
} else {
|
|
862
356
|
respondJson(
|
|
863
357
|
ctx,
|
|
@@ -868,10 +362,7 @@ function createDocumentHandlers(deps) {
|
|
|
868
362
|
respondError(ctx, err);
|
|
869
363
|
}
|
|
870
364
|
}
|
|
871
|
-
return {
|
|
872
|
-
}
|
|
873
|
-
function indexDocNotFound(docId) {
|
|
874
|
-
return new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found`, { docId });
|
|
365
|
+
return { ...reads, insert, put, patch, remove, batch, waitForWrites };
|
|
875
366
|
}
|
|
876
367
|
|
|
877
368
|
// src/server/handlers/health.ts
|
|
@@ -936,7 +427,7 @@ function* iterateNdjson(buffer, maxLineBytes) {
|
|
|
936
427
|
function yieldToEventLoop() {
|
|
937
428
|
return new Promise((resolve) => setImmediate(resolve));
|
|
938
429
|
}
|
|
939
|
-
function
|
|
430
|
+
function isPlainObject(value) {
|
|
940
431
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
941
432
|
}
|
|
942
433
|
async function runImport(engine, options) {
|
|
@@ -974,7 +465,7 @@ async function runImport(engine, options) {
|
|
|
974
465
|
recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
|
|
975
466
|
continue;
|
|
976
467
|
}
|
|
977
|
-
if (!
|
|
468
|
+
if (!isPlainObject(parsed)) {
|
|
978
469
|
recordFailure({
|
|
979
470
|
line: line.lineNumber,
|
|
980
471
|
code: ServerErrorCodes.INVALID_REQUEST,
|
|
@@ -1226,189 +717,240 @@ function createVersionHandler(deps) {
|
|
|
1226
717
|
return { report };
|
|
1227
718
|
}
|
|
1228
719
|
|
|
1229
|
-
// src/server/request.ts
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
720
|
+
// src/server/request-threads/captured-response.ts
|
|
721
|
+
var DEFAULT_STATUS = 200;
|
|
722
|
+
var NO_RESPONSE_STATUS = 500;
|
|
723
|
+
function statusOf(line) {
|
|
724
|
+
const parsed = Number.parseInt(line, 10);
|
|
725
|
+
return Number.isFinite(parsed) ? parsed : DEFAULT_STATUS;
|
|
726
|
+
}
|
|
727
|
+
function bodyBytes(body) {
|
|
728
|
+
if (body === void 0) return new Uint8Array(0);
|
|
729
|
+
if (typeof body === "string") return Buffer.from(body, "utf8");
|
|
730
|
+
return body;
|
|
731
|
+
}
|
|
732
|
+
function createCapturingResponse(remoteAddress) {
|
|
733
|
+
let status = DEFAULT_STATUS;
|
|
734
|
+
const headers = [];
|
|
735
|
+
let finished = false;
|
|
736
|
+
let resolveDone = () => void 0;
|
|
737
|
+
const done = new Promise((resolve) => {
|
|
738
|
+
resolveDone = resolve;
|
|
1236
739
|
});
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
740
|
+
function finish(body) {
|
|
741
|
+
if (finished) return;
|
|
742
|
+
finished = true;
|
|
743
|
+
resolveDone({ status, headers, body });
|
|
744
|
+
}
|
|
745
|
+
const sink = {
|
|
746
|
+
cork(callback) {
|
|
747
|
+
callback();
|
|
748
|
+
return sink;
|
|
749
|
+
},
|
|
750
|
+
writeStatus(line) {
|
|
751
|
+
if (!finished) status = statusOf(line);
|
|
752
|
+
return sink;
|
|
753
|
+
},
|
|
754
|
+
writeHeader(key, value) {
|
|
755
|
+
if (!finished) headers.push([key, value]);
|
|
756
|
+
return sink;
|
|
1240
757
|
},
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
758
|
+
end(body) {
|
|
759
|
+
finish(bodyBytes(body));
|
|
760
|
+
return sink;
|
|
761
|
+
},
|
|
762
|
+
endWithoutBody() {
|
|
763
|
+
finish(new Uint8Array(0));
|
|
764
|
+
return sink;
|
|
765
|
+
},
|
|
766
|
+
tryEnd(body) {
|
|
767
|
+
finish(body);
|
|
768
|
+
return [true, true];
|
|
769
|
+
},
|
|
770
|
+
getWriteOffset: () => 0,
|
|
771
|
+
onWritable: () => sink,
|
|
772
|
+
onAborted: () => sink,
|
|
773
|
+
onData: () => sink,
|
|
774
|
+
getRemoteAddressAsText: () => new TextEncoder().encode(remoteAddress).buffer
|
|
775
|
+
};
|
|
776
|
+
return {
|
|
777
|
+
sink,
|
|
778
|
+
done,
|
|
779
|
+
finishUnanswered() {
|
|
780
|
+
if (finished) return;
|
|
781
|
+
status = NO_RESPONSE_STATUS;
|
|
782
|
+
finish(new Uint8Array(0));
|
|
1244
783
|
}
|
|
1245
784
|
};
|
|
1246
785
|
}
|
|
1247
|
-
function
|
|
1248
|
-
return
|
|
1249
|
-
if (abort.aborted) {
|
|
1250
|
-
reject(new Error("aborted"));
|
|
1251
|
-
return;
|
|
1252
|
-
}
|
|
1253
|
-
let done = false;
|
|
1254
|
-
let total = 0;
|
|
1255
|
-
const chunks = [];
|
|
1256
|
-
abort.onAbort(() => {
|
|
1257
|
-
if (done) return;
|
|
1258
|
-
done = true;
|
|
1259
|
-
reject(new Error("aborted"));
|
|
1260
|
-
});
|
|
1261
|
-
res.onData((chunk, isLast) => {
|
|
1262
|
-
if (done || abort.aborted) return;
|
|
1263
|
-
total += chunk.byteLength;
|
|
1264
|
-
if (total > maxBytes) {
|
|
1265
|
-
done = true;
|
|
1266
|
-
if (!abort.aborted) {
|
|
1267
|
-
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
1268
|
-
}
|
|
1269
|
-
reject(new Error("payload too large"));
|
|
1270
|
-
return;
|
|
1271
|
-
}
|
|
1272
|
-
chunks.push(Buffer.from(new Uint8Array(chunk)));
|
|
1273
|
-
if (isLast) {
|
|
1274
|
-
done = true;
|
|
1275
|
-
resolve(Buffer.concat(chunks));
|
|
1276
|
-
}
|
|
1277
|
-
});
|
|
1278
|
-
});
|
|
786
|
+
function toRelayedResponse(requestId, captured) {
|
|
787
|
+
return { type: "response", requestId, status: captured.status, headers: captured.headers, body: captured.body };
|
|
1279
788
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
789
|
+
|
|
790
|
+
// src/server/request-threads/host.ts
|
|
791
|
+
var NO_ABORT = { aborted: false, onAbort: () => void 0 };
|
|
792
|
+
function isRelayRequest(message) {
|
|
793
|
+
if (typeof message !== "object" || message === null) return false;
|
|
794
|
+
const { type, requestId } = message;
|
|
795
|
+
if (type === "touch") return true;
|
|
796
|
+
return typeof requestId === "number" && (type === "request" || type === "authorize" || type === "embed");
|
|
797
|
+
}
|
|
798
|
+
async function answerRequest(handlers, request) {
|
|
799
|
+
const captured = createCapturingResponse(request.hookContext?.remoteAddress ?? "");
|
|
800
|
+
const ctx = {
|
|
801
|
+
res: captured.sink,
|
|
802
|
+
params: request.params,
|
|
803
|
+
query: new URLSearchParams(request.query),
|
|
804
|
+
contentType: request.contentType,
|
|
805
|
+
rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody),
|
|
806
|
+
abort: NO_ABORT,
|
|
807
|
+
hookContext: request.hookContext
|
|
808
|
+
};
|
|
1298
809
|
try {
|
|
1299
|
-
|
|
810
|
+
await handlers[request.route](ctx);
|
|
1300
811
|
} catch {
|
|
1301
|
-
|
|
812
|
+
sendError(ctx.res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
1302
813
|
}
|
|
814
|
+
captured.finishUnanswered();
|
|
815
|
+
return toRelayedResponse(request.requestId, await captured.done);
|
|
1303
816
|
}
|
|
1304
|
-
function
|
|
1305
|
-
|
|
817
|
+
async function answerAuthorize(authorize, request) {
|
|
818
|
+
if (authorize === void 0) {
|
|
819
|
+
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
|
|
820
|
+
}
|
|
821
|
+
const authorization = await authorize(request.context);
|
|
822
|
+
if (authorization.allowed) {
|
|
823
|
+
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
|
|
824
|
+
}
|
|
825
|
+
return {
|
|
826
|
+
type: "authorized",
|
|
827
|
+
requestId: request.requestId,
|
|
828
|
+
denial: authorization.denial,
|
|
829
|
+
hookError: authorization.denial === null
|
|
830
|
+
};
|
|
1306
831
|
}
|
|
1307
|
-
async function
|
|
832
|
+
async function answerEmbed(core, request) {
|
|
1308
833
|
try {
|
|
1309
|
-
const
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
834
|
+
const entry = core.requireIndex(request.indexName);
|
|
835
|
+
const resolved = await resolveVectorText(
|
|
836
|
+
{ vector: request.vector },
|
|
837
|
+
entry.embeddingAdapter,
|
|
838
|
+
core.abortController.signal,
|
|
839
|
+
entry.embeddingAdapterName
|
|
840
|
+
);
|
|
841
|
+
const value = resolved.vector?.value;
|
|
842
|
+
return {
|
|
843
|
+
type: "embedded",
|
|
844
|
+
requestId: request.requestId,
|
|
845
|
+
value: value === void 0 ? null : Array.from(value),
|
|
846
|
+
error: null
|
|
847
|
+
};
|
|
848
|
+
} catch (err) {
|
|
849
|
+
const error = err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: err instanceof Error ? err.message : String(err) };
|
|
850
|
+
return { type: "embedded", requestId: request.requestId, value: null, error };
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
854
|
+
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
855
|
+
}
|
|
856
|
+
async function startRequestThreads(options) {
|
|
857
|
+
const found = acceptorOf(options.app);
|
|
858
|
+
if (found === null) return null;
|
|
859
|
+
const acceptor = found;
|
|
860
|
+
const { core, handlers, authorize } = options;
|
|
861
|
+
const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
|
|
862
|
+
const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
|
|
863
|
+
const threads = /* @__PURE__ */ new Map();
|
|
864
|
+
let stopped = false;
|
|
865
|
+
function relayFrom(port) {
|
|
866
|
+
port.on("message", (message) => {
|
|
867
|
+
if (!isRelayRequest(message)) return;
|
|
868
|
+
if (message.type === "touch") {
|
|
869
|
+
core.orchestrator.noteAccess(message.indexName);
|
|
870
|
+
return;
|
|
871
|
+
}
|
|
872
|
+
const reply = message.type === "request" ? answerRequest(handlers, message) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
|
|
873
|
+
void reply.then((answer) => port.postMessage(answer));
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
async function stopThread(workerId, thread) {
|
|
877
|
+
try {
|
|
878
|
+
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
879
|
+
} catch {
|
|
1313
880
|
}
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
return false;
|
|
881
|
+
thread.port.close();
|
|
882
|
+
clearGateSlot(options.gate, gateSlotOfWorker(workerId));
|
|
883
|
+
await thread.executor.execute({ type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(() => void 0);
|
|
1318
884
|
}
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
req.forEach((key, value) => {
|
|
1345
|
-
headers[key] = value;
|
|
1346
|
-
});
|
|
1347
|
-
hookCtx = {
|
|
1348
|
-
method: req.getMethod(),
|
|
1349
|
-
path: req.getUrl(),
|
|
1350
|
-
headers,
|
|
1351
|
-
remoteAddress: Buffer.from(res.getRemoteAddressAsText()).toString()
|
|
1352
|
-
};
|
|
1353
|
-
}
|
|
1354
|
-
if (writeCors) writeCors(res, req);
|
|
1355
|
-
const abort = initAbortHandler(res);
|
|
1356
|
-
if (malformed !== null) {
|
|
1357
|
-
sendError(
|
|
1358
|
-
res,
|
|
1359
|
-
400,
|
|
1360
|
-
ServerErrorCodes.INVALID_REQUEST,
|
|
1361
|
-
"A path segment holds a percent-escape the server cannot decode",
|
|
1362
|
-
{ segment: malformed }
|
|
885
|
+
const listener = {
|
|
886
|
+
async onWorkerReady(workerId, executor) {
|
|
887
|
+
if (stopped) return;
|
|
888
|
+
const channel = new MessageChannel();
|
|
889
|
+
relayFrom(channel.port2);
|
|
890
|
+
let result;
|
|
891
|
+
try {
|
|
892
|
+
result = await executor.execute(
|
|
893
|
+
{
|
|
894
|
+
type: "serveRequests",
|
|
895
|
+
requestId: `serve-${workerId}`,
|
|
896
|
+
settings: {
|
|
897
|
+
port: channel.port1,
|
|
898
|
+
gate: options.gate,
|
|
899
|
+
gateSlot: gateSlotOfWorker(workerId),
|
|
900
|
+
maxConcurrentRequests: options.limits.maxConcurrentRequests,
|
|
901
|
+
touchIntervalMs,
|
|
902
|
+
cors: options.cors,
|
|
903
|
+
limits: options.limits,
|
|
904
|
+
build: options.build,
|
|
905
|
+
authorizes: authorize !== void 0,
|
|
906
|
+
searchHooks
|
|
907
|
+
}
|
|
908
|
+
},
|
|
909
|
+
[channel.port1]
|
|
1363
910
|
);
|
|
911
|
+
} catch (err) {
|
|
912
|
+
channel.port2.close();
|
|
913
|
+
console.warn(`Worker ${workerId} could not start taking requests:`, err);
|
|
1364
914
|
return;
|
|
1365
915
|
}
|
|
1366
|
-
const
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
};
|
|
916
|
+
const thread = { descriptor: result.descriptor, port: channel.port2, executor };
|
|
917
|
+
if (stopped) {
|
|
918
|
+
await stopThread(workerId, thread);
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
acceptor.addChildAppDescriptor(result.descriptor);
|
|
922
|
+
threads.set(workerId, thread);
|
|
923
|
+
},
|
|
924
|
+
onWorkerGone(workerId) {
|
|
925
|
+
const thread = threads.get(workerId);
|
|
926
|
+
if (thread === void 0) return;
|
|
927
|
+
threads.delete(workerId);
|
|
928
|
+
try {
|
|
929
|
+
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
930
|
+
} catch {
|
|
931
|
+
}
|
|
932
|
+
thread.port.close();
|
|
933
|
+
clearGateSlot(options.gate, gateSlotOfWorker(workerId));
|
|
934
|
+
}
|
|
1386
935
|
};
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
936
|
+
await core.orchestrator.serveRequestsOnWorkers(listener);
|
|
937
|
+
return {
|
|
938
|
+
get threadCount() {
|
|
939
|
+
return threads.size;
|
|
940
|
+
},
|
|
941
|
+
async stop() {
|
|
942
|
+
stopped = true;
|
|
943
|
+
core.orchestrator.stopRequestThreads();
|
|
944
|
+
const stopping = [...threads.entries()].map(([workerId, thread]) => {
|
|
945
|
+
threads.delete(workerId);
|
|
946
|
+
return stopThread(workerId, thread);
|
|
947
|
+
});
|
|
948
|
+
await Promise.all(stopping);
|
|
1398
949
|
}
|
|
1399
|
-
|
|
1400
|
-
return resolved;
|
|
1401
|
-
} catch (err) {
|
|
1402
|
-
throw new NarsilError(
|
|
1403
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1404
|
-
"The `uWebSockets.js` package is required to run the Narsil HTTP server. Install it with `pnpm add github:uNetworking/uWebSockets.js#v20.58.0` (or the npm or yarn equivalent).",
|
|
1405
|
-
{ cause: err instanceof Error ? err.message : String(err) }
|
|
1406
|
-
);
|
|
1407
|
-
}
|
|
950
|
+
};
|
|
1408
951
|
}
|
|
1409
952
|
|
|
1410
953
|
// src/server/task-store.ts
|
|
1411
|
-
var DEFAULT_MAX_RETAINED = 1e3;
|
|
1412
954
|
var InMemoryTaskStore = class {
|
|
1413
955
|
/**
|
|
1414
956
|
* Builds a store that keeps task records in this process alone.
|
|
@@ -1416,7 +958,7 @@ var InMemoryTaskStore = class {
|
|
|
1416
958
|
* @param maxRetained - Records to keep before the oldest finished ones are
|
|
1417
959
|
* dropped. Defaults to 1000.
|
|
1418
960
|
*/
|
|
1419
|
-
constructor(maxRetained =
|
|
961
|
+
constructor(maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
|
|
1420
962
|
this.maxRetained = maxRetained;
|
|
1421
963
|
}
|
|
1422
964
|
records = /* @__PURE__ */ new Map();
|
|
@@ -1464,10 +1006,6 @@ var InMemoryTaskStore = class {
|
|
|
1464
1006
|
}
|
|
1465
1007
|
}
|
|
1466
1008
|
};
|
|
1467
|
-
var RUNNING_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
1468
|
-
var TERMINAL_TTL_MS = 60 * 60 * 1e3;
|
|
1469
|
-
var PROGRESS_WRITE_INTERVAL_MS = 250;
|
|
1470
|
-
var DEFAULT_TASK_PAGE_SIZE = 20;
|
|
1471
1009
|
function byNewestFirst(left, right) {
|
|
1472
1010
|
if (left.createdAt !== right.createdAt) return right.createdAt - left.createdAt;
|
|
1473
1011
|
return left.id.localeCompare(right.id);
|
|
@@ -1517,7 +1055,7 @@ var TaskRegistry = class {
|
|
|
1517
1055
|
settled: false
|
|
1518
1056
|
};
|
|
1519
1057
|
this.live.set(record.id, task);
|
|
1520
|
-
await this.enqueueWrite(task, record,
|
|
1058
|
+
await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
|
|
1521
1059
|
void this.drive(task, op);
|
|
1522
1060
|
return record;
|
|
1523
1061
|
}
|
|
@@ -1552,7 +1090,7 @@ var TaskRegistry = class {
|
|
|
1552
1090
|
task.controller.abort(new Error("The task was cancelled"));
|
|
1553
1091
|
const record = { ...task.record, cancelRequestedAt: Date.now() };
|
|
1554
1092
|
task.record = record;
|
|
1555
|
-
await this.enqueueWrite(task, record,
|
|
1093
|
+
await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
|
|
1556
1094
|
return { outcome: "cancelling", record };
|
|
1557
1095
|
}
|
|
1558
1096
|
/** Fails this instance's previously-running tasks after a restart. A task can
|
|
@@ -1578,7 +1116,7 @@ var TaskRegistry = class {
|
|
|
1578
1116
|
error: { code: ServerErrorCodes.TASK_INTERRUPTED, message: "The server restarted while this task was running" }
|
|
1579
1117
|
};
|
|
1580
1118
|
try {
|
|
1581
|
-
await this.store.set(failed,
|
|
1119
|
+
await this.store.set(failed, TERMINAL_TASK_TTL_MS);
|
|
1582
1120
|
} catch {
|
|
1583
1121
|
}
|
|
1584
1122
|
}
|
|
@@ -1595,9 +1133,9 @@ var TaskRegistry = class {
|
|
|
1595
1133
|
if (task.settled) return;
|
|
1596
1134
|
task.record = { ...task.record, progress };
|
|
1597
1135
|
const now = Date.now();
|
|
1598
|
-
if (now - task.lastProgressWriteAt <
|
|
1136
|
+
if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
|
|
1599
1137
|
task.lastProgressWriteAt = now;
|
|
1600
|
-
void this.enqueueWrite(task, task.record,
|
|
1138
|
+
void this.enqueueWrite(task, task.record, RUNNING_TASK_TTL_MS);
|
|
1601
1139
|
}
|
|
1602
1140
|
async drive(task, op) {
|
|
1603
1141
|
const context = {
|
|
@@ -1620,25 +1158,24 @@ var TaskRegistry = class {
|
|
|
1620
1158
|
}
|
|
1621
1159
|
task.settled = true;
|
|
1622
1160
|
task.record = { ...settled, completedAt: Date.now() };
|
|
1623
|
-
await this.enqueueWrite(task, task.record,
|
|
1161
|
+
await this.enqueueWrite(task, task.record, TERMINAL_TASK_TTL_MS);
|
|
1624
1162
|
this.live.delete(task.record.id);
|
|
1625
1163
|
}
|
|
1626
1164
|
};
|
|
1627
1165
|
|
|
1628
1166
|
// src/server/create-server.ts
|
|
1629
|
-
var MB = 1024 * 1024;
|
|
1630
1167
|
function resolveLimits(limits) {
|
|
1631
1168
|
return {
|
|
1632
|
-
maxBodyBytes: limits?.maxBodyBytes ??
|
|
1633
|
-
maxImportBytes: limits?.maxImportBytes ??
|
|
1634
|
-
maxLineBytes: limits?.maxLineBytes ??
|
|
1635
|
-
importBatchSize: limits?.importBatchSize ??
|
|
1636
|
-
maxConcurrentRequests: limits?.maxConcurrentRequests ??
|
|
1637
|
-
maxResultWindow: limits?.maxResultWindow ??
|
|
1638
|
-
maxFetchDocuments: limits?.maxFetchDocuments ??
|
|
1639
|
-
maxImportErrors: limits?.maxImportErrors ??
|
|
1640
|
-
maxTaskPageSize: limits?.maxTaskPageSize ??
|
|
1641
|
-
maxConcurrentTasks: limits?.maxConcurrentTasks ??
|
|
1169
|
+
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
1170
|
+
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
1171
|
+
maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
|
|
1172
|
+
importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
|
|
1173
|
+
maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
|
|
1174
|
+
maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
|
|
1175
|
+
maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
|
|
1176
|
+
maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
|
|
1177
|
+
maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
|
|
1178
|
+
maxConcurrentTasks: limits?.maxConcurrentTasks ?? DEFAULT_MAX_CONCURRENT_TASKS
|
|
1642
1179
|
};
|
|
1643
1180
|
}
|
|
1644
1181
|
function resolveBuild(build) {
|
|
@@ -1659,21 +1196,79 @@ function assertSecureBinding(host, options) {
|
|
|
1659
1196
|
{ host }
|
|
1660
1197
|
);
|
|
1661
1198
|
}
|
|
1199
|
+
function mainThreadHandlers(deps) {
|
|
1200
|
+
const idx = createIndexHandlers(deps);
|
|
1201
|
+
const doc = createDocumentHandlers(deps);
|
|
1202
|
+
const search = createSearchHandlers(deps);
|
|
1203
|
+
const admin = createAdminHandlers(deps);
|
|
1204
|
+
const health = createHealthHandlers(deps);
|
|
1205
|
+
const cluster = createClusterHandlers(deps);
|
|
1206
|
+
const version = createVersionHandler(deps);
|
|
1207
|
+
const capabilities = createCapabilitiesHandler();
|
|
1208
|
+
return {
|
|
1209
|
+
livez: health.livez,
|
|
1210
|
+
readyz: health.readyz,
|
|
1211
|
+
version: version.report,
|
|
1212
|
+
capabilities: capabilities.report,
|
|
1213
|
+
clusterTopology: cluster.topology,
|
|
1214
|
+
clusterAllocation: cluster.allocation,
|
|
1215
|
+
memory: admin.memory,
|
|
1216
|
+
listTasks: admin.listTasks,
|
|
1217
|
+
getTask: admin.getTask,
|
|
1218
|
+
cancelTask: admin.cancelTask,
|
|
1219
|
+
createIndex: idx.create,
|
|
1220
|
+
listIndexes: idx.list,
|
|
1221
|
+
dropIndex: idx.drop,
|
|
1222
|
+
indexStats: idx.stats,
|
|
1223
|
+
indexPartitions: idx.partitions,
|
|
1224
|
+
clearIndex: idx.clear,
|
|
1225
|
+
openIndex: idx.open,
|
|
1226
|
+
closeIndex: idx.close,
|
|
1227
|
+
insert: doc.insert,
|
|
1228
|
+
batch: doc.batch,
|
|
1229
|
+
importNdjson: createImportHandler(deps),
|
|
1230
|
+
put: doc.put,
|
|
1231
|
+
patch: doc.patch,
|
|
1232
|
+
remove: doc.remove,
|
|
1233
|
+
waitForWrites: doc.waitForWrites,
|
|
1234
|
+
search: search.search,
|
|
1235
|
+
preflight: search.preflight,
|
|
1236
|
+
suggest: search.suggest,
|
|
1237
|
+
get: doc.get,
|
|
1238
|
+
exists: doc.exists,
|
|
1239
|
+
count: doc.count,
|
|
1240
|
+
list: doc.list,
|
|
1241
|
+
multiGet: doc.multiGet,
|
|
1242
|
+
checkpoint: admin.checkpoint,
|
|
1243
|
+
rebuildAnalysis: admin.rebuildAnalysis,
|
|
1244
|
+
snapshot: admin.snapshot,
|
|
1245
|
+
vectorMaintenance: admin.vectorMaintenance,
|
|
1246
|
+
compact: admin.compact,
|
|
1247
|
+
optimize: admin.optimize,
|
|
1248
|
+
rebalance: admin.rebalance,
|
|
1249
|
+
partitionConfig: admin.partitionConfig,
|
|
1250
|
+
restore: admin.restore
|
|
1251
|
+
};
|
|
1252
|
+
}
|
|
1662
1253
|
var NarsilHttpServer = class {
|
|
1663
1254
|
host;
|
|
1664
1255
|
port;
|
|
1665
1256
|
options;
|
|
1257
|
+
engine;
|
|
1666
1258
|
deps;
|
|
1667
1259
|
uws = null;
|
|
1668
1260
|
listenSocket = null;
|
|
1669
1261
|
ready = false;
|
|
1262
|
+
threads = null;
|
|
1670
1263
|
constructor(engine, options = {}) {
|
|
1671
1264
|
this.host = options.host ?? "127.0.0.1";
|
|
1672
1265
|
this.port = options.port ?? 9876;
|
|
1673
1266
|
this.options = options;
|
|
1267
|
+
this.engine = engine;
|
|
1674
1268
|
for (const [name, adapter] of Object.entries(options.embeddingAdapters ?? {})) {
|
|
1675
1269
|
engine.registerEmbeddingAdapter(name, adapter);
|
|
1676
1270
|
}
|
|
1271
|
+
engineCoreOf(engine)?.orchestrator.shareMainThread();
|
|
1677
1272
|
const taskStore = options.taskStore ?? new InMemoryTaskStore();
|
|
1678
1273
|
const instanceId = options.instanceId ?? randomUUID();
|
|
1679
1274
|
const limits = resolveLimits(options.limits);
|
|
@@ -1691,7 +1286,7 @@ var NarsilHttpServer = class {
|
|
|
1691
1286
|
const uws = await loadUWebSockets();
|
|
1692
1287
|
this.uws = uws;
|
|
1693
1288
|
const app = uws.App();
|
|
1694
|
-
this.registerRoutes(app);
|
|
1289
|
+
await this.registerRoutes(app);
|
|
1695
1290
|
await new Promise((resolve, reject) => {
|
|
1696
1291
|
app.listen(this.host, this.port, (socket) => {
|
|
1697
1292
|
if (socket) {
|
|
@@ -1711,121 +1306,43 @@ var NarsilHttpServer = class {
|
|
|
1711
1306
|
this.uws.us_listen_socket_close(this.listenSocket);
|
|
1712
1307
|
this.listenSocket = null;
|
|
1713
1308
|
}
|
|
1309
|
+
if (this.threads !== null) {
|
|
1310
|
+
const threads = this.threads;
|
|
1311
|
+
this.threads = null;
|
|
1312
|
+
await threads.stop();
|
|
1313
|
+
}
|
|
1714
1314
|
}
|
|
1715
1315
|
get listeningPort() {
|
|
1716
1316
|
if (!this.listenSocket || !this.uws) return -1;
|
|
1717
1317
|
return this.uws.us_socket_local_port(this.listenSocket);
|
|
1718
1318
|
}
|
|
1719
|
-
|
|
1319
|
+
get requestThreadCount() {
|
|
1320
|
+
return this.threads?.threadCount ?? 0;
|
|
1321
|
+
}
|
|
1322
|
+
async registerRoutes(app) {
|
|
1720
1323
|
const cors = resolveCors(this.options.cors);
|
|
1324
|
+
const authorize = this.options.onRequest ? authorizerFor(this.options.onRequest) : void 0;
|
|
1325
|
+
const core = engineCoreOf(this.engine);
|
|
1326
|
+
const threadCount = core === void 0 ? 0 : core.orchestrator.requestThreadCount();
|
|
1327
|
+
const gate = createSharedGateBuffer(gateSlotOfWorker(threadCount));
|
|
1328
|
+
const { limits, build } = this.deps;
|
|
1329
|
+
const handlers = mainThreadHandlers(this.deps);
|
|
1721
1330
|
const run = createRouteRunner({
|
|
1722
|
-
|
|
1723
|
-
|
|
1331
|
+
authorize,
|
|
1332
|
+
gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
|
|
1724
1333
|
writeCors: cors ? corsWriter(cors) : void 0
|
|
1725
1334
|
});
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
const cluster = createClusterHandlers(this.deps);
|
|
1733
|
-
const version = createVersionHandler(this.deps);
|
|
1734
|
-
const capabilities = createCapabilitiesHandler();
|
|
1735
|
-
const importNdjson = createImportHandler(this.deps);
|
|
1736
|
-
const json = (handler, opts) => run(handler, opts);
|
|
1737
|
-
if (cors) {
|
|
1738
|
-
app.options("/*", (res, req) => {
|
|
1739
|
-
const origin = req.getHeader("origin");
|
|
1740
|
-
res.cork(() => {
|
|
1741
|
-
res.writeStatus("204 No Content");
|
|
1742
|
-
writeCorsOrigin(res, cors, origin);
|
|
1743
|
-
res.writeHeader("Access-Control-Allow-Methods", cors.methods).writeHeader("Access-Control-Allow-Headers", cors.headers).writeHeader("Access-Control-Max-Age", "86400").endWithoutBody();
|
|
1744
|
-
});
|
|
1745
|
-
});
|
|
1335
|
+
registerServerRoutes(app, run, handlers, limits, cors);
|
|
1336
|
+
if (core === void 0 || !detectRuntime().supportsWorkerThreads) return;
|
|
1337
|
+
try {
|
|
1338
|
+
this.threads = await startRequestThreads({ core, app, handlers, authorize, gate, limits, build, cors });
|
|
1339
|
+
} catch (err) {
|
|
1340
|
+
console.warn("The server could not start request threads, so the main thread answers every request:", err);
|
|
1746
1341
|
}
|
|
1747
|
-
app.get("/livez", json(health.livez, { maxBytes: 0, skipHooks: true }));
|
|
1748
|
-
app.get("/readyz", json(health.readyz, { maxBytes: 0, skipHooks: true }));
|
|
1749
|
-
app.get("/health", json(health.livez, { maxBytes: 0, skipHooks: true }));
|
|
1750
|
-
app.get("/version", json(version.report, { maxBytes: 0, skipHooks: true }));
|
|
1751
|
-
app.get("/capabilities", json(capabilities.report, { maxBytes: 0, skipHooks: true }));
|
|
1752
|
-
app.get("/cluster", json(cluster.topology, { maxBytes: 0 }));
|
|
1753
|
-
app.get("/stats/memory", json(admin.memory, { maxBytes: 0 }));
|
|
1754
|
-
app.get("/tasks", json(admin.listTasks, { maxBytes: 0 }));
|
|
1755
|
-
app.get("/tasks/:id", json(admin.getTask, { paramCount: 1, maxBytes: 0 }));
|
|
1756
|
-
app.post("/tasks/:id/_cancel", json(admin.cancelTask, { paramCount: 1, maxBytes: 0 }));
|
|
1757
|
-
app.post("/indexes", json(idx.create, { needsBody: true, maxBytes: maxBodyBytes }));
|
|
1758
|
-
app.get("/indexes", json(idx.list, { maxBytes: 0 }));
|
|
1759
|
-
app.del("/indexes/:name", json(idx.drop, { paramCount: 1, maxBytes: 0 }));
|
|
1760
|
-
app.get("/indexes/:name/stats", json(idx.stats, { paramCount: 1, maxBytes: 0 }));
|
|
1761
|
-
app.get("/indexes/:name/partitions", json(idx.partitions, { paramCount: 1, maxBytes: 0 }));
|
|
1762
|
-
app.get("/indexes/:name/cluster", json(cluster.allocation, { paramCount: 1, maxBytes: 0 }));
|
|
1763
|
-
app.post("/indexes/:name/_clear", json(idx.clear, { paramCount: 1, maxBytes: 0 }));
|
|
1764
|
-
app.post("/indexes/:name/_open", json(idx.open, { paramCount: 1, maxBytes: 0 }));
|
|
1765
|
-
app.post("/indexes/:name/_close", json(idx.close, { paramCount: 1, maxBytes: 0 }));
|
|
1766
|
-
app.get("/indexes/:name/count", json(doc.count, { paramCount: 1, maxBytes: 0 }));
|
|
1767
|
-
app.post("/indexes/:name/documents", json(doc.insert, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1768
|
-
app.post(
|
|
1769
|
-
"/indexes/:name/documents/_batch",
|
|
1770
|
-
json(doc.batch, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1771
|
-
);
|
|
1772
|
-
app.post(
|
|
1773
|
-
"/indexes/:name/documents/_list",
|
|
1774
|
-
json(doc.list, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1775
|
-
);
|
|
1776
|
-
app.post(
|
|
1777
|
-
"/indexes/:name/documents/_multi-get",
|
|
1778
|
-
json(doc.multiGet, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1779
|
-
);
|
|
1780
|
-
app.post(
|
|
1781
|
-
"/indexes/:name/documents/_import",
|
|
1782
|
-
json(importNdjson, { paramCount: 1, needsBody: true, maxBytes: maxImportBytes })
|
|
1783
|
-
);
|
|
1784
|
-
app.get("/indexes/:name/documents/:id", json(doc.get, { paramCount: 2, maxBytes: 0 }));
|
|
1785
|
-
app.get("/indexes/:name/documents/:id/_exists", json(doc.exists, { paramCount: 2, maxBytes: 0 }));
|
|
1786
|
-
app.put("/indexes/:name/documents/:id", json(doc.put, { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1787
|
-
app.patch(
|
|
1788
|
-
"/indexes/:name/documents/:id",
|
|
1789
|
-
json(doc.patch, { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes })
|
|
1790
|
-
);
|
|
1791
|
-
app.del("/indexes/:name/documents/:id", json(doc.remove, { paramCount: 2, maxBytes: 0 }));
|
|
1792
|
-
app.post("/indexes/:name/search", json(search.search, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1793
|
-
app.post(
|
|
1794
|
-
"/indexes/:name/search/preflight",
|
|
1795
|
-
json(search.preflight, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1796
|
-
);
|
|
1797
|
-
app.post("/indexes/:name/suggest", json(search.suggest, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1798
|
-
app.post("/indexes/:name/_checkpoint", json(admin.checkpoint, { paramCount: 1, maxBytes: 0 }));
|
|
1799
|
-
app.post("/indexes/:name/_rebuild-analysis", json(admin.rebuildAnalysis, { paramCount: 1, maxBytes: 0 }));
|
|
1800
|
-
app.get("/indexes/:name/snapshot", json(admin.snapshot, { paramCount: 1, maxBytes: 0 }));
|
|
1801
|
-
app.get("/indexes/:name/vector-maintenance", json(admin.vectorMaintenance, { paramCount: 1, maxBytes: 0 }));
|
|
1802
|
-
app.post(
|
|
1803
|
-
"/indexes/:name/vectors/_compact",
|
|
1804
|
-
json(admin.compact, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1805
|
-
);
|
|
1806
|
-
app.post(
|
|
1807
|
-
"/indexes/:name/vectors/_optimize",
|
|
1808
|
-
json(admin.optimize, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1809
|
-
);
|
|
1810
|
-
app.post(
|
|
1811
|
-
"/indexes/:name/_rebalance",
|
|
1812
|
-
json(admin.rebalance, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1813
|
-
);
|
|
1814
|
-
app.post(
|
|
1815
|
-
"/indexes/:name/partition-config",
|
|
1816
|
-
json(admin.partitionConfig, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1817
|
-
);
|
|
1818
|
-
app.post(
|
|
1819
|
-
"/indexes/:name/restore",
|
|
1820
|
-
json(admin.restore, { paramCount: 1, needsBody: true, maxBytes: maxImportBytes })
|
|
1821
|
-
);
|
|
1822
|
-
app.any("/*", (res) => {
|
|
1823
|
-
sendError(res, 404, ServerErrorCodes.NOT_FOUND, "Route not found");
|
|
1824
|
-
});
|
|
1825
1342
|
}
|
|
1826
1343
|
};
|
|
1827
1344
|
function createServer(engine, options) {
|
|
1828
1345
|
return new NarsilHttpServer(engine, options);
|
|
1829
1346
|
}
|
|
1830
1347
|
|
|
1831
|
-
export {
|
|
1348
|
+
export { InMemoryTaskStore, createServer };
|