@delali/narsil 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -4
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +46 -2
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-XODT2F3h.d.ts → adapters-Bo2JQg8b.d.ts} +8 -1
- package/dist/chunk-ALB35PRQ.mjs +98 -0
- package/dist/{chunk-TFWBALU4.mjs → chunk-FS7XLIUS.mjs} +3219 -2063
- package/dist/chunk-GQEHCZG7.mjs +6825 -0
- package/dist/chunk-KMMJMDUA.mjs +44 -0
- package/dist/chunk-RQLXC2YS.mjs +269 -0
- package/dist/chunk-S7RJJMHI.mjs +1086 -0
- package/dist/{chunk-WYC635GI.mjs → chunk-TW6BXTOL.mjs} +10 -7
- package/dist/chunk-WISGW24C.mjs +455 -0
- package/dist/config-cQLlYR3U.d.ts +470 -0
- package/dist/direct-executor-FX56DYD3.mjs +4 -0
- package/dist/distribution/coordinator/etcd.d.ts +16 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -0
- package/dist/distribution/coordinator/in-memory.d.ts +7 -0
- package/dist/distribution/coordinator/in-memory.mjs +1 -0
- package/dist/distribution.d.ts +122 -0
- package/dist/distribution.mjs +7410 -0
- package/dist/embeddings/openai.d.ts +13 -0
- package/dist/embeddings/openai.mjs +248 -0
- package/dist/index.browser.mjs +12111 -5793
- package/dist/index.d.ts +10 -420
- package/dist/index.mjs +7 -2052
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
- package/dist/languages/arabic.d.ts +1 -1
- package/dist/languages/armenian.d.ts +1 -1
- package/dist/languages/bulgarian.d.ts +1 -1
- package/dist/languages/chinese.d.ts +1 -1
- package/dist/languages/dagbani.d.ts +1 -1
- package/dist/languages/danish.d.ts +1 -1
- package/dist/languages/dutch.d.ts +1 -1
- package/dist/languages/english.d.ts +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/ewe.d.ts +1 -1
- package/dist/languages/finnish.d.ts +1 -1
- package/dist/languages/french.d.ts +1 -1
- package/dist/languages/ga.d.ts +1 -1
- package/dist/languages/german.d.ts +1 -1
- package/dist/languages/greek.d.ts +1 -1
- package/dist/languages/hausa.d.ts +1 -1
- package/dist/languages/hindi.d.ts +1 -1
- package/dist/languages/hungarian.d.ts +1 -1
- package/dist/languages/igbo.d.ts +1 -1
- package/dist/languages/indonesian.d.ts +1 -1
- package/dist/languages/irish.d.ts +1 -1
- package/dist/languages/italian.d.ts +1 -1
- package/dist/languages/japanese.d.ts +1 -1
- package/dist/languages/nepali.d.ts +1 -1
- package/dist/languages/norwegian.d.ts +1 -1
- package/dist/languages/portuguese.d.ts +1 -1
- package/dist/languages/romanian.d.ts +1 -1
- package/dist/languages/russian.d.ts +1 -1
- package/dist/languages/sanskrit.d.ts +1 -1
- package/dist/languages/serbian.d.ts +1 -1
- package/dist/languages/slovenian.d.ts +1 -1
- package/dist/languages/spanish.d.ts +1 -1
- package/dist/languages/swahili.d.ts +1 -1
- package/dist/languages/swedish.d.ts +1 -1
- package/dist/languages/tamil.d.ts +1 -1
- package/dist/languages/turkish.d.ts +1 -1
- package/dist/languages/twi.d.ts +1 -1
- package/dist/languages/ukrainian.d.ts +1 -1
- package/dist/languages/yoruba.d.ts +1 -1
- package/dist/languages/zulu.d.ts +1 -1
- package/dist/narsil-ChTdkwPu.d.ts +68 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
- package/dist/schema-BVaw7nqz.d.ts +58 -0
- package/dist/serialization/crc32-worker.d.ts +20 -0
- package/dist/serialization/crc32-worker.mjs +61 -0
- package/dist/server.d.ts +218 -0
- package/dist/server.mjs +1185 -0
- package/dist/types-Bt7c4hDF.d.ts +73 -0
- package/dist/vector/hnsw-build-worker.d.ts +9 -2
- package/dist/vector/hnsw-build-worker.mjs +54 -10
- package/dist/workers/entry.mjs +1 -1
- package/package.json +46 -4
- package/dist/chunk-7GKPWGCN.mjs +0 -43
- package/dist/chunk-GVN2W4VA.mjs +0 -570
- package/dist/direct-executor-6TSG7GWH.mjs +0 -4
package/dist/server.mjs
ADDED
|
@@ -0,0 +1,1185 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-ALB35PRQ.mjs';
|
|
2
|
+
import { randomUUID } from 'crypto';
|
|
3
|
+
|
|
4
|
+
// src/server/cors.ts
|
|
5
|
+
function resolveCors(cors) {
|
|
6
|
+
if (!cors) return null;
|
|
7
|
+
if (cors === true) {
|
|
8
|
+
return { origin: "*", methods: "GET, POST, PUT, PATCH, DELETE, OPTIONS", headers: "Content-Type, Authorization" };
|
|
9
|
+
}
|
|
10
|
+
return {
|
|
11
|
+
origin: cors.origin ?? "*",
|
|
12
|
+
methods: cors.methods?.join(", ") ?? "GET, POST, PUT, PATCH, DELETE, OPTIONS",
|
|
13
|
+
headers: cors.headers?.join(", ") ?? "Content-Type, Authorization"
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function matchOrigin(cors, requestOrigin) {
|
|
17
|
+
if (cors.origin === "*") return "*";
|
|
18
|
+
if (typeof cors.origin === "string") return cors.origin;
|
|
19
|
+
return cors.origin.includes(requestOrigin) ? requestOrigin : null;
|
|
20
|
+
}
|
|
21
|
+
function writeCorsOrigin(res, cors, requestOrigin) {
|
|
22
|
+
const allowed = matchOrigin(cors, requestOrigin);
|
|
23
|
+
if (!allowed) return;
|
|
24
|
+
res.writeHeader("Access-Control-Allow-Origin", allowed);
|
|
25
|
+
if (allowed !== "*") res.writeHeader("Vary", "Origin");
|
|
26
|
+
}
|
|
27
|
+
function corsWriter(cors) {
|
|
28
|
+
return (res, req) => writeCorsOrigin(res, cors, req.getHeader("origin"));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// src/server/errors.ts
|
|
32
|
+
var ServerErrorCodes = {
|
|
33
|
+
INVALID_REQUEST: "INVALID_REQUEST",
|
|
34
|
+
INVALID_JSON: "INVALID_JSON",
|
|
35
|
+
EMPTY_BODY: "EMPTY_BODY",
|
|
36
|
+
PAYLOAD_TOO_LARGE: "PAYLOAD_TOO_LARGE",
|
|
37
|
+
UNSUPPORTED_MEDIA_TYPE: "UNSUPPORTED_MEDIA_TYPE",
|
|
38
|
+
NOT_FOUND: "NOT_FOUND",
|
|
39
|
+
TASK_NOT_FOUND: "TASK_NOT_FOUND",
|
|
40
|
+
TOO_MANY_REQUESTS: "TOO_MANY_REQUESTS",
|
|
41
|
+
HOOK_ERROR: "HOOK_ERROR",
|
|
42
|
+
INTERNAL_ERROR: "INTERNAL_ERROR",
|
|
43
|
+
REQUEST_ABORTED: "REQUEST_ABORTED"
|
|
44
|
+
};
|
|
45
|
+
var STATUS_BY_CODE = {
|
|
46
|
+
SCHEMA_INVALID_TYPE: 400,
|
|
47
|
+
SCHEMA_MISSING_FIELD: 400,
|
|
48
|
+
SCHEMA_DEPTH_EXCEEDED: 400,
|
|
49
|
+
SCHEMA_INVALID_VECTOR_DIMENSION: 400,
|
|
50
|
+
SCHEMA_INVALID_GEOPOINT: 400,
|
|
51
|
+
DOC_VALIDATION_FAILED: 400,
|
|
52
|
+
DOC_MISSING_REQUIRED_FIELD: 400,
|
|
53
|
+
SEARCH_INVALID_FIELD: 400,
|
|
54
|
+
SEARCH_INVALID_VECTOR_SIZE: 400,
|
|
55
|
+
VECTOR_DIMENSION_MISMATCH: 400,
|
|
56
|
+
SEARCH_INVALID_FILTER: 400,
|
|
57
|
+
SEARCH_INVALID_MODE: 400,
|
|
58
|
+
SEARCH_INVALID_CURSOR: 400,
|
|
59
|
+
LANGUAGE_NOT_SUPPORTED: 400,
|
|
60
|
+
CONFIG_INVALID: 400,
|
|
61
|
+
EMBEDDING_DIMENSION_MISMATCH: 400,
|
|
62
|
+
EMBEDDING_NO_SOURCE: 400,
|
|
63
|
+
EMBEDDING_CONFIG_INVALID: 400,
|
|
64
|
+
INDEX_NOT_FOUND: 404,
|
|
65
|
+
DOC_NOT_FOUND: 404,
|
|
66
|
+
INDEX_ALREADY_EXISTS: 409,
|
|
67
|
+
DOC_ALREADY_EXISTS: 409,
|
|
68
|
+
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
69
|
+
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
70
|
+
WORKER_BUSY: 429,
|
|
71
|
+
PARTITION_CORRUPTED: 503,
|
|
72
|
+
WORKER_CRASHED: 503,
|
|
73
|
+
WORKER_TIMEOUT: 503,
|
|
74
|
+
PERSISTENCE_SAVE_FAILED: 503,
|
|
75
|
+
PERSISTENCE_LOAD_FAILED: 503,
|
|
76
|
+
PERSISTENCE_DELETE_FAILED: 503,
|
|
77
|
+
PERSISTENCE_CRC_MISMATCH: 503,
|
|
78
|
+
PERSISTENCE_WAL_CORRUPT: 503,
|
|
79
|
+
PERSISTENCE_FSYNC_FAILED: 503,
|
|
80
|
+
EMBEDDING_FAILED: 503,
|
|
81
|
+
QUERY_ROUTING_FAILED: 503,
|
|
82
|
+
QUERY_PARTIAL_FAILURE: 503,
|
|
83
|
+
QUERY_NODE_TIMEOUT: 503,
|
|
84
|
+
QUERY_NO_ACTIVE_REPLICA: 503
|
|
85
|
+
};
|
|
86
|
+
function httpStatusForNarsilError(code) {
|
|
87
|
+
return STATUS_BY_CODE[code] ?? 500;
|
|
88
|
+
}
|
|
89
|
+
function serializeNarsilError(err) {
|
|
90
|
+
const hasDetails = err.details && Object.keys(err.details).length > 0;
|
|
91
|
+
return hasDetails ? { code: err.code, message: err.message, details: err.details } : { code: err.code, message: err.message };
|
|
92
|
+
}
|
|
93
|
+
function toHttpError(err) {
|
|
94
|
+
if (err instanceof NarsilError) {
|
|
95
|
+
return { status: httpStatusForNarsilError(err.code), body: serializeNarsilError(err) };
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
status: 500,
|
|
99
|
+
body: { code: ServerErrorCodes.INTERNAL_ERROR, message: "An unexpected error occurred" }
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// src/server/response.ts
|
|
104
|
+
var STATUS_TEXT = {
|
|
105
|
+
200: "200 OK",
|
|
106
|
+
201: "201 Created",
|
|
107
|
+
202: "202 Accepted",
|
|
108
|
+
204: "204 No Content",
|
|
109
|
+
400: "400 Bad Request",
|
|
110
|
+
401: "401 Unauthorized",
|
|
111
|
+
403: "403 Forbidden",
|
|
112
|
+
404: "404 Not Found",
|
|
113
|
+
405: "405 Method Not Allowed",
|
|
114
|
+
409: "409 Conflict",
|
|
115
|
+
413: "413 Payload Too Large",
|
|
116
|
+
415: "415 Unsupported Media Type",
|
|
117
|
+
429: "429 Too Many Requests",
|
|
118
|
+
500: "500 Internal Server Error",
|
|
119
|
+
503: "503 Service Unavailable"
|
|
120
|
+
};
|
|
121
|
+
function statusLine(status) {
|
|
122
|
+
return STATUS_TEXT[status] ?? `${status}`;
|
|
123
|
+
}
|
|
124
|
+
function sendJson(res, data, status = 200) {
|
|
125
|
+
const payload = JSON.stringify(data);
|
|
126
|
+
res.cork(() => {
|
|
127
|
+
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
function sendError(res, status, code, message, details) {
|
|
131
|
+
const body = { error: details ? { code, message, details } : { code, message } };
|
|
132
|
+
const payload = JSON.stringify(body);
|
|
133
|
+
res.cork(() => {
|
|
134
|
+
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
function sendBinary(res, data, status = 200) {
|
|
138
|
+
res.cork(() => {
|
|
139
|
+
res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/octet-stream").end(data);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// src/server/handler-utils.ts
|
|
144
|
+
function parseJson(ctx) {
|
|
145
|
+
const raw = ctx.rawBody;
|
|
146
|
+
if (!raw || raw.length === 0) {
|
|
147
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
148
|
+
return void 0;
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
return JSON.parse(raw.toString("utf8"));
|
|
152
|
+
} catch {
|
|
153
|
+
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
154
|
+
return void 0;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function parseJsonOptional(ctx) {
|
|
158
|
+
const raw = ctx.rawBody;
|
|
159
|
+
if (!raw || raw.length === 0) return {};
|
|
160
|
+
try {
|
|
161
|
+
return JSON.parse(raw.toString("utf8"));
|
|
162
|
+
} catch {
|
|
163
|
+
sendError(ctx.res, 400, ServerErrorCodes.INVALID_JSON, "Request body is not valid JSON");
|
|
164
|
+
return void 0;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function respondError(ctx, err) {
|
|
168
|
+
if (ctx.abort.aborted) return;
|
|
169
|
+
const { status, body } = toHttpError(err);
|
|
170
|
+
sendError(ctx.res, status, body.code, body.message, body.details);
|
|
171
|
+
}
|
|
172
|
+
function respondJson(ctx, data, status = 200) {
|
|
173
|
+
if (ctx.abort.aborted) return;
|
|
174
|
+
sendJson(ctx.res, data, status);
|
|
175
|
+
}
|
|
176
|
+
function badRequest(res, message, details) {
|
|
177
|
+
sendError(res, 400, ServerErrorCodes.INVALID_REQUEST, message, details);
|
|
178
|
+
}
|
|
179
|
+
function serializeBatchResult(result) {
|
|
180
|
+
return {
|
|
181
|
+
succeeded: result.succeeded,
|
|
182
|
+
failed: result.failed.map((f) => ({
|
|
183
|
+
docId: f.docId,
|
|
184
|
+
error: f.error instanceof NarsilError ? serializeNarsilError(f.error) : { code: "INTERNAL_ERROR", message: String(f.error) }
|
|
185
|
+
}))
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// src/server/handlers/admin.ts
|
|
190
|
+
function taskResponse(record) {
|
|
191
|
+
return { taskId: record.id, type: record.type, status: record.status };
|
|
192
|
+
}
|
|
193
|
+
function createAdminHandlers(deps) {
|
|
194
|
+
const { engine, tasks } = deps;
|
|
195
|
+
async function checkpoint(ctx) {
|
|
196
|
+
try {
|
|
197
|
+
await engine.checkpoint(ctx.params[0]);
|
|
198
|
+
respondJson(ctx, { ok: true });
|
|
199
|
+
} catch (err) {
|
|
200
|
+
respondError(ctx, err);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
async function snapshot(ctx) {
|
|
204
|
+
try {
|
|
205
|
+
const bytes = await engine.snapshot(ctx.params[0]);
|
|
206
|
+
if (ctx.abort.aborted) return;
|
|
207
|
+
sendBinary(ctx.res, bytes);
|
|
208
|
+
} catch (err) {
|
|
209
|
+
respondError(ctx, err);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function vectorMaintenance(ctx) {
|
|
213
|
+
try {
|
|
214
|
+
respondJson(ctx, { fields: engine.vectorMaintenanceStatus(ctx.params[0]) });
|
|
215
|
+
} catch (err) {
|
|
216
|
+
respondError(ctx, err);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
async function compact(ctx) {
|
|
220
|
+
const body = parseJsonOptional(ctx);
|
|
221
|
+
if (!body) return;
|
|
222
|
+
try {
|
|
223
|
+
await engine.compactVectors(ctx.params[0], body.field);
|
|
224
|
+
respondJson(ctx, { ok: true });
|
|
225
|
+
} catch (err) {
|
|
226
|
+
respondError(ctx, err);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
async function partitionConfig(ctx) {
|
|
230
|
+
const body = parseJsonOptional(ctx);
|
|
231
|
+
if (!body) return;
|
|
232
|
+
try {
|
|
233
|
+
await engine.updatePartitionConfig(ctx.params[0], body);
|
|
234
|
+
respondJson(ctx, { ok: true });
|
|
235
|
+
} catch (err) {
|
|
236
|
+
respondError(ctx, err);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
async function memory(ctx) {
|
|
240
|
+
try {
|
|
241
|
+
respondJson(ctx, await engine.getMemoryStats());
|
|
242
|
+
} catch (err) {
|
|
243
|
+
respondError(ctx, err);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
function requireIndexExists(name) {
|
|
247
|
+
engine.getStats(name);
|
|
248
|
+
}
|
|
249
|
+
async function optimize(ctx) {
|
|
250
|
+
const body = parseJsonOptional(ctx);
|
|
251
|
+
if (!body) return;
|
|
252
|
+
const name = ctx.params[0];
|
|
253
|
+
try {
|
|
254
|
+
requireIndexExists(name);
|
|
255
|
+
} catch (err) {
|
|
256
|
+
respondError(ctx, err);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
const record = await tasks.start("optimizeVectors", name, () => engine.optimizeVectors(name, body.field));
|
|
260
|
+
respondJson(ctx, taskResponse(record), 202);
|
|
261
|
+
}
|
|
262
|
+
async function rebalance(ctx) {
|
|
263
|
+
const body = parseJsonOptional(ctx);
|
|
264
|
+
if (!body) return;
|
|
265
|
+
const target = body.targetPartitionCount;
|
|
266
|
+
if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
|
|
267
|
+
badRequest(ctx.res, 'Field "targetPartitionCount" is required and must be a positive integer');
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
const name = ctx.params[0];
|
|
271
|
+
try {
|
|
272
|
+
requireIndexExists(name);
|
|
273
|
+
} catch (err) {
|
|
274
|
+
respondError(ctx, err);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
const record = await tasks.start("rebalance", name, () => engine.rebalance(name, target));
|
|
278
|
+
respondJson(ctx, taskResponse(record), 202);
|
|
279
|
+
}
|
|
280
|
+
async function restore(ctx) {
|
|
281
|
+
const raw = ctx.rawBody;
|
|
282
|
+
if (!raw || raw.length === 0) {
|
|
283
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
const name = ctx.params[0];
|
|
287
|
+
const bytes = new Uint8Array(raw);
|
|
288
|
+
const record = await tasks.start("restore", name, () => engine.restore(name, bytes));
|
|
289
|
+
respondJson(ctx, taskResponse(record), 202);
|
|
290
|
+
}
|
|
291
|
+
async function getTask(ctx) {
|
|
292
|
+
const record = await tasks.get(ctx.params[0]);
|
|
293
|
+
if (!record) {
|
|
294
|
+
sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${ctx.params[0]}" not found`);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
respondJson(ctx, record);
|
|
298
|
+
}
|
|
299
|
+
async function listTasks(ctx) {
|
|
300
|
+
respondJson(ctx, { tasks: await tasks.list() });
|
|
301
|
+
}
|
|
302
|
+
return {
|
|
303
|
+
checkpoint,
|
|
304
|
+
snapshot,
|
|
305
|
+
vectorMaintenance,
|
|
306
|
+
compact,
|
|
307
|
+
partitionConfig,
|
|
308
|
+
memory,
|
|
309
|
+
optimize,
|
|
310
|
+
rebalance,
|
|
311
|
+
restore,
|
|
312
|
+
getTask,
|
|
313
|
+
listTasks
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// src/server/handlers/documents.ts
|
|
318
|
+
function isPlainObject(value) {
|
|
319
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
320
|
+
}
|
|
321
|
+
function createDocumentHandlers(deps) {
|
|
322
|
+
const { engine } = deps;
|
|
323
|
+
async function insert(ctx) {
|
|
324
|
+
const body = parseJson(ctx);
|
|
325
|
+
if (!body) return;
|
|
326
|
+
if (!isPlainObject(body.document)) {
|
|
327
|
+
badRequest(ctx.res, 'Field "document" is required and must be an object');
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
try {
|
|
331
|
+
const id = await engine.insert(ctx.params[0], body.document, body.id, body.options);
|
|
332
|
+
respondJson(ctx, { id }, 201);
|
|
333
|
+
} catch (err) {
|
|
334
|
+
respondError(ctx, err);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
async function get(ctx) {
|
|
338
|
+
try {
|
|
339
|
+
const document = await engine.get(ctx.params[0], ctx.params[1]);
|
|
340
|
+
if (document === void 0) {
|
|
341
|
+
respondError(ctx, indexDocNotFound(ctx.params[1]));
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
respondJson(ctx, { document });
|
|
345
|
+
} catch (err) {
|
|
346
|
+
respondError(ctx, err);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
async function exists(ctx) {
|
|
350
|
+
try {
|
|
351
|
+
respondJson(ctx, { exists: await engine.has(ctx.params[0], ctx.params[1]) });
|
|
352
|
+
} catch (err) {
|
|
353
|
+
respondError(ctx, err);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
async function put(ctx) {
|
|
357
|
+
const body = parseJson(ctx);
|
|
358
|
+
if (!body) return;
|
|
359
|
+
if (!isPlainObject(body.document)) {
|
|
360
|
+
badRequest(ctx.res, 'Field "document" is required and must be an object');
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
const [name, id] = ctx.params;
|
|
364
|
+
try {
|
|
365
|
+
if (await engine.has(name, id)) {
|
|
366
|
+
await engine.update(name, id, body.document);
|
|
367
|
+
respondJson(ctx, { id, created: false });
|
|
368
|
+
} else {
|
|
369
|
+
const created = await engine.insert(name, body.document, id);
|
|
370
|
+
respondJson(ctx, { id: created, created: true }, 201);
|
|
371
|
+
}
|
|
372
|
+
} catch (err) {
|
|
373
|
+
respondError(ctx, err);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
async function patch(ctx) {
|
|
377
|
+
const body = parseJson(ctx);
|
|
378
|
+
if (!body) return;
|
|
379
|
+
if (!isPlainObject(body.document)) {
|
|
380
|
+
badRequest(ctx.res, 'Field "document" is required and must be an object');
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
try {
|
|
384
|
+
await engine.update(ctx.params[0], ctx.params[1], body.document);
|
|
385
|
+
respondJson(ctx, { id: ctx.params[1] });
|
|
386
|
+
} catch (err) {
|
|
387
|
+
respondError(ctx, err);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
async function remove(ctx) {
|
|
391
|
+
try {
|
|
392
|
+
await engine.remove(ctx.params[0], ctx.params[1]);
|
|
393
|
+
respondJson(ctx, { id: ctx.params[1], removed: true });
|
|
394
|
+
} catch (err) {
|
|
395
|
+
respondError(ctx, err);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
async function count(ctx) {
|
|
399
|
+
try {
|
|
400
|
+
respondJson(ctx, { count: await engine.countDocuments(ctx.params[0]) });
|
|
401
|
+
} catch (err) {
|
|
402
|
+
respondError(ctx, err);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
async function multiGet(ctx) {
|
|
406
|
+
const body = parseJson(ctx);
|
|
407
|
+
if (!body) return;
|
|
408
|
+
if (!Array.isArray(body.docIds)) {
|
|
409
|
+
badRequest(ctx.res, 'Field "docIds" is required and must be an array of strings');
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
try {
|
|
413
|
+
const found = await engine.getMultiple(ctx.params[0], body.docIds);
|
|
414
|
+
const documents = {};
|
|
415
|
+
for (const [id, doc] of found) documents[id] = doc;
|
|
416
|
+
respondJson(ctx, { documents });
|
|
417
|
+
} catch (err) {
|
|
418
|
+
respondError(ctx, err);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
async function batch(ctx) {
|
|
422
|
+
const body = parseJson(ctx);
|
|
423
|
+
if (!body) return;
|
|
424
|
+
const action = body.action ?? "insert";
|
|
425
|
+
try {
|
|
426
|
+
if (action === "insert") {
|
|
427
|
+
if (!Array.isArray(body.documents)) {
|
|
428
|
+
badRequest(ctx.res, 'Batch insert requires a "documents" array');
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
respondJson(ctx, serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents, body.options)));
|
|
432
|
+
} else if (action === "update") {
|
|
433
|
+
if (!Array.isArray(body.updates)) {
|
|
434
|
+
badRequest(ctx.res, 'Batch update requires an "updates" array');
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
respondJson(ctx, serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates)));
|
|
438
|
+
} else if (action === "delete") {
|
|
439
|
+
if (!Array.isArray(body.docIds)) {
|
|
440
|
+
badRequest(ctx.res, 'Batch delete requires a "docIds" array');
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds)));
|
|
444
|
+
} else {
|
|
445
|
+
badRequest(ctx.res, 'Field "action" must be one of "insert", "update", or "delete"');
|
|
446
|
+
}
|
|
447
|
+
} catch (err) {
|
|
448
|
+
respondError(ctx, err);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return { insert, get, exists, put, patch, remove, count, multiGet, batch };
|
|
452
|
+
}
|
|
453
|
+
function indexDocNotFound(docId) {
|
|
454
|
+
return new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found`, { docId });
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// src/server/handlers/health.ts
|
|
458
|
+
function createHealthHandlers(deps) {
|
|
459
|
+
function livez(ctx) {
|
|
460
|
+
respondJson(ctx, { status: "ok" });
|
|
461
|
+
}
|
|
462
|
+
function readyz(ctx) {
|
|
463
|
+
if (deps.isReady()) {
|
|
464
|
+
respondJson(ctx, { status: "ready" });
|
|
465
|
+
} else {
|
|
466
|
+
respondJson(ctx, { status: "unavailable" }, 503);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return { livez, readyz };
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// src/server/ndjson.ts
|
|
473
|
+
var NEWLINE = 10;
|
|
474
|
+
var CARRIAGE_RETURN = 13;
|
|
475
|
+
var NdjsonLineTooLongError = class extends Error {
|
|
476
|
+
constructor(lineNumber) {
|
|
477
|
+
super(`NDJSON line ${lineNumber} exceeds the configured per-line size limit`);
|
|
478
|
+
this.lineNumber = lineNumber;
|
|
479
|
+
this.name = "NdjsonLineTooLongError";
|
|
480
|
+
}
|
|
481
|
+
};
|
|
482
|
+
function* iterateNdjson(buffer, maxLineBytes) {
|
|
483
|
+
let start = 0;
|
|
484
|
+
let lineNumber = 0;
|
|
485
|
+
const length = buffer.length;
|
|
486
|
+
while (start < length) {
|
|
487
|
+
let end = buffer.indexOf(NEWLINE, start);
|
|
488
|
+
if (end === -1) end = length;
|
|
489
|
+
if (end - start > maxLineBytes) throw new NdjsonLineTooLongError(lineNumber + 1);
|
|
490
|
+
let contentEnd = end;
|
|
491
|
+
if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
|
|
492
|
+
if (contentEnd > start) {
|
|
493
|
+
lineNumber++;
|
|
494
|
+
yield { lineNumber, text: buffer.toString("utf8", start, contentEnd) };
|
|
495
|
+
}
|
|
496
|
+
start = end + 1;
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// src/server/handlers/import.ts
|
|
501
|
+
function yieldToEventLoop() {
|
|
502
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
503
|
+
}
|
|
504
|
+
function isPlainObject2(value) {
|
|
505
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
506
|
+
}
|
|
507
|
+
function createImportHandler(deps) {
|
|
508
|
+
const { engine, limits } = deps;
|
|
509
|
+
return async function importNdjson(ctx) {
|
|
510
|
+
const raw = ctx.rawBody;
|
|
511
|
+
if (!raw || raw.length === 0) {
|
|
512
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
const name = ctx.params[0];
|
|
516
|
+
const errors = [];
|
|
517
|
+
let indexed = 0;
|
|
518
|
+
let pending = [];
|
|
519
|
+
const flush = async () => {
|
|
520
|
+
if (pending.length === 0) return true;
|
|
521
|
+
const documents = pending.map((p) => p.document);
|
|
522
|
+
try {
|
|
523
|
+
const result = await engine.insertBatch(name, documents);
|
|
524
|
+
indexed += result.succeeded.length;
|
|
525
|
+
for (const failure of result.failed) {
|
|
526
|
+
const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: "INTERNAL_ERROR", message: String(failure.error) };
|
|
527
|
+
errors.push({ docId: failure.docId, code: serialized.code, message: serialized.message });
|
|
528
|
+
}
|
|
529
|
+
pending = [];
|
|
530
|
+
return true;
|
|
531
|
+
} catch (err) {
|
|
532
|
+
respondError(ctx, err);
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
};
|
|
536
|
+
try {
|
|
537
|
+
for (const { lineNumber, text } of iterateNdjson(raw, limits.maxLineBytes)) {
|
|
538
|
+
let parsed;
|
|
539
|
+
try {
|
|
540
|
+
parsed = JSON.parse(text);
|
|
541
|
+
} catch {
|
|
542
|
+
errors.push({ line: lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
if (!isPlainObject2(parsed)) {
|
|
546
|
+
errors.push({
|
|
547
|
+
line: lineNumber,
|
|
548
|
+
code: ServerErrorCodes.INVALID_REQUEST,
|
|
549
|
+
message: "Line is not a JSON object"
|
|
550
|
+
});
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
pending.push({ document: parsed, line: lineNumber });
|
|
554
|
+
if (pending.length >= limits.importBatchSize) {
|
|
555
|
+
if (!await flush()) return;
|
|
556
|
+
if (ctx.abort.aborted) return;
|
|
557
|
+
await yieldToEventLoop();
|
|
558
|
+
if (ctx.abort.aborted) return;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
} catch (err) {
|
|
562
|
+
if (err instanceof NdjsonLineTooLongError) {
|
|
563
|
+
sendError(ctx.res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, err.message, { line: err.lineNumber });
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
respondError(ctx, err);
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
569
|
+
if (!await flush()) return;
|
|
570
|
+
if (ctx.abort.aborted) return;
|
|
571
|
+
respondJson(ctx, { indexed, failed: errors.length, errors });
|
|
572
|
+
};
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// src/server/config-mapping.ts
|
|
576
|
+
function mapHttpIndexConfig(http, adapters) {
|
|
577
|
+
if (typeof http !== "object" || http === null) {
|
|
578
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index config must be an object");
|
|
579
|
+
}
|
|
580
|
+
if (typeof http.schema !== "object" || http.schema === null || Array.isArray(http.schema)) {
|
|
581
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Index config requires a "schema" object');
|
|
582
|
+
}
|
|
583
|
+
const config = { schema: http.schema };
|
|
584
|
+
if (http.language !== void 0) config.language = http.language;
|
|
585
|
+
if (http.partitions !== void 0) config.partitions = http.partitions;
|
|
586
|
+
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
587
|
+
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
588
|
+
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
589
|
+
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
590
|
+
if (http.strict !== void 0) config.strict = http.strict;
|
|
591
|
+
if (http.required !== void 0) config.required = http.required;
|
|
592
|
+
if (http.stopWords !== void 0) {
|
|
593
|
+
if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
|
|
594
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "stopWords" must be an array of strings');
|
|
595
|
+
}
|
|
596
|
+
config.stopWords = new Set(http.stopWords);
|
|
597
|
+
}
|
|
598
|
+
if (http.embedding !== void 0) {
|
|
599
|
+
config.embedding = mapEmbedding(http.embedding, adapters);
|
|
600
|
+
}
|
|
601
|
+
return config;
|
|
602
|
+
}
|
|
603
|
+
function mapEmbedding(embedding, adapters) {
|
|
604
|
+
if (typeof embedding !== "object" || embedding === null) {
|
|
605
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding" must be an object');
|
|
606
|
+
}
|
|
607
|
+
if (typeof embedding.fields !== "object" || embedding.fields === null) {
|
|
608
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding.fields" is required');
|
|
609
|
+
}
|
|
610
|
+
const mapped = { fields: embedding.fields };
|
|
611
|
+
if (embedding.adapter !== void 0) {
|
|
612
|
+
const adapter = adapters[embedding.adapter];
|
|
613
|
+
if (!adapter) {
|
|
614
|
+
throw new NarsilError(
|
|
615
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
616
|
+
`Embedding adapter "${embedding.adapter}" is not registered on this server`,
|
|
617
|
+
{ available: Object.keys(adapters) }
|
|
618
|
+
);
|
|
619
|
+
}
|
|
620
|
+
mapped.adapter = adapter;
|
|
621
|
+
}
|
|
622
|
+
return mapped;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// src/server/handlers/indexes.ts
|
|
626
|
+
function createIndexHandlers(deps) {
|
|
627
|
+
const { engine, adapters } = deps;
|
|
628
|
+
async function create(ctx) {
|
|
629
|
+
const body = parseJson(ctx);
|
|
630
|
+
if (!body) return;
|
|
631
|
+
if (typeof body.name !== "string" || body.name.length === 0) {
|
|
632
|
+
badRequest(ctx.res, 'Field "name" is required and must be a non-empty string');
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
if (typeof body.config !== "object" || body.config === null) {
|
|
636
|
+
badRequest(ctx.res, 'Field "config" is required and must be an object');
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
try {
|
|
640
|
+
const config = mapHttpIndexConfig(body.config, adapters);
|
|
641
|
+
await engine.createIndex(body.name, config);
|
|
642
|
+
respondJson(ctx, { name: body.name }, 201);
|
|
643
|
+
} catch (err) {
|
|
644
|
+
respondError(ctx, err);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
function list(ctx) {
|
|
648
|
+
try {
|
|
649
|
+
respondJson(ctx, { indexes: engine.listIndexes() });
|
|
650
|
+
} catch (err) {
|
|
651
|
+
respondError(ctx, err);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
async function drop(ctx) {
|
|
655
|
+
try {
|
|
656
|
+
await engine.dropIndex(ctx.params[0]);
|
|
657
|
+
respondJson(ctx, { name: ctx.params[0], dropped: true });
|
|
658
|
+
} catch (err) {
|
|
659
|
+
respondError(ctx, err);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function stats(ctx) {
|
|
663
|
+
try {
|
|
664
|
+
respondJson(ctx, engine.getStats(ctx.params[0]));
|
|
665
|
+
} catch (err) {
|
|
666
|
+
respondError(ctx, err);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
function partitions(ctx) {
|
|
670
|
+
try {
|
|
671
|
+
respondJson(ctx, { partitions: engine.getPartitionStats(ctx.params[0]) });
|
|
672
|
+
} catch (err) {
|
|
673
|
+
respondError(ctx, err);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
async function clear(ctx) {
|
|
677
|
+
try {
|
|
678
|
+
await engine.clear(ctx.params[0]);
|
|
679
|
+
respondJson(ctx, { name: ctx.params[0], cleared: true });
|
|
680
|
+
} catch (err) {
|
|
681
|
+
respondError(ctx, err);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
return { create, list, drop, stats, partitions, clear };
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// src/server/handlers/search.ts
|
|
688
|
+
function rejectsCustomReducer(ctx, params) {
|
|
689
|
+
if (params.group && typeof params.group === "object" && "reduce" in params.group) {
|
|
690
|
+
badRequest(
|
|
691
|
+
ctx.res,
|
|
692
|
+
'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
|
|
693
|
+
);
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
698
|
+
function createSearchHandlers(deps) {
|
|
699
|
+
const { engine } = deps;
|
|
700
|
+
async function search(ctx) {
|
|
701
|
+
const params = parseJson(ctx);
|
|
702
|
+
if (!params) return;
|
|
703
|
+
if (rejectsCustomReducer(ctx, params)) return;
|
|
704
|
+
try {
|
|
705
|
+
respondJson(ctx, await engine.query(ctx.params[0], params));
|
|
706
|
+
} catch (err) {
|
|
707
|
+
respondError(ctx, err);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
async function preflight(ctx) {
|
|
711
|
+
const params = parseJson(ctx);
|
|
712
|
+
if (!params) return;
|
|
713
|
+
if (rejectsCustomReducer(ctx, params)) return;
|
|
714
|
+
try {
|
|
715
|
+
respondJson(ctx, await engine.preflight(ctx.params[0], params));
|
|
716
|
+
} catch (err) {
|
|
717
|
+
respondError(ctx, err);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
async function suggest(ctx) {
|
|
721
|
+
const params = parseJson(ctx);
|
|
722
|
+
if (!params) return;
|
|
723
|
+
if (typeof params.prefix !== "string") {
|
|
724
|
+
badRequest(ctx.res, 'Field "prefix" is required and must be a string');
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
try {
|
|
728
|
+
respondJson(ctx, await engine.suggest(ctx.params[0], params));
|
|
729
|
+
} catch (err) {
|
|
730
|
+
respondError(ctx, err);
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
return { search, preflight, suggest };
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// src/server/handlers/version.ts
|
|
737
|
+
function createVersionHandler(deps) {
|
|
738
|
+
function report(ctx) {
|
|
739
|
+
respondJson(ctx, { name: "narsil", ...deps.build });
|
|
740
|
+
}
|
|
741
|
+
return { report };
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// src/server/request.ts
|
|
745
|
+
function initAbortHandler(res) {
|
|
746
|
+
const listeners = [];
|
|
747
|
+
let aborted = false;
|
|
748
|
+
res.onAborted(() => {
|
|
749
|
+
aborted = true;
|
|
750
|
+
for (const fn of listeners) fn();
|
|
751
|
+
});
|
|
752
|
+
return {
|
|
753
|
+
get aborted() {
|
|
754
|
+
return aborted;
|
|
755
|
+
},
|
|
756
|
+
onAbort(fn) {
|
|
757
|
+
if (aborted) fn();
|
|
758
|
+
else listeners.push(fn);
|
|
759
|
+
}
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
function readBody(res, maxBytes, abort) {
|
|
763
|
+
return new Promise((resolve, reject) => {
|
|
764
|
+
if (abort.aborted) {
|
|
765
|
+
reject(new Error("aborted"));
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
let done = false;
|
|
769
|
+
let total = 0;
|
|
770
|
+
const chunks = [];
|
|
771
|
+
abort.onAbort(() => {
|
|
772
|
+
if (done) return;
|
|
773
|
+
done = true;
|
|
774
|
+
reject(new Error("aborted"));
|
|
775
|
+
});
|
|
776
|
+
res.onData((chunk, isLast) => {
|
|
777
|
+
if (done || abort.aborted) return;
|
|
778
|
+
total += chunk.byteLength;
|
|
779
|
+
if (total > maxBytes) {
|
|
780
|
+
done = true;
|
|
781
|
+
if (!abort.aborted) {
|
|
782
|
+
sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
|
|
783
|
+
}
|
|
784
|
+
reject(new Error("payload too large"));
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
chunks.push(Buffer.from(new Uint8Array(chunk)));
|
|
788
|
+
if (isLast) {
|
|
789
|
+
done = true;
|
|
790
|
+
resolve(Buffer.concat(chunks));
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
var ConcurrencyGate = class {
|
|
796
|
+
constructor(max) {
|
|
797
|
+
this.max = max;
|
|
798
|
+
}
|
|
799
|
+
inFlight = 0;
|
|
800
|
+
tryAcquire() {
|
|
801
|
+
if (this.max <= 0) return true;
|
|
802
|
+
if (this.inFlight >= this.max) return false;
|
|
803
|
+
this.inFlight++;
|
|
804
|
+
return true;
|
|
805
|
+
}
|
|
806
|
+
release() {
|
|
807
|
+
if (this.max <= 0) return;
|
|
808
|
+
if (this.inFlight > 0) this.inFlight--;
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
function isDenial(value) {
|
|
812
|
+
return typeof value === "object" && value !== null && "status" in value;
|
|
813
|
+
}
|
|
814
|
+
async function runHook(res, ctx, hook) {
|
|
815
|
+
try {
|
|
816
|
+
const result = await hook(ctx);
|
|
817
|
+
if (isDenial(result)) {
|
|
818
|
+
sendError(res, result.status, result.code, result.message);
|
|
819
|
+
return false;
|
|
820
|
+
}
|
|
821
|
+
return true;
|
|
822
|
+
} catch {
|
|
823
|
+
sendError(res, 500, ServerErrorCodes.HOOK_ERROR, "The onRequest hook threw an error");
|
|
824
|
+
return false;
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
function createRouteRunner(deps) {
|
|
828
|
+
const gate = new ConcurrencyGate(deps.maxConcurrentRequests);
|
|
829
|
+
const { onRequest, writeCors } = deps;
|
|
830
|
+
return (handler, opts) => {
|
|
831
|
+
const paramCount = opts.paramCount ?? 0;
|
|
832
|
+
const needsBody = opts.needsBody ?? false;
|
|
833
|
+
const useHook = Boolean(onRequest) && !opts.skipHooks;
|
|
834
|
+
const gated = !opts.skipHooks;
|
|
835
|
+
return (res, req) => {
|
|
836
|
+
const params = [];
|
|
837
|
+
for (let i = 0; i < paramCount; i++) params.push(req.getParameter(i) ?? "");
|
|
838
|
+
const query = new URLSearchParams(req.getQuery() ?? "");
|
|
839
|
+
const contentType = req.getHeader("content-type");
|
|
840
|
+
let hookCtx = null;
|
|
841
|
+
if (useHook) {
|
|
842
|
+
const headers = {};
|
|
843
|
+
req.forEach((key, value) => {
|
|
844
|
+
headers[key] = value;
|
|
845
|
+
});
|
|
846
|
+
hookCtx = {
|
|
847
|
+
method: req.getMethod(),
|
|
848
|
+
path: req.getUrl(),
|
|
849
|
+
headers,
|
|
850
|
+
remoteAddress: Buffer.from(res.getRemoteAddressAsText()).toString()
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
if (writeCors) writeCors(res, req);
|
|
854
|
+
const abort = initAbortHandler(res);
|
|
855
|
+
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort) : Promise.resolve(null);
|
|
856
|
+
const hookPromise = hookCtx && onRequest ? runHook(res, hookCtx, onRequest) : Promise.resolve(true);
|
|
857
|
+
Promise.all([bodyPromise, hookPromise]).then(async ([rawBody, allowed]) => {
|
|
858
|
+
if (abort.aborted || !allowed) return;
|
|
859
|
+
if (gated && !gate.tryAcquire()) {
|
|
860
|
+
sendError(res, 503, ServerErrorCodes.TOO_MANY_REQUESTS, "The server is at capacity; retry shortly");
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
try {
|
|
864
|
+
await handler({ res, params, query, contentType, rawBody, abort });
|
|
865
|
+
} catch (err) {
|
|
866
|
+
if (!abort.aborted) {
|
|
867
|
+
sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
868
|
+
}
|
|
869
|
+
} finally {
|
|
870
|
+
if (gated) gate.release();
|
|
871
|
+
}
|
|
872
|
+
}).catch(() => {
|
|
873
|
+
});
|
|
874
|
+
};
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// src/server/runtime.ts
|
|
879
|
+
var cached = null;
|
|
880
|
+
async function loadUWebSockets() {
|
|
881
|
+
if (cached) return cached;
|
|
882
|
+
try {
|
|
883
|
+
const mod = await import('uWebSockets.js');
|
|
884
|
+
const resolved = typeof mod.App === "function" ? mod : mod.default;
|
|
885
|
+
if (!resolved || typeof resolved.App !== "function") {
|
|
886
|
+
throw new Error("uWebSockets.js did not expose an App factory");
|
|
887
|
+
}
|
|
888
|
+
cached = resolved;
|
|
889
|
+
return resolved;
|
|
890
|
+
} catch (err) {
|
|
891
|
+
throw new NarsilError(
|
|
892
|
+
ErrorCodes.CONFIG_INVALID,
|
|
893
|
+
"The `uWebSockets.js` package is required to run the Narsil HTTP server. Install it with `pnpm add github:uNetworking/uWebSockets.js#v20.58.0` (or the npm or yarn equivalent).",
|
|
894
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
895
|
+
);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// src/server/task-store.ts
|
|
900
|
+
var DEFAULT_MAX_RETAINED = 1e3;
|
|
901
|
+
var InMemoryTaskStore = class {
|
|
902
|
+
constructor(maxRetained = DEFAULT_MAX_RETAINED) {
|
|
903
|
+
this.maxRetained = maxRetained;
|
|
904
|
+
}
|
|
905
|
+
records = /* @__PURE__ */ new Map();
|
|
906
|
+
async set(record) {
|
|
907
|
+
this.records.set(record.id, { ...record });
|
|
908
|
+
this.prune();
|
|
909
|
+
}
|
|
910
|
+
async get(id) {
|
|
911
|
+
const record = this.records.get(id);
|
|
912
|
+
return record ? { ...record } : null;
|
|
913
|
+
}
|
|
914
|
+
async list() {
|
|
915
|
+
return [...this.records.values()].map((record) => ({ ...record }));
|
|
916
|
+
}
|
|
917
|
+
async delete(id) {
|
|
918
|
+
this.records.delete(id);
|
|
919
|
+
}
|
|
920
|
+
prune() {
|
|
921
|
+
if (this.records.size <= this.maxRetained) return;
|
|
922
|
+
for (const [id, record] of this.records) {
|
|
923
|
+
if (this.records.size <= this.maxRetained) break;
|
|
924
|
+
if (record.status === "succeeded" || record.status === "failed") this.records.delete(id);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
};
|
|
928
|
+
var RUNNING_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
929
|
+
var TERMINAL_TTL_MS = 60 * 60 * 1e3;
|
|
930
|
+
var TaskRegistry = class {
|
|
931
|
+
constructor(store, instanceId) {
|
|
932
|
+
this.store = store;
|
|
933
|
+
this.instanceId = instanceId;
|
|
934
|
+
}
|
|
935
|
+
/** Records a task as running and drives `op` to completion in the background.
|
|
936
|
+
* Returns the record once it is persisted so the caller can respond 202. */
|
|
937
|
+
async start(type, indexName, op) {
|
|
938
|
+
const now = Date.now();
|
|
939
|
+
const record = {
|
|
940
|
+
id: randomUUID(),
|
|
941
|
+
type,
|
|
942
|
+
indexName,
|
|
943
|
+
owner: this.instanceId,
|
|
944
|
+
status: "running",
|
|
945
|
+
createdAt: now,
|
|
946
|
+
startedAt: now
|
|
947
|
+
};
|
|
948
|
+
await this.store.set(record, RUNNING_TTL_MS);
|
|
949
|
+
void this.drive(record, op);
|
|
950
|
+
return record;
|
|
951
|
+
}
|
|
952
|
+
get(id) {
|
|
953
|
+
return this.store.get(id);
|
|
954
|
+
}
|
|
955
|
+
list() {
|
|
956
|
+
return this.store.list();
|
|
957
|
+
}
|
|
958
|
+
/** Fails this instance's previously-running tasks after a restart. A task can
|
|
959
|
+
* only be advanced by the process that owns it, so its own running tasks from
|
|
960
|
+
* a prior life are dead and must not keep showing a stale running status. This
|
|
961
|
+
* is effective only with a stable instanceId; with the default random id it
|
|
962
|
+
* finds none of its own prior tasks, which is also correct for the in-memory
|
|
963
|
+
* default (which starts empty). */
|
|
964
|
+
async reconcile() {
|
|
965
|
+
let records;
|
|
966
|
+
try {
|
|
967
|
+
records = await this.store.list();
|
|
968
|
+
} catch {
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
for (const record of records) {
|
|
972
|
+
if (record.owner !== this.instanceId) continue;
|
|
973
|
+
if (record.status !== "running" && record.status !== "queued") continue;
|
|
974
|
+
const failed = {
|
|
975
|
+
...record,
|
|
976
|
+
status: "failed",
|
|
977
|
+
completedAt: Date.now(),
|
|
978
|
+
error: { code: "TASK_INTERRUPTED", message: "The server restarted while this task was running" }
|
|
979
|
+
};
|
|
980
|
+
try {
|
|
981
|
+
await this.store.set(failed, TERMINAL_TTL_MS);
|
|
982
|
+
} catch {
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
async drive(record, op) {
|
|
987
|
+
const result = { ...record };
|
|
988
|
+
try {
|
|
989
|
+
await op();
|
|
990
|
+
result.status = "succeeded";
|
|
991
|
+
} catch (err) {
|
|
992
|
+
result.status = "failed";
|
|
993
|
+
result.error = err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: "The operation failed" };
|
|
994
|
+
}
|
|
995
|
+
result.completedAt = Date.now();
|
|
996
|
+
try {
|
|
997
|
+
await this.store.set(result, TERMINAL_TTL_MS);
|
|
998
|
+
} catch {
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
};
|
|
1002
|
+
|
|
1003
|
+
// src/server/create-server.ts
|
|
1004
|
+
var MB = 1024 * 1024;
|
|
1005
|
+
function resolveLimits(limits) {
|
|
1006
|
+
return {
|
|
1007
|
+
maxBodyBytes: limits?.maxBodyBytes ?? 16 * MB,
|
|
1008
|
+
maxImportBytes: limits?.maxImportBytes ?? 100 * MB,
|
|
1009
|
+
maxLineBytes: limits?.maxLineBytes ?? 4 * MB,
|
|
1010
|
+
importBatchSize: limits?.importBatchSize ?? 1e3,
|
|
1011
|
+
maxConcurrentRequests: limits?.maxConcurrentRequests ?? 0
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
function resolveBuild(build) {
|
|
1015
|
+
return {
|
|
1016
|
+
version: build?.version ?? null,
|
|
1017
|
+
gitSha: build?.gitSha ?? null,
|
|
1018
|
+
dirty: build?.dirty ?? false
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
function isLoopback(host) {
|
|
1022
|
+
return host === "127.0.0.1" || host === "::1" || host === "localhost" || host.startsWith("127.");
|
|
1023
|
+
}
|
|
1024
|
+
function assertSecureBinding(host, options) {
|
|
1025
|
+
if (isLoopback(host) || options.onRequest || options.allowInsecure) return;
|
|
1026
|
+
throw new NarsilError(
|
|
1027
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1028
|
+
`Refusing to start: bound to a non-loopback address ("${host}") with no onRequest auth hook. The server exposes destructive admin endpoints (restore, drop, clear, rebalance, optimize). Resolve one of: (1) set options.onRequest to authenticate requests, (2) bind to 127.0.0.1, or (3) set options.allowInsecure: true if this address is on a trusted private network.`,
|
|
1029
|
+
{ host }
|
|
1030
|
+
);
|
|
1031
|
+
}
|
|
1032
|
+
var NarsilHttpServer = class {
|
|
1033
|
+
host;
|
|
1034
|
+
port;
|
|
1035
|
+
options;
|
|
1036
|
+
deps;
|
|
1037
|
+
uws = null;
|
|
1038
|
+
listenSocket = null;
|
|
1039
|
+
ready = false;
|
|
1040
|
+
constructor(engine, options = {}) {
|
|
1041
|
+
this.host = options.host ?? "127.0.0.1";
|
|
1042
|
+
this.port = options.port ?? 9876;
|
|
1043
|
+
this.options = options;
|
|
1044
|
+
const adapters = options.embeddingAdapters ?? {};
|
|
1045
|
+
const taskStore = options.taskStore ?? new InMemoryTaskStore();
|
|
1046
|
+
const instanceId = options.instanceId ?? randomUUID();
|
|
1047
|
+
this.deps = {
|
|
1048
|
+
engine,
|
|
1049
|
+
tasks: new TaskRegistry(taskStore, instanceId),
|
|
1050
|
+
adapters,
|
|
1051
|
+
limits: resolveLimits(options.limits),
|
|
1052
|
+
isReady: () => this.ready,
|
|
1053
|
+
build: resolveBuild(options.build)
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
async listen() {
|
|
1057
|
+
assertSecureBinding(this.host, this.options);
|
|
1058
|
+
const uws = await loadUWebSockets();
|
|
1059
|
+
this.uws = uws;
|
|
1060
|
+
const app = uws.App();
|
|
1061
|
+
this.registerRoutes(app);
|
|
1062
|
+
await new Promise((resolve, reject) => {
|
|
1063
|
+
app.listen(this.host, this.port, (socket) => {
|
|
1064
|
+
if (socket) {
|
|
1065
|
+
this.listenSocket = socket;
|
|
1066
|
+
resolve();
|
|
1067
|
+
} else {
|
|
1068
|
+
reject(new Error(`Failed to listen on ${this.host}:${this.port}`));
|
|
1069
|
+
}
|
|
1070
|
+
});
|
|
1071
|
+
});
|
|
1072
|
+
await this.deps.tasks.reconcile();
|
|
1073
|
+
this.ready = true;
|
|
1074
|
+
}
|
|
1075
|
+
async close() {
|
|
1076
|
+
this.ready = false;
|
|
1077
|
+
if (this.listenSocket && this.uws) {
|
|
1078
|
+
this.uws.us_listen_socket_close(this.listenSocket);
|
|
1079
|
+
this.listenSocket = null;
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
get listeningPort() {
|
|
1083
|
+
if (!this.listenSocket || !this.uws) return -1;
|
|
1084
|
+
return this.uws.us_socket_local_port(this.listenSocket);
|
|
1085
|
+
}
|
|
1086
|
+
registerRoutes(app) {
|
|
1087
|
+
const cors = resolveCors(this.options.cors);
|
|
1088
|
+
const run = createRouteRunner({
|
|
1089
|
+
onRequest: this.options.onRequest,
|
|
1090
|
+
maxConcurrentRequests: this.deps.limits.maxConcurrentRequests,
|
|
1091
|
+
writeCors: cors ? corsWriter(cors) : void 0
|
|
1092
|
+
});
|
|
1093
|
+
const { maxBodyBytes, maxImportBytes } = this.deps.limits;
|
|
1094
|
+
const idx = createIndexHandlers(this.deps);
|
|
1095
|
+
const doc = createDocumentHandlers(this.deps);
|
|
1096
|
+
const search = createSearchHandlers(this.deps);
|
|
1097
|
+
const admin = createAdminHandlers(this.deps);
|
|
1098
|
+
const health = createHealthHandlers(this.deps);
|
|
1099
|
+
const version = createVersionHandler(this.deps);
|
|
1100
|
+
const importNdjson = createImportHandler(this.deps);
|
|
1101
|
+
const json = (handler, opts) => run(handler, opts);
|
|
1102
|
+
if (cors) {
|
|
1103
|
+
app.options("/*", (res, req) => {
|
|
1104
|
+
const origin = req.getHeader("origin");
|
|
1105
|
+
res.cork(() => {
|
|
1106
|
+
res.writeStatus("204 No Content");
|
|
1107
|
+
writeCorsOrigin(res, cors, origin);
|
|
1108
|
+
res.writeHeader("Access-Control-Allow-Methods", cors.methods).writeHeader("Access-Control-Allow-Headers", cors.headers).writeHeader("Access-Control-Max-Age", "86400").endWithoutBody();
|
|
1109
|
+
});
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
app.get("/livez", json(health.livez, { maxBytes: 0, skipHooks: true }));
|
|
1113
|
+
app.get("/readyz", json(health.readyz, { maxBytes: 0, skipHooks: true }));
|
|
1114
|
+
app.get("/health", json(health.livez, { maxBytes: 0, skipHooks: true }));
|
|
1115
|
+
app.get("/version", json(version.report, { maxBytes: 0, skipHooks: true }));
|
|
1116
|
+
app.get("/stats/memory", json(admin.memory, { maxBytes: 0 }));
|
|
1117
|
+
app.get("/tasks", json(admin.listTasks, { maxBytes: 0 }));
|
|
1118
|
+
app.get("/tasks/:id", json(admin.getTask, { paramCount: 1, maxBytes: 0 }));
|
|
1119
|
+
app.post("/indexes", json(idx.create, { needsBody: true, maxBytes: maxBodyBytes }));
|
|
1120
|
+
app.get("/indexes", json(idx.list, { maxBytes: 0 }));
|
|
1121
|
+
app.del("/indexes/:name", json(idx.drop, { paramCount: 1, maxBytes: 0 }));
|
|
1122
|
+
app.get("/indexes/:name/stats", json(idx.stats, { paramCount: 1, maxBytes: 0 }));
|
|
1123
|
+
app.get("/indexes/:name/partitions", json(idx.partitions, { paramCount: 1, maxBytes: 0 }));
|
|
1124
|
+
app.post("/indexes/:name/_clear", json(idx.clear, { paramCount: 1, maxBytes: 0 }));
|
|
1125
|
+
app.get("/indexes/:name/count", json(doc.count, { paramCount: 1, maxBytes: 0 }));
|
|
1126
|
+
app.post("/indexes/:name/documents", json(doc.insert, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1127
|
+
app.post(
|
|
1128
|
+
"/indexes/:name/documents/_batch",
|
|
1129
|
+
json(doc.batch, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1130
|
+
);
|
|
1131
|
+
app.post(
|
|
1132
|
+
"/indexes/:name/documents/_multi-get",
|
|
1133
|
+
json(doc.multiGet, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1134
|
+
);
|
|
1135
|
+
app.post(
|
|
1136
|
+
"/indexes/:name/documents/_import",
|
|
1137
|
+
json(importNdjson, { paramCount: 1, needsBody: true, maxBytes: maxImportBytes })
|
|
1138
|
+
);
|
|
1139
|
+
app.get("/indexes/:name/documents/:id", json(doc.get, { paramCount: 2, maxBytes: 0 }));
|
|
1140
|
+
app.get("/indexes/:name/documents/:id/_exists", json(doc.exists, { paramCount: 2, maxBytes: 0 }));
|
|
1141
|
+
app.put("/indexes/:name/documents/:id", json(doc.put, { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1142
|
+
app.patch(
|
|
1143
|
+
"/indexes/:name/documents/:id",
|
|
1144
|
+
json(doc.patch, { paramCount: 2, needsBody: true, maxBytes: maxBodyBytes })
|
|
1145
|
+
);
|
|
1146
|
+
app.del("/indexes/:name/documents/:id", json(doc.remove, { paramCount: 2, maxBytes: 0 }));
|
|
1147
|
+
app.post("/indexes/:name/search", json(search.search, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1148
|
+
app.post(
|
|
1149
|
+
"/indexes/:name/search/preflight",
|
|
1150
|
+
json(search.preflight, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1151
|
+
);
|
|
1152
|
+
app.post("/indexes/:name/suggest", json(search.suggest, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes }));
|
|
1153
|
+
app.post("/indexes/:name/_checkpoint", json(admin.checkpoint, { paramCount: 1, maxBytes: 0 }));
|
|
1154
|
+
app.get("/indexes/:name/snapshot", json(admin.snapshot, { paramCount: 1, maxBytes: 0 }));
|
|
1155
|
+
app.get("/indexes/:name/vector-maintenance", json(admin.vectorMaintenance, { paramCount: 1, maxBytes: 0 }));
|
|
1156
|
+
app.post(
|
|
1157
|
+
"/indexes/:name/vectors/_compact",
|
|
1158
|
+
json(admin.compact, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1159
|
+
);
|
|
1160
|
+
app.post(
|
|
1161
|
+
"/indexes/:name/vectors/_optimize",
|
|
1162
|
+
json(admin.optimize, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1163
|
+
);
|
|
1164
|
+
app.post(
|
|
1165
|
+
"/indexes/:name/_rebalance",
|
|
1166
|
+
json(admin.rebalance, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1167
|
+
);
|
|
1168
|
+
app.post(
|
|
1169
|
+
"/indexes/:name/partition-config",
|
|
1170
|
+
json(admin.partitionConfig, { paramCount: 1, needsBody: true, maxBytes: maxBodyBytes })
|
|
1171
|
+
);
|
|
1172
|
+
app.post(
|
|
1173
|
+
"/indexes/:name/restore",
|
|
1174
|
+
json(admin.restore, { paramCount: 1, needsBody: true, maxBytes: maxImportBytes })
|
|
1175
|
+
);
|
|
1176
|
+
app.any("/*", (res) => {
|
|
1177
|
+
sendError(res, 404, ServerErrorCodes.NOT_FOUND, "Route not found");
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
function createServer(engine, options) {
|
|
1182
|
+
return new NarsilHttpServer(engine, options);
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
export { InMemoryTaskStore, ServerErrorCodes, createServer, httpStatusForNarsilError };
|