opencode-rag-plugin 1.19.4 → 1.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker/base.js +19 -5
- package/dist/chunker/factory.js +27 -9
- package/dist/chunker/grammar.d.ts +18 -1
- package/dist/chunker/grammar.js +48 -10
- package/dist/chunker/pdf.js +30 -14
- package/dist/cli/commands/init-helpers.js +15 -2
- package/dist/cli/commands/init.js +31 -21
- package/dist/cli/commands/quirk.js +9 -3
- package/dist/cli/commands/setup.js +5 -2
- package/dist/cli/commands/status.js +14 -5
- package/dist/cli/commands/ui.js +23 -9
- package/dist/cli/commands/update.js +4 -5
- package/dist/cli/format.d.ts +5 -2
- package/dist/cli/format.js +14 -5
- package/dist/content/image.js +33 -11
- package/dist/content/reader.js +74 -13
- package/dist/core/bootstrap.js +10 -3
- package/dist/core/config.js +31 -0
- package/dist/core/desc-cache.d.ts +8 -2
- package/dist/core/desc-cache.js +10 -3
- package/dist/core/doc-progress.js +5 -2
- package/dist/core/provider-defaults.d.ts +2 -0
- package/dist/core/provider-defaults.js +19 -4
- package/dist/core/runtime-overrides.d.ts +0 -6
- package/dist/core/version-check.d.ts +5 -0
- package/dist/core/version-check.js +8 -2
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +19 -5
- package/dist/describer/describer.js +15 -2
- package/dist/describer/gemini.js +25 -10
- package/dist/embedder/factory.d.ts +5 -3
- package/dist/embedder/factory.js +41 -8
- package/dist/embedder/health.js +19 -19
- package/dist/embedder/http.d.ts +14 -1
- package/dist/embedder/http.js +60 -6
- package/dist/eval/session-logger.js +7 -0
- package/dist/eval/storage.js +8 -0
- package/dist/indexer/git-diff.d.ts +1 -1
- package/dist/indexer/git-diff.js +5 -1
- package/dist/indexer/pipeline.js +421 -344
- package/dist/indexer/stats.d.ts +2 -0
- package/dist/indexer/stats.js +1 -0
- package/dist/indexer/watch.js +8 -1
- package/dist/indexer/worker.js +21 -0
- package/dist/mcp/cli.js +4 -0
- package/dist/mcp/handlers.js +16 -5
- package/dist/mcp/server.js +3 -0
- package/dist/opencode/create-read-tool.js +14 -3
- package/dist/opencode/tool-args.js +23 -1
- package/dist/plugin.js +66 -152
- package/dist/quirks/auto-capture.js +5 -0
- package/dist/quirks/quirk-store.d.ts +1 -1
- package/dist/quirks/quirk-store.js +56 -17
- package/dist/retriever/context-optimizer.js +18 -4
- package/dist/retriever/keyword-index.d.ts +2 -0
- package/dist/retriever/keyword-index.js +38 -4
- package/dist/retriever/retriever.js +6 -1
- package/dist/tui.js +41 -4
- package/dist/vectorstore/lancedb.d.ts +25 -1
- package/dist/vectorstore/lancedb.js +157 -11
- package/dist/vectorstore/memory.js +5 -1
- package/dist/watcher.js +30 -4
- package/dist/web/api.d.ts +5 -1
- package/dist/web/api.js +195 -69
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +66 -28
- package/dist/web/static.d.ts +5 -2
- package/dist/web/static.js +9 -5
- package/dist/web/ui/assets/index-BDPYdtA1.js +3 -0
- package/dist/web/ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/ui/assets/index-CJBvt6e0.js +0 -3
package/dist/web/api.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
2
|
import { extname, join, resolve as resolvePathModule } from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { listSessions, getSession, deleteSession, compareSessions, validateSessionID } from "../eval/storage.js";
|
|
@@ -30,14 +30,53 @@ function parseQuery(url) {
|
|
|
30
30
|
params: new URLSearchParams(queryString ?? ""),
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Determine whether a browser Origin header may call the API.
|
|
35
|
+
*
|
|
36
|
+
* Only same-machine origins are allowed (the server binds to 127.0.0.1). Any
|
|
37
|
+
* other origin — e.g. a random website doing a drive-by fetch — is rejected.
|
|
38
|
+
* Returns the origin to echo in `Access-Control-Allow-Origin`, or `null`.
|
|
39
|
+
*/
|
|
40
|
+
function isAllowedOrigin(origin) {
|
|
41
|
+
if (!origin)
|
|
42
|
+
return null;
|
|
43
|
+
try {
|
|
44
|
+
const u = new URL(origin);
|
|
45
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
46
|
+
return null;
|
|
47
|
+
return u.hostname === "127.0.0.1" || u.hostname === "localhost" || u.hostname === "[::1]" ? origin : null;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Compare two tokens in constant time to avoid timing side-channels. */
|
|
54
|
+
function tokensEqual(a, b) {
|
|
55
|
+
if (a.length !== b.length)
|
|
56
|
+
return false;
|
|
57
|
+
let diff = 0;
|
|
58
|
+
for (let i = 0; i < a.length; i++)
|
|
59
|
+
diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
60
|
+
return diff === 0;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Serialise an {@link ApiResponse} as JSON and write it to the HTTP response.
|
|
64
|
+
*
|
|
65
|
+
* CORS headers are only emitted for allowed (localhost) origins; cross-origin
|
|
66
|
+
* callers get no CORS headers at all, so browsers block reading the response.
|
|
67
|
+
*/
|
|
68
|
+
function sendJson(res, response, origin) {
|
|
69
|
+
const headers = {
|
|
36
70
|
"Content-Type": "application/json",
|
|
37
|
-
"Access-Control-Allow-Origin": "*",
|
|
38
71
|
"Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
|
|
39
|
-
"Access-Control-Allow-Headers": "Content-Type",
|
|
40
|
-
|
|
72
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
73
|
+
"X-Content-Type-Options": "nosniff",
|
|
74
|
+
};
|
|
75
|
+
if (origin) {
|
|
76
|
+
headers["Access-Control-Allow-Origin"] = origin;
|
|
77
|
+
headers["Vary"] = "Origin";
|
|
78
|
+
}
|
|
79
|
+
res.writeHead(response.status, headers);
|
|
41
80
|
res.end(JSON.stringify(response.body));
|
|
42
81
|
}
|
|
43
82
|
/**
|
|
@@ -57,11 +96,43 @@ function sendJson(res, response) {
|
|
|
57
96
|
* @param cfg - Active RAG configuration (used by quirk endpoints).
|
|
58
97
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
59
98
|
*/
|
|
60
|
-
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder) {
|
|
99
|
+
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder, token) {
|
|
61
100
|
return async (req, res) => {
|
|
62
101
|
const url = req.url ?? "/";
|
|
63
102
|
const method = req.method ?? "GET";
|
|
64
103
|
const { path, params } = parseQuery(url);
|
|
104
|
+
const origin = isAllowedOrigin(req.headers.origin);
|
|
105
|
+
// CORS preflight — only answer for allowed localhost origins
|
|
106
|
+
if (method === "OPTIONS") {
|
|
107
|
+
if (!origin) {
|
|
108
|
+
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
109
|
+
res.end("Forbidden");
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
res.writeHead(204, {
|
|
113
|
+
"Access-Control-Allow-Origin": origin,
|
|
114
|
+
"Vary": "Origin",
|
|
115
|
+
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
|
116
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
117
|
+
});
|
|
118
|
+
res.end();
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
// Same-machine-only: reject cross-origin requests outright (defeats drive-by
|
|
122
|
+
// website fetches and DNS rebinding even where CORS headers would block reads).
|
|
123
|
+
if (origin && !isAllowedOrigin(origin)) {
|
|
124
|
+
sendJson(res, { status: 403, body: { error: "Forbidden origin" } }, null);
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
// Token auth for every API request when a token is configured
|
|
128
|
+
if (token) {
|
|
129
|
+
const header = req.headers.authorization;
|
|
130
|
+
const supplied = header?.startsWith("Bearer ") ? header.slice("Bearer ".length) : params.get("token");
|
|
131
|
+
if (!supplied || !tokensEqual(supplied, token)) {
|
|
132
|
+
sendJson(res, { status: 401, body: { error: "Unauthorized — missing or invalid token" } }, origin);
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
65
136
|
// Quirk store dependencies (embedder is a no-op stub for the read-only UI context).
|
|
66
137
|
const quirkDeps = {
|
|
67
138
|
embedder: stubEmbedder,
|
|
@@ -70,16 +141,6 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
70
141
|
cfg: cfg ?? {},
|
|
71
142
|
storePath,
|
|
72
143
|
};
|
|
73
|
-
// CORS preflight
|
|
74
|
-
if (method === "OPTIONS") {
|
|
75
|
-
res.writeHead(204, {
|
|
76
|
-
"Access-Control-Allow-Origin": "*",
|
|
77
|
-
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
|
78
|
-
"Access-Control-Allow-Headers": "Content-Type",
|
|
79
|
-
});
|
|
80
|
-
res.end();
|
|
81
|
-
return true;
|
|
82
|
-
}
|
|
83
144
|
let response;
|
|
84
145
|
try {
|
|
85
146
|
// Existing endpoints
|
|
@@ -155,6 +216,14 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
155
216
|
else if (path === "/api/eval/sessions" && method === "GET") {
|
|
156
217
|
response = await handleEvalSessions(storePath);
|
|
157
218
|
}
|
|
219
|
+
else if (path === "/api/eval/compare" && method === "GET") {
|
|
220
|
+
response = await handleEvalCompare(storePath, params);
|
|
221
|
+
}
|
|
222
|
+
// Token analysis endpoints — must precede the generic `/api/eval/sessions/:id` route
|
|
223
|
+
else if (path.startsWith("/api/eval/sessions/") && path.endsWith("/analysis") && method === "GET") {
|
|
224
|
+
const id = path.slice("/api/eval/sessions/".length, -"/analysis".length);
|
|
225
|
+
response = await handleEvalAnalysis(storePath, id);
|
|
226
|
+
}
|
|
158
227
|
else if (path.startsWith("/api/eval/sessions/") && method === "GET") {
|
|
159
228
|
const id = path.slice("/api/eval/sessions/".length);
|
|
160
229
|
response = await handleEvalSession(storePath, id);
|
|
@@ -163,14 +232,6 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
163
232
|
const id = path.slice("/api/eval/sessions/".length);
|
|
164
233
|
response = await handleEvalDeleteSession(storePath, id);
|
|
165
234
|
}
|
|
166
|
-
else if (path === "/api/eval/compare" && method === "GET") {
|
|
167
|
-
response = await handleEvalCompare(storePath, params);
|
|
168
|
-
}
|
|
169
|
-
// Token analysis endpoints
|
|
170
|
-
else if (path.startsWith("/api/eval/sessions/") && path.endsWith("/analysis") && method === "GET") {
|
|
171
|
-
const id = path.slice("/api/eval/sessions/".length, -"/analysis".length);
|
|
172
|
-
response = await handleEvalAnalysis(storePath, id);
|
|
173
|
-
}
|
|
174
235
|
else if (path === "/api/eval/token-compare" && method === "GET") {
|
|
175
236
|
response = await handleEvalTokenCompare(storePath, params);
|
|
176
237
|
}
|
|
@@ -197,12 +258,20 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
197
258
|
else {
|
|
198
259
|
return false;
|
|
199
260
|
}
|
|
200
|
-
sendJson(res, response);
|
|
261
|
+
sendJson(res, response, origin);
|
|
201
262
|
return true;
|
|
202
263
|
}
|
|
203
264
|
catch (err) {
|
|
265
|
+
if (err instanceof BodyTooLargeError) {
|
|
266
|
+
if (!res.destroyed) {
|
|
267
|
+
sendJson(res, { status: 413, body: { error: err.message } }, origin);
|
|
268
|
+
}
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
204
271
|
const message = err instanceof Error ? err.message : String(err);
|
|
205
|
-
|
|
272
|
+
if (!res.destroyed) {
|
|
273
|
+
sendJson(res, { status: 500, body: { error: message } }, origin);
|
|
274
|
+
}
|
|
206
275
|
return true;
|
|
207
276
|
}
|
|
208
277
|
};
|
|
@@ -260,23 +329,19 @@ async function handleQuirkDelete(deps, id) {
|
|
|
260
329
|
/**
|
|
261
330
|
* Respond with a paginated, optionally filtered list of chunks.
|
|
262
331
|
*
|
|
263
|
-
* Query params: `offset` (default 0), `limit` (default 50
|
|
332
|
+
* Query params: `offset` (default 0, clamped >= 0), `limit` (default 50,
|
|
333
|
+
* clamped 1..500), `lang`, `file`. Filtering and pagination are pushed
|
|
334
|
+
* down into the store query — loading 100k rows per request was a memory
|
|
335
|
+
* blowup on large stores.
|
|
264
336
|
*/
|
|
265
337
|
async function handleChunks(store, params) {
|
|
266
|
-
const
|
|
267
|
-
const
|
|
338
|
+
const rawOffset = parseInt(params.get("offset") ?? "0", 10);
|
|
339
|
+
const rawLimit = parseInt(params.get("limit") ?? "50", 10);
|
|
340
|
+
const offset = Number.isFinite(rawOffset) ? Math.max(0, rawOffset) : 0;
|
|
341
|
+
const limit = Number.isFinite(rawLimit) ? Math.min(500, Math.max(1, rawLimit)) : 50;
|
|
268
342
|
const langFilter = params.get("lang");
|
|
269
343
|
const fileFilter = params.get("file");
|
|
270
|
-
const
|
|
271
|
-
let filtered = allChunks;
|
|
272
|
-
if (langFilter) {
|
|
273
|
-
filtered = filtered.filter((c) => c.language === langFilter);
|
|
274
|
-
}
|
|
275
|
-
if (fileFilter) {
|
|
276
|
-
filtered = filtered.filter((c) => c.filePath.startsWith(fileFilter));
|
|
277
|
-
}
|
|
278
|
-
const total = filtered.length;
|
|
279
|
-
const chunks = filtered.slice(offset, offset + limit);
|
|
344
|
+
const { chunks, total } = await store.getChunksFiltered(offset, limit, langFilter || undefined, fileFilter || undefined);
|
|
280
345
|
return {
|
|
281
346
|
status: 200,
|
|
282
347
|
body: { chunks, total, offset, limit },
|
|
@@ -284,8 +349,7 @@ async function handleChunks(store, params) {
|
|
|
284
349
|
}
|
|
285
350
|
/** Respond with a single chunk identified by its ID, or 404 if not found. */
|
|
286
351
|
async function handleChunkById(store, id) {
|
|
287
|
-
const
|
|
288
|
-
const chunk = chunks.find((c) => c.id === id);
|
|
352
|
+
const chunk = await store.getChunkById(id);
|
|
289
353
|
if (!chunk) {
|
|
290
354
|
return { status: 404, body: { error: "Chunk not found" } };
|
|
291
355
|
}
|
|
@@ -294,7 +358,8 @@ async function handleChunkById(store, id) {
|
|
|
294
358
|
/** Run a keyword search against the index and return ranked results. Query param: `q` (query string), `topK` (default 20). */
|
|
295
359
|
async function handleSearch(keywordIndex, params) {
|
|
296
360
|
const query = params.get("q") ?? "";
|
|
297
|
-
const
|
|
361
|
+
const rawTopK = parseInt(params.get("topK") ?? "20", 10);
|
|
362
|
+
const topK = Number.isFinite(rawTopK) ? Math.min(100, Math.max(1, rawTopK)) : 20;
|
|
298
363
|
if (!query.trim()) {
|
|
299
364
|
return { status: 200, body: { results: [] } };
|
|
300
365
|
}
|
|
@@ -337,9 +402,14 @@ async function handleRetrieve(store, keywordIndex, getEmbedder, cfg, params) {
|
|
|
337
402
|
catch (err) {
|
|
338
403
|
return { status: 503, body: { error: `Embedding model unavailable: ${err.message}. Check that your embedding provider is running.` } };
|
|
339
404
|
}
|
|
340
|
-
const
|
|
341
|
-
const
|
|
342
|
-
const
|
|
405
|
+
const rawTopK = parseInt(params.get("topK") ?? "10", 10);
|
|
406
|
+
const rawMinScore = parseFloat(params.get("minScore") ?? "0.35");
|
|
407
|
+
const rawKeywordWeight = parseFloat(params.get("keywordWeight") ?? "0.4");
|
|
408
|
+
// Clamp all numeric params — a topK of 1e9 would overfetch 3x via the
|
|
409
|
+
// retriever's overfetch factor and blow up memory.
|
|
410
|
+
const topK = Number.isFinite(rawTopK) ? Math.min(100, Math.max(1, rawTopK)) : 10;
|
|
411
|
+
const minScore = Number.isFinite(rawMinScore) ? Math.min(1, Math.max(0, rawMinScore)) : 0.35;
|
|
412
|
+
const keywordWeight = Number.isFinite(rawKeywordWeight) ? Math.min(1, Math.max(0, rawKeywordWeight)) : 0.4;
|
|
343
413
|
const hybrid = params.get("hybrid") !== "false";
|
|
344
414
|
const explain = params.get("explain") !== "false";
|
|
345
415
|
const pathFilter = params.get("path") ?? undefined;
|
|
@@ -404,10 +474,15 @@ async function handleCompare(store, params) {
|
|
|
404
474
|
if (ids.length === 0) {
|
|
405
475
|
return { status: 400, body: { error: "No chunk IDs provided" } };
|
|
406
476
|
}
|
|
407
|
-
|
|
408
|
-
|
|
477
|
+
if (ids.length > 100) {
|
|
478
|
+
return { status: 400, body: { error: "Too many chunk IDs (max 100)" } };
|
|
479
|
+
}
|
|
480
|
+
const chunks = await store.getChunksByIds(ids);
|
|
409
481
|
return { status: 200, body: { chunks } };
|
|
410
482
|
}
|
|
483
|
+
// Re-hashing every manifest file per status request is expensive — cache the
|
|
484
|
+
// result keyed on the manifest file's mtime+size (invalidated on any write).
|
|
485
|
+
let statusCache = null;
|
|
411
486
|
/**
|
|
412
487
|
* Return indexing status — manifest stats, staleness, and a placeholder for watcher state.
|
|
413
488
|
*/
|
|
@@ -431,20 +506,37 @@ async function handleIndexingStatus(storePath, cwd) {
|
|
|
431
506
|
if (manifest?.lastIndexedAt) {
|
|
432
507
|
lastIndexedAt = new Date(manifest.lastIndexedAt).toISOString();
|
|
433
508
|
}
|
|
434
|
-
// Count stale files by comparing manifest file list against current disk state
|
|
509
|
+
// Count stale files by comparing manifest file list against current disk state.
|
|
510
|
+
// Cached by manifest mtime+size so the poll loop doesn't hash every file
|
|
511
|
+
// synchronously on the event loop per request.
|
|
435
512
|
if (cwd && manifest?.files) {
|
|
436
|
-
const
|
|
437
|
-
for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
|
|
513
|
+
const cacheKey = (() => {
|
|
438
514
|
try {
|
|
439
|
-
const
|
|
440
|
-
|
|
441
|
-
const hash = createHash("sha256").update(content).digest("hex");
|
|
442
|
-
if (hash !== fileMeta.hash)
|
|
443
|
-
staleFileCount++;
|
|
515
|
+
const st = statSync(manifestPath);
|
|
516
|
+
return `${st.mtimeMs}:${st.size}`;
|
|
444
517
|
}
|
|
445
518
|
catch {
|
|
446
|
-
|
|
519
|
+
return "missing";
|
|
447
520
|
}
|
|
521
|
+
})();
|
|
522
|
+
if (statusCache && statusCache.key === cacheKey) {
|
|
523
|
+
staleFileCount = statusCache.staleFileCount;
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
const storedFiles = manifest.files;
|
|
527
|
+
for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
|
|
528
|
+
try {
|
|
529
|
+
const fullPath = filePath;
|
|
530
|
+
const content = readFileSync(fullPath, "utf-8");
|
|
531
|
+
const hash = createHash("sha256").update(content).digest("hex");
|
|
532
|
+
if (hash !== fileMeta.hash)
|
|
533
|
+
staleFileCount++;
|
|
534
|
+
}
|
|
535
|
+
catch {
|
|
536
|
+
staleFileCount++; // file was deleted or unreadable
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
statusCache = { key: cacheKey, staleFileCount, manifest };
|
|
448
540
|
}
|
|
449
541
|
}
|
|
450
542
|
return {
|
|
@@ -456,22 +548,38 @@ async function handleIndexingStatus(storePath, cwd) {
|
|
|
456
548
|
},
|
|
457
549
|
};
|
|
458
550
|
}
|
|
551
|
+
/** Guards concurrent reindex requests — only one pass may run at a time. */
|
|
552
|
+
let reindexInFlight = false;
|
|
459
553
|
/**
|
|
460
554
|
* Trigger a one-shot reindex pass in the background.
|
|
461
555
|
*/
|
|
462
556
|
async function handleReindex(cwd, cfg, storePath, store, getEmbedder) {
|
|
557
|
+
if (reindexInFlight) {
|
|
558
|
+
return { status: 409, body: { error: "A reindex is already running" } };
|
|
559
|
+
}
|
|
463
560
|
try {
|
|
464
561
|
const { runIndexPass } = await import("../indexer.js");
|
|
465
562
|
const embedder = getEmbedder ? await getEmbedder() : undefined;
|
|
466
563
|
if (!embedder) {
|
|
467
564
|
return { status: 503, body: { error: "Embedder not available" } };
|
|
468
565
|
}
|
|
469
|
-
|
|
566
|
+
reindexInFlight = true;
|
|
567
|
+
runIndexPass({ cwd, storePath, config: cfg, store, embedder })
|
|
568
|
+
.then(() => {
|
|
569
|
+
reindexInFlight = false;
|
|
570
|
+
statusCache = null; // invalidate the status cache after a pass
|
|
571
|
+
projectionCache = null;
|
|
572
|
+
})
|
|
573
|
+
.catch((err) => {
|
|
574
|
+
reindexInFlight = false;
|
|
575
|
+
statusCache = null;
|
|
576
|
+
projectionCache = null;
|
|
470
577
|
console.error("Background reindex failed:", err);
|
|
471
578
|
});
|
|
472
579
|
return { status: 200, body: { started: true } };
|
|
473
580
|
}
|
|
474
581
|
catch (err) {
|
|
582
|
+
reindexInFlight = false;
|
|
475
583
|
return { status: 500, body: { error: `Failed to start reindex: ${err.message}` } };
|
|
476
584
|
}
|
|
477
585
|
}
|
|
@@ -485,7 +593,7 @@ function handleConfig(cfg) {
|
|
|
485
593
|
}
|
|
486
594
|
function redactKeys(obj) {
|
|
487
595
|
for (const key of Object.keys(obj)) {
|
|
488
|
-
if (key.
|
|
596
|
+
if (/api\s*key|apikey|password|passwd|secret|token|authorization|credential/i.test(key)) {
|
|
489
597
|
obj[key] = "***";
|
|
490
598
|
}
|
|
491
599
|
else if (typeof obj[key] === "object" && obj[key] !== null) {
|
|
@@ -495,16 +603,28 @@ function redactKeys(obj) {
|
|
|
495
603
|
}
|
|
496
604
|
/**
|
|
497
605
|
* Project chunk embeddings to 2D via PCA for the Embedding Space Explorer.
|
|
606
|
+
* Capped at 5000 chunks and memoized per (storePath, maxChunks) so the
|
|
607
|
+
* O(n·dim²) computation does not run on every visit.
|
|
498
608
|
*/
|
|
609
|
+
let projectionCache = null;
|
|
499
610
|
async function handleEmbeddingProjection(store, params) {
|
|
500
|
-
const
|
|
611
|
+
const rawMaxChunks = parseInt(params.get("maxChunks") ?? "5000", 10);
|
|
612
|
+
const maxChunks = Number.isFinite(rawMaxChunks) ? Math.min(5000, Math.max(1, rawMaxChunks)) : 5000;
|
|
501
613
|
try {
|
|
614
|
+
// Invalidated after a reindex pass completes (see handleReindex)
|
|
615
|
+
const cacheKey = `${maxChunks}`;
|
|
616
|
+
if (projectionCache && projectionCache.key === cacheKey) {
|
|
617
|
+
return { status: 200, body: projectionCache.body };
|
|
618
|
+
}
|
|
502
619
|
const chunks = await store.getChunksWithEmbeddings(maxChunks);
|
|
503
620
|
if (chunks.length === 0) {
|
|
504
|
-
|
|
621
|
+
projectionCache = { key: cacheKey, body: { points: [], totalChunks: 0 } };
|
|
622
|
+
return { status: 200, body: projectionCache.body };
|
|
505
623
|
}
|
|
506
624
|
if (chunks.length === 1) {
|
|
507
|
-
|
|
625
|
+
const body = { points: [{ id: chunks[0].id, x: 0.5, y: 0.5, filePath: chunks[0].filePath, startLine: chunks[0].startLine, endLine: chunks[0].endLine, language: chunks[0].language, description: chunks[0].description }], totalChunks: 1, displayedChunks: 1 };
|
|
626
|
+
projectionCache = { key: cacheKey, body };
|
|
627
|
+
return { status: 200, body };
|
|
508
628
|
}
|
|
509
629
|
const { computePCA } = await import("./pca.js");
|
|
510
630
|
const vectors = chunks.map(c => c.embedding);
|
|
@@ -519,10 +639,9 @@ async function handleEmbeddingProjection(store, params) {
|
|
|
519
639
|
language: c.language,
|
|
520
640
|
description: c.description,
|
|
521
641
|
}));
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
};
|
|
642
|
+
const body = { points, totalChunks: chunks.length, displayedChunks: points.length };
|
|
643
|
+
projectionCache = { key: cacheKey, body };
|
|
644
|
+
return { status: 200, body };
|
|
526
645
|
}
|
|
527
646
|
catch (err) {
|
|
528
647
|
return { status: 500, body: { error: `Projection failed: ${err.message}` } };
|
|
@@ -654,6 +773,13 @@ export function handleEvalProjectSavings(body) {
|
|
|
654
773
|
}
|
|
655
774
|
/** Collect the full request body as a Buffer and parse it as JSON. Returns `{}` on empty or invalid input. */
|
|
656
775
|
const MAX_BODY_BYTES = 1_048_576; // 1 MB
|
|
776
|
+
/** Thrown when the request body exceeds {@link MAX_BODY_BYTES}; mapped to a 413 response. */
|
|
777
|
+
export class BodyTooLargeError extends Error {
|
|
778
|
+
constructor() {
|
|
779
|
+
super(`Request body exceeds ${MAX_BODY_BYTES} byte limit`);
|
|
780
|
+
this.name = "BodyTooLargeError";
|
|
781
|
+
}
|
|
782
|
+
}
|
|
657
783
|
async function readBody(req) {
|
|
658
784
|
const chunks = [];
|
|
659
785
|
let totalSize = 0;
|
|
@@ -661,8 +787,8 @@ async function readBody(req) {
|
|
|
661
787
|
const buf = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
662
788
|
totalSize += buf.length;
|
|
663
789
|
if (totalSize > MAX_BODY_BYTES) {
|
|
664
|
-
req.destroy(
|
|
665
|
-
throw new
|
|
790
|
+
req.destroy();
|
|
791
|
+
throw new BodyTooLargeError();
|
|
666
792
|
}
|
|
667
793
|
chunks.push(buf);
|
|
668
794
|
}
|
package/dist/web/server.d.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
export interface WebUiServer {
|
|
3
3
|
/** The port the HTTP server is listening on. */
|
|
4
4
|
port: number;
|
|
5
|
+
/** Random per-run token required on every `/api/*` request (Bearer header or `?token=`). */
|
|
6
|
+
token: string;
|
|
5
7
|
/** Gracefully shut down the HTTP server. */
|
|
6
8
|
close: () => Promise<void>;
|
|
7
9
|
}
|
package/dist/web/server.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
6
6
|
import { dirname, extname, join, sep } from "node:path";
|
|
7
|
+
import { randomBytes } from "node:crypto";
|
|
7
8
|
import { fileURLToPath } from "node:url";
|
|
8
9
|
import { LanceDbStore } from "../vectorstore/lancedb.js";
|
|
9
10
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
@@ -26,7 +27,12 @@ const MIME_TYPES = {
|
|
|
26
27
|
};
|
|
27
28
|
/** Serve an HTML string as the HTTP response with UTF-8 content type. */
|
|
28
29
|
function serveStatic(res, html) {
|
|
29
|
-
res.writeHead(200, {
|
|
30
|
+
res.writeHead(200, {
|
|
31
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
32
|
+
"Cache-Control": "no-cache",
|
|
33
|
+
"X-Content-Type-Options": "nosniff",
|
|
34
|
+
"Referrer-Policy": "no-referrer",
|
|
35
|
+
});
|
|
30
36
|
res.end(html);
|
|
31
37
|
}
|
|
32
38
|
/** Read a UI asset file from disk and serve it with the correct MIME type. Falls back to 404 if the file is missing. */
|
|
@@ -35,7 +41,13 @@ function serveUiAsset(res, filePath) {
|
|
|
35
41
|
const data = readFileSync(filePath);
|
|
36
42
|
const ext = extname(filePath);
|
|
37
43
|
const contentType = MIME_TYPES[ext] ?? "application/octet-stream";
|
|
38
|
-
|
|
44
|
+
// Vite emits hashed filenames, so assets can be cached aggressively
|
|
45
|
+
const cacheControl = /[.-][a-f0-9]{8,}\./.test(filePath) ? "public, max-age=31536000, immutable" : "no-cache";
|
|
46
|
+
res.writeHead(200, {
|
|
47
|
+
"Content-Type": contentType,
|
|
48
|
+
"Cache-Control": cacheControl,
|
|
49
|
+
"X-Content-Type-Options": "nosniff",
|
|
50
|
+
});
|
|
39
51
|
res.end(data);
|
|
40
52
|
}
|
|
41
53
|
catch {
|
|
@@ -69,49 +81,75 @@ export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cf
|
|
|
69
81
|
return embedderPromise;
|
|
70
82
|
}
|
|
71
83
|
const html = getStaticHtml();
|
|
72
|
-
const
|
|
84
|
+
const token = randomBytes(24).toString("hex");
|
|
85
|
+
const apiHandler = createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder, token);
|
|
73
86
|
const server = createServer(async (req, res) => {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const decoded = decodeURIComponent(url.slice("/ui/".length));
|
|
81
|
-
if (decoded.includes("..") || decoded === "") {
|
|
82
|
-
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
83
|
-
res.end("Forbidden");
|
|
87
|
+
try {
|
|
88
|
+
// Strip the query string before routing — the auth token arrives as
|
|
89
|
+
// `/?token=...` and must not break the root route match.
|
|
90
|
+
const url = (req.url ?? "/").split("?")[0] || "/";
|
|
91
|
+
if (url === "/" || url === "/index.html") {
|
|
92
|
+
serveStatic(res, html);
|
|
84
93
|
return;
|
|
85
94
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
95
|
+
if (url.startsWith("/ui/")) {
|
|
96
|
+
let decoded;
|
|
97
|
+
try {
|
|
98
|
+
decoded = decodeURIComponent(url.slice("/ui/".length));
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
res.writeHead(400, { "Content-Type": "text/plain" });
|
|
102
|
+
res.end("Bad Request");
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (decoded.includes("..") || decoded === "") {
|
|
106
|
+
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
107
|
+
res.end("Forbidden");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
// Try production build output first, fall back to dev source
|
|
111
|
+
const distAsset = resolveDistAsset(decoded);
|
|
112
|
+
if (distAsset) {
|
|
113
|
+
serveUiAsset(res, distAsset);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
const devPath = join(uiDir, decoded);
|
|
117
|
+
if (devPath.startsWith(uiDir + sep) && existsSync(devPath)) {
|
|
118
|
+
serveUiAsset(res, devPath);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
122
|
+
res.end("Not Found");
|
|
90
123
|
return;
|
|
91
124
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
125
|
+
if (url.startsWith("/api/")) {
|
|
126
|
+
const handled = await apiHandler(req, res);
|
|
127
|
+
if (handled)
|
|
128
|
+
return;
|
|
96
129
|
}
|
|
97
130
|
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
98
131
|
res.end("Not Found");
|
|
99
|
-
return;
|
|
100
132
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if (
|
|
104
|
-
|
|
133
|
+
catch (err) {
|
|
134
|
+
// Never let a malformed request crash the process
|
|
135
|
+
if (!res.headersSent) {
|
|
136
|
+
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
137
|
+
res.end("Internal Server Error");
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
res.end();
|
|
141
|
+
}
|
|
105
142
|
}
|
|
106
|
-
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
107
|
-
res.end("Not Found");
|
|
108
143
|
});
|
|
109
144
|
return new Promise((resolve, reject) => {
|
|
110
145
|
server.on("error", reject);
|
|
111
146
|
server.listen(port, "127.0.0.1", () => {
|
|
112
147
|
resolve({
|
|
113
148
|
port,
|
|
149
|
+
token,
|
|
114
150
|
close: () => new Promise((resolveClose) => {
|
|
151
|
+
// Close idle keep-alive connections so `server.close()` cannot hang
|
|
152
|
+
server.closeAllConnections();
|
|
115
153
|
server.close(() => {
|
|
116
154
|
store.close().catch(() => { });
|
|
117
155
|
keywordIndex.close();
|
package/dist/web/static.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Read and cache the Web UI `index.html` from disk.
|
|
3
3
|
*
|
|
4
|
-
* Reads from the Vite production build output (`dist/web/ui/index.html`)
|
|
5
|
-
*
|
|
4
|
+
* Reads from the Vite production build output (`dist/web/ui/index.html`).
|
|
5
|
+
* There is deliberately NO fallback to `src/web/ui/index.html`: the dev
|
|
6
|
+
* source references `/src/main.tsx`, which the embedded server does not
|
|
7
|
+
* serve — a "working" fallback would render a blank page. Use
|
|
8
|
+
* `npm run build` (or `npm run dev:ui` with the Vite dev server) instead.
|
|
6
9
|
*
|
|
7
10
|
* @returns The full HTML string of the Web UI entry page.
|
|
8
11
|
*/
|
package/dist/web/static.js
CHANGED
|
@@ -18,8 +18,11 @@ let cachedHtml = null;
|
|
|
18
18
|
/**
|
|
19
19
|
* Read and cache the Web UI `index.html` from disk.
|
|
20
20
|
*
|
|
21
|
-
* Reads from the Vite production build output (`dist/web/ui/index.html`)
|
|
22
|
-
*
|
|
21
|
+
* Reads from the Vite production build output (`dist/web/ui/index.html`).
|
|
22
|
+
* There is deliberately NO fallback to `src/web/ui/index.html`: the dev
|
|
23
|
+
* source references `/src/main.tsx`, which the embedded server does not
|
|
24
|
+
* serve — a "working" fallback would render a blank page. Use
|
|
25
|
+
* `npm run build` (or `npm run dev:ui` with the Vite dev server) instead.
|
|
23
26
|
*
|
|
24
27
|
* @returns The full HTML string of the Web UI entry page.
|
|
25
28
|
*/
|
|
@@ -28,9 +31,10 @@ export function getStaticHtml() {
|
|
|
28
31
|
return cachedHtml;
|
|
29
32
|
const root = projectRoot();
|
|
30
33
|
const prodPath = join(root, "dist", "web", "ui", "index.html");
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
+
if (!existsSync(prodPath)) {
|
|
35
|
+
throw new Error("Web UI not built — run `npm run build` (or use `npm run dev:ui` with the Vite dev server).");
|
|
36
|
+
}
|
|
37
|
+
cachedHtml = readFileSync(prodPath, "utf-8");
|
|
34
38
|
return cachedHtml;
|
|
35
39
|
}
|
|
36
40
|
/**
|