opencode-rag-plugin 1.19.3 → 1.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +1 -1
- package/dist/api.js +2 -0
- package/dist/chunker/base.js +19 -5
- package/dist/chunker/factory.js +27 -9
- package/dist/chunker/grammar.d.ts +18 -1
- package/dist/chunker/grammar.js +48 -10
- package/dist/chunker/pdf.js +30 -14
- package/dist/cli/commands/init-helpers.js +15 -2
- package/dist/cli/commands/init.js +31 -21
- package/dist/cli/commands/query.js +2 -0
- package/dist/cli/commands/quirk.js +9 -3
- package/dist/cli/commands/setup.js +5 -2
- package/dist/cli/commands/status.js +14 -5
- package/dist/cli/commands/ui.js +23 -9
- package/dist/cli/commands/update.js +4 -5
- package/dist/cli/format.d.ts +5 -2
- package/dist/cli/format.js +14 -5
- package/dist/content/image.js +33 -11
- package/dist/content/reader.js +79 -17
- package/dist/core/bootstrap.js +10 -3
- package/dist/core/config.js +31 -0
- package/dist/core/desc-cache.d.ts +8 -2
- package/dist/core/desc-cache.js +10 -3
- package/dist/core/doc-progress.js +5 -2
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/core/interfaces.js +8 -1
- package/dist/core/provider-defaults.d.ts +2 -0
- package/dist/core/provider-defaults.js +19 -4
- package/dist/core/runtime-overrides.d.ts +0 -6
- package/dist/core/version-check.d.ts +5 -0
- package/dist/core/version-check.js +8 -2
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +19 -5
- package/dist/describer/describer.js +15 -2
- package/dist/describer/gemini.js +25 -10
- package/dist/embedder/factory.d.ts +5 -3
- package/dist/embedder/factory.js +41 -8
- package/dist/embedder/health.js +19 -19
- package/dist/embedder/http.d.ts +14 -1
- package/dist/embedder/http.js +60 -6
- package/dist/eval/session-logger.js +7 -0
- package/dist/eval/storage.js +8 -0
- package/dist/indexer/git-diff.d.ts +1 -1
- package/dist/indexer/git-diff.js +5 -1
- package/dist/indexer/pipeline.js +421 -344
- package/dist/indexer/stats.d.ts +2 -0
- package/dist/indexer/stats.js +1 -0
- package/dist/indexer/watch.js +8 -1
- package/dist/indexer/worker.js +21 -0
- package/dist/mcp/cli.js +4 -0
- package/dist/mcp/handlers.d.ts +1 -1
- package/dist/mcp/handlers.js +23 -6
- package/dist/mcp/server.js +3 -0
- package/dist/opencode/create-read-tool.d.ts +1 -1
- package/dist/opencode/create-read-tool.js +17 -5
- package/dist/opencode/tool-args.js +23 -1
- package/dist/opencode/tools.d.ts +1 -1
- package/dist/opencode/tools.js +3 -1
- package/dist/plugin.d.ts +1 -1
- package/dist/plugin.js +69 -152
- package/dist/quirks/auto-capture.js +5 -0
- package/dist/quirks/quirk-store.d.ts +1 -1
- package/dist/quirks/quirk-store.js +56 -17
- package/dist/retriever/context-optimizer.js +18 -4
- package/dist/retriever/keyword-index.d.ts +2 -0
- package/dist/retriever/keyword-index.js +38 -4
- package/dist/retriever/retriever.js +6 -1
- package/dist/tui.js +41 -4
- package/dist/vectorstore/lancedb.d.ts +25 -1
- package/dist/vectorstore/lancedb.js +157 -11
- package/dist/vectorstore/memory.js +5 -1
- package/dist/watcher.js +30 -4
- package/dist/web/api.d.ts +6 -2
- package/dist/web/api.js +198 -70
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +66 -28
- package/dist/web/static.d.ts +5 -2
- package/dist/web/static.js +9 -5
- package/dist/web/ui/assets/index-BDPYdtA1.js +3 -0
- package/dist/web/ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/ui/assets/index-CJBvt6e0.js +0 -3
package/dist/web/api.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
2
|
import { extname, join, resolve as resolvePathModule } from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { listSessions, getSession, deleteSession, compareSessions, validateSessionID } from "../eval/storage.js";
|
|
5
5
|
import { analyzeTokenUsage, compareTokenAnalyses, projectTokenSavings } from "../eval/token-analysis.js";
|
|
6
6
|
import { listQuirks, lintQuirks, removeQuirk } from "../quirks/quirk-store.js";
|
|
7
7
|
import { retrieve } from "../retriever/retriever.js";
|
|
8
|
+
import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
|
|
8
9
|
const FILE_MIME_TYPES = {
|
|
9
10
|
".png": "image/png",
|
|
10
11
|
".jpg": "image/jpeg",
|
|
@@ -29,14 +30,53 @@ function parseQuery(url) {
|
|
|
29
30
|
params: new URLSearchParams(queryString ?? ""),
|
|
30
31
|
};
|
|
31
32
|
}
|
|
32
|
-
/**
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Determine whether a browser Origin header may call the API.
|
|
35
|
+
*
|
|
36
|
+
* Only same-machine origins are allowed (the server binds to 127.0.0.1). Any
|
|
37
|
+
* other origin — e.g. a random website doing a drive-by fetch — is rejected.
|
|
38
|
+
* Returns the origin to echo in `Access-Control-Allow-Origin`, or `null`.
|
|
39
|
+
*/
|
|
40
|
+
function isAllowedOrigin(origin) {
|
|
41
|
+
if (!origin)
|
|
42
|
+
return null;
|
|
43
|
+
try {
|
|
44
|
+
const u = new URL(origin);
|
|
45
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
46
|
+
return null;
|
|
47
|
+
return u.hostname === "127.0.0.1" || u.hostname === "localhost" || u.hostname === "[::1]" ? origin : null;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Compare two tokens in constant time to avoid timing side-channels. */
|
|
54
|
+
function tokensEqual(a, b) {
|
|
55
|
+
if (a.length !== b.length)
|
|
56
|
+
return false;
|
|
57
|
+
let diff = 0;
|
|
58
|
+
for (let i = 0; i < a.length; i++)
|
|
59
|
+
diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
60
|
+
return diff === 0;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Serialise an {@link ApiResponse} as JSON and write it to the HTTP response.
|
|
64
|
+
*
|
|
65
|
+
* CORS headers are only emitted for allowed (localhost) origins; cross-origin
|
|
66
|
+
* callers get no CORS headers at all, so browsers block reading the response.
|
|
67
|
+
*/
|
|
68
|
+
function sendJson(res, response, origin) {
|
|
69
|
+
const headers = {
|
|
35
70
|
"Content-Type": "application/json",
|
|
36
|
-
"Access-Control-Allow-Origin": "*",
|
|
37
71
|
"Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
|
|
38
|
-
"Access-Control-Allow-Headers": "Content-Type",
|
|
39
|
-
|
|
72
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
73
|
+
"X-Content-Type-Options": "nosniff",
|
|
74
|
+
};
|
|
75
|
+
if (origin) {
|
|
76
|
+
headers["Access-Control-Allow-Origin"] = origin;
|
|
77
|
+
headers["Vary"] = "Origin";
|
|
78
|
+
}
|
|
79
|
+
res.writeHead(response.status, headers);
|
|
40
80
|
res.end(JSON.stringify(response.body));
|
|
41
81
|
}
|
|
42
82
|
/**
|
|
@@ -56,11 +96,43 @@ function sendJson(res, response) {
|
|
|
56
96
|
* @param cfg - Active RAG configuration (used by quirk endpoints).
|
|
57
97
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
58
98
|
*/
|
|
59
|
-
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder) {
|
|
99
|
+
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder, token) {
|
|
60
100
|
return async (req, res) => {
|
|
61
101
|
const url = req.url ?? "/";
|
|
62
102
|
const method = req.method ?? "GET";
|
|
63
103
|
const { path, params } = parseQuery(url);
|
|
104
|
+
const origin = isAllowedOrigin(req.headers.origin);
|
|
105
|
+
// CORS preflight — only answer for allowed localhost origins
|
|
106
|
+
if (method === "OPTIONS") {
|
|
107
|
+
if (!origin) {
|
|
108
|
+
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
109
|
+
res.end("Forbidden");
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
res.writeHead(204, {
|
|
113
|
+
"Access-Control-Allow-Origin": origin,
|
|
114
|
+
"Vary": "Origin",
|
|
115
|
+
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
|
116
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
117
|
+
});
|
|
118
|
+
res.end();
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
// Same-machine-only: reject cross-origin requests outright (defeats drive-by
|
|
122
|
+
// website fetches and DNS rebinding even where CORS headers would block reads).
|
|
123
|
+
if (origin && !isAllowedOrigin(origin)) {
|
|
124
|
+
sendJson(res, { status: 403, body: { error: "Forbidden origin" } }, null);
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
// Token auth for every API request when a token is configured
|
|
128
|
+
if (token) {
|
|
129
|
+
const header = req.headers.authorization;
|
|
130
|
+
const supplied = header?.startsWith("Bearer ") ? header.slice("Bearer ".length) : params.get("token");
|
|
131
|
+
if (!supplied || !tokensEqual(supplied, token)) {
|
|
132
|
+
sendJson(res, { status: 401, body: { error: "Unauthorized — missing or invalid token" } }, origin);
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
64
136
|
// Quirk store dependencies (embedder is a no-op stub for the read-only UI context).
|
|
65
137
|
const quirkDeps = {
|
|
66
138
|
embedder: stubEmbedder,
|
|
@@ -69,16 +141,6 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
69
141
|
cfg: cfg ?? {},
|
|
70
142
|
storePath,
|
|
71
143
|
};
|
|
72
|
-
// CORS preflight
|
|
73
|
-
if (method === "OPTIONS") {
|
|
74
|
-
res.writeHead(204, {
|
|
75
|
-
"Access-Control-Allow-Origin": "*",
|
|
76
|
-
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
|
77
|
-
"Access-Control-Allow-Headers": "Content-Type",
|
|
78
|
-
});
|
|
79
|
-
res.end();
|
|
80
|
-
return true;
|
|
81
|
-
}
|
|
82
144
|
let response;
|
|
83
145
|
try {
|
|
84
146
|
// Existing endpoints
|
|
@@ -154,6 +216,14 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
154
216
|
else if (path === "/api/eval/sessions" && method === "GET") {
|
|
155
217
|
response = await handleEvalSessions(storePath);
|
|
156
218
|
}
|
|
219
|
+
else if (path === "/api/eval/compare" && method === "GET") {
|
|
220
|
+
response = await handleEvalCompare(storePath, params);
|
|
221
|
+
}
|
|
222
|
+
// Token analysis endpoints — must precede the generic `/api/eval/sessions/:id` route
|
|
223
|
+
else if (path.startsWith("/api/eval/sessions/") && path.endsWith("/analysis") && method === "GET") {
|
|
224
|
+
const id = path.slice("/api/eval/sessions/".length, -"/analysis".length);
|
|
225
|
+
response = await handleEvalAnalysis(storePath, id);
|
|
226
|
+
}
|
|
157
227
|
else if (path.startsWith("/api/eval/sessions/") && method === "GET") {
|
|
158
228
|
const id = path.slice("/api/eval/sessions/".length);
|
|
159
229
|
response = await handleEvalSession(storePath, id);
|
|
@@ -162,14 +232,6 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
162
232
|
const id = path.slice("/api/eval/sessions/".length);
|
|
163
233
|
response = await handleEvalDeleteSession(storePath, id);
|
|
164
234
|
}
|
|
165
|
-
else if (path === "/api/eval/compare" && method === "GET") {
|
|
166
|
-
response = await handleEvalCompare(storePath, params);
|
|
167
|
-
}
|
|
168
|
-
// Token analysis endpoints
|
|
169
|
-
else if (path.startsWith("/api/eval/sessions/") && path.endsWith("/analysis") && method === "GET") {
|
|
170
|
-
const id = path.slice("/api/eval/sessions/".length, -"/analysis".length);
|
|
171
|
-
response = await handleEvalAnalysis(storePath, id);
|
|
172
|
-
}
|
|
173
235
|
else if (path === "/api/eval/token-compare" && method === "GET") {
|
|
174
236
|
response = await handleEvalTokenCompare(storePath, params);
|
|
175
237
|
}
|
|
@@ -196,12 +258,20 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEm
|
|
|
196
258
|
else {
|
|
197
259
|
return false;
|
|
198
260
|
}
|
|
199
|
-
sendJson(res, response);
|
|
261
|
+
sendJson(res, response, origin);
|
|
200
262
|
return true;
|
|
201
263
|
}
|
|
202
264
|
catch (err) {
|
|
265
|
+
if (err instanceof BodyTooLargeError) {
|
|
266
|
+
if (!res.destroyed) {
|
|
267
|
+
sendJson(res, { status: 413, body: { error: err.message } }, origin);
|
|
268
|
+
}
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
203
271
|
const message = err instanceof Error ? err.message : String(err);
|
|
204
|
-
|
|
272
|
+
if (!res.destroyed) {
|
|
273
|
+
sendJson(res, { status: 500, body: { error: message } }, origin);
|
|
274
|
+
}
|
|
205
275
|
return true;
|
|
206
276
|
}
|
|
207
277
|
};
|
|
@@ -259,23 +329,19 @@ async function handleQuirkDelete(deps, id) {
|
|
|
259
329
|
/**
|
|
260
330
|
* Respond with a paginated, optionally filtered list of chunks.
|
|
261
331
|
*
|
|
262
|
-
* Query params: `offset` (default 0), `limit` (default 50
|
|
332
|
+
* Query params: `offset` (default 0, clamped >= 0), `limit` (default 50,
|
|
333
|
+
* clamped 1..500), `lang`, `file`. Filtering and pagination are pushed
|
|
334
|
+
* down into the store query — loading 100k rows per request was a memory
|
|
335
|
+
* blowup on large stores.
|
|
263
336
|
*/
|
|
264
337
|
async function handleChunks(store, params) {
|
|
265
|
-
const
|
|
266
|
-
const
|
|
338
|
+
const rawOffset = parseInt(params.get("offset") ?? "0", 10);
|
|
339
|
+
const rawLimit = parseInt(params.get("limit") ?? "50", 10);
|
|
340
|
+
const offset = Number.isFinite(rawOffset) ? Math.max(0, rawOffset) : 0;
|
|
341
|
+
const limit = Number.isFinite(rawLimit) ? Math.min(500, Math.max(1, rawLimit)) : 50;
|
|
267
342
|
const langFilter = params.get("lang");
|
|
268
343
|
const fileFilter = params.get("file");
|
|
269
|
-
const
|
|
270
|
-
let filtered = allChunks;
|
|
271
|
-
if (langFilter) {
|
|
272
|
-
filtered = filtered.filter((c) => c.language === langFilter);
|
|
273
|
-
}
|
|
274
|
-
if (fileFilter) {
|
|
275
|
-
filtered = filtered.filter((c) => c.filePath.startsWith(fileFilter));
|
|
276
|
-
}
|
|
277
|
-
const total = filtered.length;
|
|
278
|
-
const chunks = filtered.slice(offset, offset + limit);
|
|
344
|
+
const { chunks, total } = await store.getChunksFiltered(offset, limit, langFilter || undefined, fileFilter || undefined);
|
|
279
345
|
return {
|
|
280
346
|
status: 200,
|
|
281
347
|
body: { chunks, total, offset, limit },
|
|
@@ -283,8 +349,7 @@ async function handleChunks(store, params) {
|
|
|
283
349
|
}
|
|
284
350
|
/** Respond with a single chunk identified by its ID, or 404 if not found. */
|
|
285
351
|
async function handleChunkById(store, id) {
|
|
286
|
-
const
|
|
287
|
-
const chunk = chunks.find((c) => c.id === id);
|
|
352
|
+
const chunk = await store.getChunkById(id);
|
|
288
353
|
if (!chunk) {
|
|
289
354
|
return { status: 404, body: { error: "Chunk not found" } };
|
|
290
355
|
}
|
|
@@ -293,11 +358,12 @@ async function handleChunkById(store, id) {
|
|
|
293
358
|
/** Run a keyword search against the index and return ranked results. Query param: `q` (query string), `topK` (default 20). */
|
|
294
359
|
async function handleSearch(keywordIndex, params) {
|
|
295
360
|
const query = params.get("q") ?? "";
|
|
296
|
-
const
|
|
361
|
+
const rawTopK = parseInt(params.get("topK") ?? "20", 10);
|
|
362
|
+
const topK = Number.isFinite(rawTopK) ? Math.min(100, Math.max(1, rawTopK)) : 20;
|
|
297
363
|
if (!query.trim()) {
|
|
298
364
|
return { status: 200, body: { results: [] } };
|
|
299
365
|
}
|
|
300
|
-
const results = keywordIndex.search(query, topK);
|
|
366
|
+
const results = keywordIndex.search(query, topK, CODE_SEARCH_FILTER);
|
|
301
367
|
return {
|
|
302
368
|
status: 200,
|
|
303
369
|
body: {
|
|
@@ -336,9 +402,14 @@ async function handleRetrieve(store, keywordIndex, getEmbedder, cfg, params) {
|
|
|
336
402
|
catch (err) {
|
|
337
403
|
return { status: 503, body: { error: `Embedding model unavailable: ${err.message}. Check that your embedding provider is running.` } };
|
|
338
404
|
}
|
|
339
|
-
const
|
|
340
|
-
const
|
|
341
|
-
const
|
|
405
|
+
const rawTopK = parseInt(params.get("topK") ?? "10", 10);
|
|
406
|
+
const rawMinScore = parseFloat(params.get("minScore") ?? "0.35");
|
|
407
|
+
const rawKeywordWeight = parseFloat(params.get("keywordWeight") ?? "0.4");
|
|
408
|
+
// Clamp all numeric params — a topK of 1e9 would overfetch 3x via the
|
|
409
|
+
// retriever's overfetch factor and blow up memory.
|
|
410
|
+
const topK = Number.isFinite(rawTopK) ? Math.min(100, Math.max(1, rawTopK)) : 10;
|
|
411
|
+
const minScore = Number.isFinite(rawMinScore) ? Math.min(1, Math.max(0, rawMinScore)) : 0.35;
|
|
412
|
+
const keywordWeight = Number.isFinite(rawKeywordWeight) ? Math.min(1, Math.max(0, rawKeywordWeight)) : 0.4;
|
|
342
413
|
const hybrid = params.get("hybrid") !== "false";
|
|
343
414
|
const explain = params.get("explain") !== "false";
|
|
344
415
|
const pathFilter = params.get("path") ?? undefined;
|
|
@@ -355,6 +426,7 @@ async function handleRetrieve(store, keywordIndex, getEmbedder, cfg, params) {
|
|
|
355
426
|
filter: {
|
|
356
427
|
pathPatterns: pathFilter ? pathFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
|
|
357
428
|
languages: langFilter ? langFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
|
|
429
|
+
kinds: CODE_SEARCH_FILTER.kinds,
|
|
358
430
|
},
|
|
359
431
|
});
|
|
360
432
|
return {
|
|
@@ -402,10 +474,15 @@ async function handleCompare(store, params) {
|
|
|
402
474
|
if (ids.length === 0) {
|
|
403
475
|
return { status: 400, body: { error: "No chunk IDs provided" } };
|
|
404
476
|
}
|
|
405
|
-
|
|
406
|
-
|
|
477
|
+
if (ids.length > 100) {
|
|
478
|
+
return { status: 400, body: { error: "Too many chunk IDs (max 100)" } };
|
|
479
|
+
}
|
|
480
|
+
const chunks = await store.getChunksByIds(ids);
|
|
407
481
|
return { status: 200, body: { chunks } };
|
|
408
482
|
}
|
|
483
|
+
// Re-hashing every manifest file per status request is expensive — cache the
|
|
484
|
+
// result keyed on the manifest file's mtime+size (invalidated on any write).
|
|
485
|
+
let statusCache = null;
|
|
409
486
|
/**
|
|
410
487
|
* Return indexing status — manifest stats, staleness, and a placeholder for watcher state.
|
|
411
488
|
*/
|
|
@@ -429,20 +506,37 @@ async function handleIndexingStatus(storePath, cwd) {
|
|
|
429
506
|
if (manifest?.lastIndexedAt) {
|
|
430
507
|
lastIndexedAt = new Date(manifest.lastIndexedAt).toISOString();
|
|
431
508
|
}
|
|
432
|
-
// Count stale files by comparing manifest file list against current disk state
|
|
509
|
+
// Count stale files by comparing manifest file list against current disk state.
|
|
510
|
+
// Cached by manifest mtime+size so the poll loop doesn't hash every file
|
|
511
|
+
// synchronously on the event loop per request.
|
|
433
512
|
if (cwd && manifest?.files) {
|
|
434
|
-
const
|
|
435
|
-
for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
|
|
513
|
+
const cacheKey = (() => {
|
|
436
514
|
try {
|
|
437
|
-
const
|
|
438
|
-
|
|
439
|
-
const hash = createHash("sha256").update(content).digest("hex");
|
|
440
|
-
if (hash !== fileMeta.hash)
|
|
441
|
-
staleFileCount++;
|
|
515
|
+
const st = statSync(manifestPath);
|
|
516
|
+
return `${st.mtimeMs}:${st.size}`;
|
|
442
517
|
}
|
|
443
518
|
catch {
|
|
444
|
-
|
|
519
|
+
return "missing";
|
|
445
520
|
}
|
|
521
|
+
})();
|
|
522
|
+
if (statusCache && statusCache.key === cacheKey) {
|
|
523
|
+
staleFileCount = statusCache.staleFileCount;
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
const storedFiles = manifest.files;
|
|
527
|
+
for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
|
|
528
|
+
try {
|
|
529
|
+
const fullPath = filePath;
|
|
530
|
+
const content = readFileSync(fullPath, "utf-8");
|
|
531
|
+
const hash = createHash("sha256").update(content).digest("hex");
|
|
532
|
+
if (hash !== fileMeta.hash)
|
|
533
|
+
staleFileCount++;
|
|
534
|
+
}
|
|
535
|
+
catch {
|
|
536
|
+
staleFileCount++; // file was deleted or unreadable
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
statusCache = { key: cacheKey, staleFileCount, manifest };
|
|
446
540
|
}
|
|
447
541
|
}
|
|
448
542
|
return {
|
|
@@ -454,22 +548,38 @@ async function handleIndexingStatus(storePath, cwd) {
|
|
|
454
548
|
},
|
|
455
549
|
};
|
|
456
550
|
}
|
|
551
|
+
/** Guards concurrent reindex requests — only one pass may run at a time. */
|
|
552
|
+
let reindexInFlight = false;
|
|
457
553
|
/**
|
|
458
554
|
* Trigger a one-shot reindex pass in the background.
|
|
459
555
|
*/
|
|
460
556
|
async function handleReindex(cwd, cfg, storePath, store, getEmbedder) {
|
|
557
|
+
if (reindexInFlight) {
|
|
558
|
+
return { status: 409, body: { error: "A reindex is already running" } };
|
|
559
|
+
}
|
|
461
560
|
try {
|
|
462
561
|
const { runIndexPass } = await import("../indexer.js");
|
|
463
562
|
const embedder = getEmbedder ? await getEmbedder() : undefined;
|
|
464
563
|
if (!embedder) {
|
|
465
564
|
return { status: 503, body: { error: "Embedder not available" } };
|
|
466
565
|
}
|
|
467
|
-
|
|
566
|
+
reindexInFlight = true;
|
|
567
|
+
runIndexPass({ cwd, storePath, config: cfg, store, embedder })
|
|
568
|
+
.then(() => {
|
|
569
|
+
reindexInFlight = false;
|
|
570
|
+
statusCache = null; // invalidate the status cache after a pass
|
|
571
|
+
projectionCache = null;
|
|
572
|
+
})
|
|
573
|
+
.catch((err) => {
|
|
574
|
+
reindexInFlight = false;
|
|
575
|
+
statusCache = null;
|
|
576
|
+
projectionCache = null;
|
|
468
577
|
console.error("Background reindex failed:", err);
|
|
469
578
|
});
|
|
470
579
|
return { status: 200, body: { started: true } };
|
|
471
580
|
}
|
|
472
581
|
catch (err) {
|
|
582
|
+
reindexInFlight = false;
|
|
473
583
|
return { status: 500, body: { error: `Failed to start reindex: ${err.message}` } };
|
|
474
584
|
}
|
|
475
585
|
}
|
|
@@ -483,7 +593,7 @@ function handleConfig(cfg) {
|
|
|
483
593
|
}
|
|
484
594
|
function redactKeys(obj) {
|
|
485
595
|
for (const key of Object.keys(obj)) {
|
|
486
|
-
if (key.
|
|
596
|
+
if (/api\s*key|apikey|password|passwd|secret|token|authorization|credential/i.test(key)) {
|
|
487
597
|
obj[key] = "***";
|
|
488
598
|
}
|
|
489
599
|
else if (typeof obj[key] === "object" && obj[key] !== null) {
|
|
@@ -493,16 +603,28 @@ function redactKeys(obj) {
|
|
|
493
603
|
}
|
|
494
604
|
/**
|
|
495
605
|
* Project chunk embeddings to 2D via PCA for the Embedding Space Explorer.
|
|
606
|
+
* Capped at 5000 chunks and memoized per (storePath, maxChunks) so the
|
|
607
|
+
* O(n·dim²) computation does not run on every visit.
|
|
496
608
|
*/
|
|
609
|
+
let projectionCache = null;
|
|
497
610
|
async function handleEmbeddingProjection(store, params) {
|
|
498
|
-
const
|
|
611
|
+
const rawMaxChunks = parseInt(params.get("maxChunks") ?? "5000", 10);
|
|
612
|
+
const maxChunks = Number.isFinite(rawMaxChunks) ? Math.min(5000, Math.max(1, rawMaxChunks)) : 5000;
|
|
499
613
|
try {
|
|
614
|
+
// Invalidated after a reindex pass completes (see handleReindex)
|
|
615
|
+
const cacheKey = `${maxChunks}`;
|
|
616
|
+
if (projectionCache && projectionCache.key === cacheKey) {
|
|
617
|
+
return { status: 200, body: projectionCache.body };
|
|
618
|
+
}
|
|
500
619
|
const chunks = await store.getChunksWithEmbeddings(maxChunks);
|
|
501
620
|
if (chunks.length === 0) {
|
|
502
|
-
|
|
621
|
+
projectionCache = { key: cacheKey, body: { points: [], totalChunks: 0 } };
|
|
622
|
+
return { status: 200, body: projectionCache.body };
|
|
503
623
|
}
|
|
504
624
|
if (chunks.length === 1) {
|
|
505
|
-
|
|
625
|
+
const body = { points: [{ id: chunks[0].id, x: 0.5, y: 0.5, filePath: chunks[0].filePath, startLine: chunks[0].startLine, endLine: chunks[0].endLine, language: chunks[0].language, description: chunks[0].description }], totalChunks: 1, displayedChunks: 1 };
|
|
626
|
+
projectionCache = { key: cacheKey, body };
|
|
627
|
+
return { status: 200, body };
|
|
506
628
|
}
|
|
507
629
|
const { computePCA } = await import("./pca.js");
|
|
508
630
|
const vectors = chunks.map(c => c.embedding);
|
|
@@ -517,10 +639,9 @@ async function handleEmbeddingProjection(store, params) {
|
|
|
517
639
|
language: c.language,
|
|
518
640
|
description: c.description,
|
|
519
641
|
}));
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
};
|
|
642
|
+
const body = { points, totalChunks: chunks.length, displayedChunks: points.length };
|
|
643
|
+
projectionCache = { key: cacheKey, body };
|
|
644
|
+
return { status: 200, body };
|
|
524
645
|
}
|
|
525
646
|
catch (err) {
|
|
526
647
|
return { status: 500, body: { error: `Projection failed: ${err.message}` } };
|
|
@@ -652,6 +773,13 @@ export function handleEvalProjectSavings(body) {
|
|
|
652
773
|
}
|
|
653
774
|
/** Collect the full request body as a Buffer and parse it as JSON. Returns `{}` on empty or invalid input. */
|
|
654
775
|
const MAX_BODY_BYTES = 1_048_576; // 1 MB
|
|
776
|
+
/** Thrown when the request body exceeds {@link MAX_BODY_BYTES}; mapped to a 413 response. */
|
|
777
|
+
export class BodyTooLargeError extends Error {
|
|
778
|
+
constructor() {
|
|
779
|
+
super(`Request body exceeds ${MAX_BODY_BYTES} byte limit`);
|
|
780
|
+
this.name = "BodyTooLargeError";
|
|
781
|
+
}
|
|
782
|
+
}
|
|
655
783
|
async function readBody(req) {
|
|
656
784
|
const chunks = [];
|
|
657
785
|
let totalSize = 0;
|
|
@@ -659,8 +787,8 @@ async function readBody(req) {
|
|
|
659
787
|
const buf = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
660
788
|
totalSize += buf.length;
|
|
661
789
|
if (totalSize > MAX_BODY_BYTES) {
|
|
662
|
-
req.destroy(
|
|
663
|
-
throw new
|
|
790
|
+
req.destroy();
|
|
791
|
+
throw new BodyTooLargeError();
|
|
664
792
|
}
|
|
665
793
|
chunks.push(buf);
|
|
666
794
|
}
|
package/dist/web/server.d.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
export interface WebUiServer {
|
|
3
3
|
/** The port the HTTP server is listening on. */
|
|
4
4
|
port: number;
|
|
5
|
+
/** Random per-run token required on every `/api/*` request (Bearer header or `?token=`). */
|
|
6
|
+
token: string;
|
|
5
7
|
/** Gracefully shut down the HTTP server. */
|
|
6
8
|
close: () => Promise<void>;
|
|
7
9
|
}
|
package/dist/web/server.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
6
6
|
import { dirname, extname, join, sep } from "node:path";
|
|
7
|
+
import { randomBytes } from "node:crypto";
|
|
7
8
|
import { fileURLToPath } from "node:url";
|
|
8
9
|
import { LanceDbStore } from "../vectorstore/lancedb.js";
|
|
9
10
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
@@ -26,7 +27,12 @@ const MIME_TYPES = {
|
|
|
26
27
|
};
|
|
27
28
|
/** Serve an HTML string as the HTTP response with UTF-8 content type. */
|
|
28
29
|
function serveStatic(res, html) {
|
|
29
|
-
res.writeHead(200, {
|
|
30
|
+
res.writeHead(200, {
|
|
31
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
32
|
+
"Cache-Control": "no-cache",
|
|
33
|
+
"X-Content-Type-Options": "nosniff",
|
|
34
|
+
"Referrer-Policy": "no-referrer",
|
|
35
|
+
});
|
|
30
36
|
res.end(html);
|
|
31
37
|
}
|
|
32
38
|
/** Read a UI asset file from disk and serve it with the correct MIME type. Falls back to 404 if the file is missing. */
|
|
@@ -35,7 +41,13 @@ function serveUiAsset(res, filePath) {
|
|
|
35
41
|
const data = readFileSync(filePath);
|
|
36
42
|
const ext = extname(filePath);
|
|
37
43
|
const contentType = MIME_TYPES[ext] ?? "application/octet-stream";
|
|
38
|
-
|
|
44
|
+
// Vite emits hashed filenames, so assets can be cached aggressively
|
|
45
|
+
const cacheControl = /[.-][a-f0-9]{8,}\./.test(filePath) ? "public, max-age=31536000, immutable" : "no-cache";
|
|
46
|
+
res.writeHead(200, {
|
|
47
|
+
"Content-Type": contentType,
|
|
48
|
+
"Cache-Control": cacheControl,
|
|
49
|
+
"X-Content-Type-Options": "nosniff",
|
|
50
|
+
});
|
|
39
51
|
res.end(data);
|
|
40
52
|
}
|
|
41
53
|
catch {
|
|
@@ -69,49 +81,75 @@ export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cf
|
|
|
69
81
|
return embedderPromise;
|
|
70
82
|
}
|
|
71
83
|
const html = getStaticHtml();
|
|
72
|
-
const
|
|
84
|
+
const token = randomBytes(24).toString("hex");
|
|
85
|
+
const apiHandler = createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder, token);
|
|
73
86
|
const server = createServer(async (req, res) => {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const decoded = decodeURIComponent(url.slice("/ui/".length));
|
|
81
|
-
if (decoded.includes("..") || decoded === "") {
|
|
82
|
-
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
83
|
-
res.end("Forbidden");
|
|
87
|
+
try {
|
|
88
|
+
// Strip the query string before routing — the auth token arrives as
|
|
89
|
+
// `/?token=...` and must not break the root route match.
|
|
90
|
+
const url = (req.url ?? "/").split("?")[0] || "/";
|
|
91
|
+
if (url === "/" || url === "/index.html") {
|
|
92
|
+
serveStatic(res, html);
|
|
84
93
|
return;
|
|
85
94
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
95
|
+
if (url.startsWith("/ui/")) {
|
|
96
|
+
let decoded;
|
|
97
|
+
try {
|
|
98
|
+
decoded = decodeURIComponent(url.slice("/ui/".length));
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
res.writeHead(400, { "Content-Type": "text/plain" });
|
|
102
|
+
res.end("Bad Request");
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (decoded.includes("..") || decoded === "") {
|
|
106
|
+
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
107
|
+
res.end("Forbidden");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
// Try production build output first, fall back to dev source
|
|
111
|
+
const distAsset = resolveDistAsset(decoded);
|
|
112
|
+
if (distAsset) {
|
|
113
|
+
serveUiAsset(res, distAsset);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
const devPath = join(uiDir, decoded);
|
|
117
|
+
if (devPath.startsWith(uiDir + sep) && existsSync(devPath)) {
|
|
118
|
+
serveUiAsset(res, devPath);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
122
|
+
res.end("Not Found");
|
|
90
123
|
return;
|
|
91
124
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
125
|
+
if (url.startsWith("/api/")) {
|
|
126
|
+
const handled = await apiHandler(req, res);
|
|
127
|
+
if (handled)
|
|
128
|
+
return;
|
|
96
129
|
}
|
|
97
130
|
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
98
131
|
res.end("Not Found");
|
|
99
|
-
return;
|
|
100
132
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if (
|
|
104
|
-
|
|
133
|
+
catch (err) {
|
|
134
|
+
// Never let a malformed request crash the process
|
|
135
|
+
if (!res.headersSent) {
|
|
136
|
+
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
137
|
+
res.end("Internal Server Error");
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
res.end();
|
|
141
|
+
}
|
|
105
142
|
}
|
|
106
|
-
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
107
|
-
res.end("Not Found");
|
|
108
143
|
});
|
|
109
144
|
return new Promise((resolve, reject) => {
|
|
110
145
|
server.on("error", reject);
|
|
111
146
|
server.listen(port, "127.0.0.1", () => {
|
|
112
147
|
resolve({
|
|
113
148
|
port,
|
|
149
|
+
token,
|
|
114
150
|
close: () => new Promise((resolveClose) => {
|
|
151
|
+
// Close idle keep-alive connections so `server.close()` cannot hang
|
|
152
|
+
server.closeAllConnections();
|
|
115
153
|
server.close(() => {
|
|
116
154
|
store.close().catch(() => { });
|
|
117
155
|
keywordIndex.close();
|
package/dist/web/static.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Read and cache the Web UI `index.html` from disk.
|
|
3
3
|
*
|
|
4
|
-
* Reads from the Vite production build output (`dist/web/ui/index.html`)
|
|
5
|
-
*
|
|
4
|
+
* Reads from the Vite production build output (`dist/web/ui/index.html`).
|
|
5
|
+
* There is deliberately NO fallback to `src/web/ui/index.html`: the dev
|
|
6
|
+
* source references `/src/main.tsx`, which the embedded server does not
|
|
7
|
+
* serve — a "working" fallback would render a blank page. Use
|
|
8
|
+
* `npm run build` (or `npm run dev:ui` with the Vite dev server) instead.
|
|
6
9
|
*
|
|
7
10
|
* @returns The full HTML string of the Web UI entry page.
|
|
8
11
|
*/
|
package/dist/web/static.js
CHANGED
|
@@ -18,8 +18,11 @@ let cachedHtml = null;
|
|
|
18
18
|
/**
|
|
19
19
|
* Read and cache the Web UI `index.html` from disk.
|
|
20
20
|
*
|
|
21
|
-
* Reads from the Vite production build output (`dist/web/ui/index.html`)
|
|
22
|
-
*
|
|
21
|
+
* Reads from the Vite production build output (`dist/web/ui/index.html`).
|
|
22
|
+
* There is deliberately NO fallback to `src/web/ui/index.html`: the dev
|
|
23
|
+
* source references `/src/main.tsx`, which the embedded server does not
|
|
24
|
+
* serve — a "working" fallback would render a blank page. Use
|
|
25
|
+
* `npm run build` (or `npm run dev:ui` with the Vite dev server) instead.
|
|
23
26
|
*
|
|
24
27
|
* @returns The full HTML string of the Web UI entry page.
|
|
25
28
|
*/
|
|
@@ -28,9 +31,10 @@ export function getStaticHtml() {
|
|
|
28
31
|
return cachedHtml;
|
|
29
32
|
const root = projectRoot();
|
|
30
33
|
const prodPath = join(root, "dist", "web", "ui", "index.html");
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
+
if (!existsSync(prodPath)) {
|
|
35
|
+
throw new Error("Web UI not built — run `npm run build` (or use `npm run dev:ui` with the Vite dev server).");
|
|
36
|
+
}
|
|
37
|
+
cachedHtml = readFileSync(prodPath, "utf-8");
|
|
34
38
|
return cachedHtml;
|
|
35
39
|
}
|
|
36
40
|
/**
|