opencode-rag-plugin 1.18.2 → 1.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +1 -1
- package/dist/content/reader.d.ts +2 -1
- package/dist/content/reader.js +9 -15
- package/dist/core/config.d.ts +28 -2
- package/dist/core/config.js +13 -1
- package/dist/core/exclude.d.ts +4 -0
- package/dist/core/exclude.js +45 -0
- package/dist/core/interfaces.d.ts +6 -0
- package/dist/embedder/factory.d.ts +10 -3
- package/dist/embedder/factory.js +38 -9
- package/dist/indexer/pipeline.js +24 -3
- package/dist/indexer/watch.js +4 -3
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +1 -0
- package/dist/plugin.js +15 -1
- package/dist/quirks/quirk-store.d.ts +15 -0
- package/dist/quirks/quirk-store.js +27 -0
- package/dist/vectorstore/lancedb.d.ts +33 -0
- package/dist/vectorstore/lancedb.js +169 -80
- package/dist/web/api.d.ts +2 -1
- package/dist/web/api.js +219 -3
- package/dist/web/pca.d.ts +7 -0
- package/dist/web/pca.js +87 -0
- package/dist/web/server.js +23 -8
- package/dist/web/static.d.ts +7 -2
- package/dist/web/static.js +31 -7
- package/dist/web/ui/assets/index-CJBvt6e0.js +3 -0
- package/dist/web/ui/assets/index-CKdp79Tw.css +1 -0
- package/dist/web/ui/assets/vendor-Dy7HKFCY.js +1 -0
- package/dist/web/ui/index.html +9 -1632
- package/package.json +10 -4
- package/dist/web/ui/app.css +0 -1
- package/dist/web/ui/github-dark.css +0 -118
- package/dist/web/ui/highlight.min.js +0 -1213
|
@@ -384,26 +384,28 @@ export class LanceDbStore {
|
|
|
384
384
|
* @returns An array of file summaries sorted by file path.
|
|
385
385
|
*/
|
|
386
386
|
async listFiles() {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
existing
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
387
|
+
return this.withCorruptionRecovery(async () => {
|
|
388
|
+
const table = await this.getTable();
|
|
389
|
+
const count = await table.countRows();
|
|
390
|
+
if (count === 0)
|
|
391
|
+
return [];
|
|
392
|
+
const rows = await table.query().select(["filePath", "language"]).limit(count).toArray();
|
|
393
|
+
const fileMap = new Map();
|
|
394
|
+
for (const row of rows) {
|
|
395
|
+
const filePath = row.filePath;
|
|
396
|
+
const language = row.language;
|
|
397
|
+
const existing = fileMap.get(filePath);
|
|
398
|
+
if (existing) {
|
|
399
|
+
existing.chunkCount++;
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
fileMap.set(filePath, { language, chunkCount: 1 });
|
|
403
|
+
}
|
|
402
404
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
405
|
+
return Array.from(fileMap.entries())
|
|
406
|
+
.map(([filePath, info]) => ({ filePath, ...info }))
|
|
407
|
+
.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
408
|
+
});
|
|
407
409
|
}
|
|
408
410
|
/**
|
|
409
411
|
* Retrieve all chunks for a specific file path, sorted by start line.
|
|
@@ -411,39 +413,41 @@ export class LanceDbStore {
|
|
|
411
413
|
* @returns An array of chunks for that file.
|
|
412
414
|
*/
|
|
413
415
|
async getChunksByFilePath(filePath) {
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
.
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
416
|
+
return this.withCorruptionRecovery(async () => {
|
|
417
|
+
const table = await this.getTable();
|
|
418
|
+
const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
|
|
419
|
+
const rows = await table.query()
|
|
420
|
+
.select(QUERY_COLUMNS)
|
|
421
|
+
.where(`filePath = '${normalizedPath}'`)
|
|
422
|
+
.toArray();
|
|
423
|
+
return rows
|
|
424
|
+
.map((row) => {
|
|
425
|
+
let tags;
|
|
426
|
+
try {
|
|
427
|
+
const raw = row.tags;
|
|
428
|
+
if (raw)
|
|
429
|
+
tags = JSON.parse(raw);
|
|
430
|
+
}
|
|
431
|
+
catch {
|
|
432
|
+
tags = undefined;
|
|
433
|
+
}
|
|
434
|
+
return {
|
|
435
|
+
id: row.id,
|
|
436
|
+
content: row.content,
|
|
437
|
+
description: row.description ?? "",
|
|
438
|
+
metadata: {
|
|
439
|
+
filePath: row.filePath,
|
|
440
|
+
startLine: row.startLine,
|
|
441
|
+
endLine: row.endLine,
|
|
442
|
+
language: row.language,
|
|
443
|
+
kind: row.kind || undefined,
|
|
444
|
+
quirkType: row.quirkType || undefined,
|
|
445
|
+
tags,
|
|
446
|
+
},
|
|
447
|
+
};
|
|
448
|
+
})
|
|
449
|
+
.sort((a, b) => a.metadata.startLine - b.metadata.startLine);
|
|
450
|
+
});
|
|
447
451
|
}
|
|
448
452
|
/**
|
|
449
453
|
* Retrieve a paginated list of all chunks without embeddings.
|
|
@@ -452,24 +456,51 @@ export class LanceDbStore {
|
|
|
452
456
|
* @returns An array of chunk summaries.
|
|
453
457
|
*/
|
|
454
458
|
async getChunks(offset, limit) {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
.
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
459
|
+
return this.withCorruptionRecovery(async () => {
|
|
460
|
+
const table = await this.getTable();
|
|
461
|
+
const rows = await table.query()
|
|
462
|
+
.select(QUERY_COLUMNS)
|
|
463
|
+
.offset(offset)
|
|
464
|
+
.limit(limit)
|
|
465
|
+
.toArray();
|
|
466
|
+
return rows.map((row) => ({
|
|
467
|
+
id: row.id,
|
|
468
|
+
filePath: row.filePath,
|
|
469
|
+
language: row.language,
|
|
470
|
+
startLine: row.startLine,
|
|
471
|
+
endLine: row.endLine,
|
|
472
|
+
content: row.content,
|
|
473
|
+
description: row.description ?? "",
|
|
474
|
+
kind: row.kind ?? "",
|
|
475
|
+
quirkType: row.quirkType ?? "",
|
|
476
|
+
tags: row.tags ?? "",
|
|
477
|
+
}));
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Fetch chunks with their embedding vectors included (for embedding projection).
|
|
482
|
+
* @param limit - Maximum number of rows to return.
|
|
483
|
+
* @returns Array of { id, filePath, language, startLine, endLine, description, embedding }.
|
|
484
|
+
*/
|
|
485
|
+
async getChunksWithEmbeddings(limit) {
|
|
486
|
+
return this.withCorruptionRecovery(async () => {
|
|
487
|
+
const table = await this.getTable();
|
|
488
|
+
const rows = await table.query()
|
|
489
|
+
.select([...QUERY_COLUMNS, "embedding"])
|
|
490
|
+
.limit(limit)
|
|
491
|
+
.toArray();
|
|
492
|
+
return rows
|
|
493
|
+
.filter((r) => r.embedding && typeof r.embedding === "object")
|
|
494
|
+
.map((row) => ({
|
|
495
|
+
id: row.id,
|
|
496
|
+
filePath: row.filePath,
|
|
497
|
+
language: row.language,
|
|
498
|
+
startLine: row.startLine,
|
|
499
|
+
endLine: row.endLine,
|
|
500
|
+
description: row.description ?? "",
|
|
501
|
+
embedding: Array.from(row.embedding),
|
|
502
|
+
}));
|
|
503
|
+
});
|
|
473
504
|
}
|
|
474
505
|
/**
|
|
475
506
|
* Perform ANN search internally, returning results scored as cosine similarity (0-1).
|
|
@@ -548,7 +579,13 @@ export class LanceDbStore {
|
|
|
548
579
|
async optimize() {
|
|
549
580
|
try {
|
|
550
581
|
const table = await this.getTable();
|
|
551
|
-
|
|
582
|
+
// Clean up versions older than 1 hour — not "right now" — so in-flight
|
|
583
|
+
// queries (e.g. Web UI search, background auto-index) can finish before
|
|
584
|
+
// their data files are reclaimed. Using new Date() here caused data-file
|
|
585
|
+
// race conditions where a reader got "Not found: … .lance" because the
|
|
586
|
+
// GC deleted fragments that the current version still referenced.
|
|
587
|
+
const threshold = new Date(Date.now() - 60 * 60 * 1000);
|
|
588
|
+
await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
|
|
552
589
|
}
|
|
553
590
|
catch {
|
|
554
591
|
// Optimize is best-effort — must not break indexing.
|
|
@@ -564,15 +601,17 @@ export class LanceDbStore {
|
|
|
564
601
|
const tableNames = await db.tableNames();
|
|
565
602
|
if (!tableNames.includes(TABLE_NAME))
|
|
566
603
|
return [];
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
const
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
604
|
+
return this.withCorruptionRecovery(async () => {
|
|
605
|
+
const table = await this.getTable();
|
|
606
|
+
const rows = await table.query().select(["filePath"]).toArray();
|
|
607
|
+
const paths = new Set();
|
|
608
|
+
for (const row of rows) {
|
|
609
|
+
const fp = row.filePath;
|
|
610
|
+
if (fp)
|
|
611
|
+
paths.add(fp);
|
|
612
|
+
}
|
|
613
|
+
return Array.from(paths);
|
|
614
|
+
});
|
|
576
615
|
}
|
|
577
616
|
catch {
|
|
578
617
|
return [];
|
|
@@ -708,6 +747,56 @@ export class LanceDbStore {
|
|
|
708
747
|
const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
|
|
709
748
|
await table.delete(`filePath = '${normalizedPath}'`);
|
|
710
749
|
}
|
|
750
|
+
/**
|
|
751
|
+
* Verify that data in the store is actually readable.
|
|
752
|
+
* Reads the `content` column (a large text field stored in data fragments,
|
|
753
|
+
* not in the version manifest) for the first few rows. If any of the
|
|
754
|
+
* underlying `.lance` data files are missing from disk, LanceDB throws a
|
|
755
|
+
* corruption error ("Not found: ... .lance") and we return false.
|
|
756
|
+
*
|
|
757
|
+
* This catches silent corruption where countRows() returns metadata from
|
|
758
|
+
* the version manifest while the actual row data on disk is gone.
|
|
759
|
+
*/
|
|
760
|
+
async checkIntegrity() {
|
|
761
|
+
try {
|
|
762
|
+
const db = await this.getDb();
|
|
763
|
+
const tableNames = await db.tableNames();
|
|
764
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
765
|
+
return true;
|
|
766
|
+
const table = await this.getTable();
|
|
767
|
+
const count = await table.countRows();
|
|
768
|
+
if (count === 0)
|
|
769
|
+
return true;
|
|
770
|
+
// Read the `content` column (stored in data fragments, not in the
|
|
771
|
+
// version manifest) for the first few rows. If a fragment is
|
|
772
|
+
// missing, LanceDB will fail with a "Not found" corruption error.
|
|
773
|
+
const rows = await table.query().select(["id", "content"]).limit(10).toArray();
|
|
774
|
+
return rows.length > 0;
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
if (isCorruptionError(err))
|
|
778
|
+
return false;
|
|
779
|
+
return true;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Execute an async function with automatic corruption recovery.
|
|
784
|
+
* If the function throws a LanceDB corruption error, tryRepair() is run
|
|
785
|
+
* and the function is retried once. If repair fails, the original error
|
|
786
|
+
* is re-thrown so callers/higher layers can handle it (e.g. return
|
|
787
|
+
* fallback data, clear the manifest, or trigger a rebuild).
|
|
788
|
+
*/
|
|
789
|
+
async withCorruptionRecovery(fn) {
|
|
790
|
+
try {
|
|
791
|
+
return await fn();
|
|
792
|
+
}
|
|
793
|
+
catch (err) {
|
|
794
|
+
if (isCorruptionError(err) && await this.tryRepair()) {
|
|
795
|
+
return fn();
|
|
796
|
+
}
|
|
797
|
+
throw err;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
711
800
|
async tryRepair() {
|
|
712
801
|
try {
|
|
713
802
|
this.table = null;
|
package/dist/web/api.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
|
|
|
5
5
|
import { LanceDbStore } from "../vectorstore/lancedb.js";
|
|
6
6
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
7
7
|
import type { RagConfig } from "../core/config.js";
|
|
8
|
+
import type { EmbeddingProvider } from "../core/interfaces.js";
|
|
8
9
|
/** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
|
|
9
10
|
interface ApiResponse {
|
|
10
11
|
status: number;
|
|
@@ -27,7 +28,7 @@ interface ApiResponse {
|
|
|
27
28
|
* @param cfg - Active RAG configuration (used by quirk endpoints).
|
|
28
29
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
29
30
|
*/
|
|
30
|
-
export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
31
|
+
export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
31
32
|
/**
|
|
32
33
|
* Perform token-usage analysis for a single evaluation session.
|
|
33
34
|
*
|
package/dist/web/api.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
-
import { extname, resolve as resolvePathModule } from "node:path";
|
|
2
|
+
import { extname, join, resolve as resolvePathModule } from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
import { listSessions, getSession, deleteSession, compareSessions, validateSessionID } from "../eval/storage.js";
|
|
4
5
|
import { analyzeTokenUsage, compareTokenAnalyses, projectTokenSavings } from "../eval/token-analysis.js";
|
|
5
6
|
import { listQuirks, lintQuirks, removeQuirk } from "../quirks/quirk-store.js";
|
|
7
|
+
import { retrieve } from "../retriever/retriever.js";
|
|
6
8
|
const FILE_MIME_TYPES = {
|
|
7
9
|
".png": "image/png",
|
|
8
10
|
".jpg": "image/jpeg",
|
|
@@ -54,7 +56,7 @@ function sendJson(res, response) {
|
|
|
54
56
|
* @param cfg - Active RAG configuration (used by quirk endpoints).
|
|
55
57
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
56
58
|
*/
|
|
57
|
-
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
|
|
59
|
+
export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder) {
|
|
58
60
|
return async (req, res) => {
|
|
59
61
|
const url = req.url ?? "/";
|
|
60
62
|
const method = req.method ?? "GET";
|
|
@@ -71,7 +73,7 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
|
|
|
71
73
|
if (method === "OPTIONS") {
|
|
72
74
|
res.writeHead(204, {
|
|
73
75
|
"Access-Control-Allow-Origin": "*",
|
|
74
|
-
"Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
|
|
76
|
+
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
|
75
77
|
"Access-Control-Allow-Headers": "Content-Type",
|
|
76
78
|
});
|
|
77
79
|
res.end();
|
|
@@ -99,6 +101,21 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
|
|
|
99
101
|
else if (path === "/api/compare") {
|
|
100
102
|
response = await handleCompare(store, params);
|
|
101
103
|
}
|
|
104
|
+
else if (path === "/api/retrieve") {
|
|
105
|
+
response = await handleRetrieve(store, keywordIndex, getEmbedder, cfg, params);
|
|
106
|
+
}
|
|
107
|
+
else if (path === "/api/indexing/status") {
|
|
108
|
+
response = await handleIndexingStatus(storePath, cwd);
|
|
109
|
+
}
|
|
110
|
+
else if (path === "/api/indexing/reindex" && method === "POST") {
|
|
111
|
+
response = await handleReindex(cwd, cfg, storePath, store, getEmbedder);
|
|
112
|
+
}
|
|
113
|
+
else if (path === "/api/config") {
|
|
114
|
+
response = await handleConfig(cfg);
|
|
115
|
+
}
|
|
116
|
+
else if (path === "/api/embeddings/projection") {
|
|
117
|
+
response = await handleEmbeddingProjection(store, params);
|
|
118
|
+
}
|
|
102
119
|
// File content endpoint (for serving images)
|
|
103
120
|
else if (path === "/api/file" && method === "GET") {
|
|
104
121
|
if (!cwd) {
|
|
@@ -290,6 +307,85 @@ async function handleSearch(keywordIndex, params) {
|
|
|
290
307
|
},
|
|
291
308
|
};
|
|
292
309
|
}
|
|
310
|
+
/**
|
|
311
|
+
* Perform a full vector+hybrid semantic search via the retrieve() pipeline.
|
|
312
|
+
* Accepts GET or POST with query parameters: q, topK, minScore, keywordWeight, hybrid, path, lang, explain.
|
|
313
|
+
* The embedder is lazily initialized on the first call; returns 202 if still initializing.
|
|
314
|
+
*/
|
|
315
|
+
async function handleRetrieve(store, keywordIndex, getEmbedder, cfg, params) {
|
|
316
|
+
if (!getEmbedder) {
|
|
317
|
+
return { status: 503, body: { error: "Embedder not configured for this server instance" } };
|
|
318
|
+
}
|
|
319
|
+
const q = params.get("q") ?? "";
|
|
320
|
+
if (!q.trim()) {
|
|
321
|
+
return { status: 400, body: { error: "Missing 'q' query parameter" } };
|
|
322
|
+
}
|
|
323
|
+
let embedder;
|
|
324
|
+
try {
|
|
325
|
+
embedder = await getEmbedder();
|
|
326
|
+
}
|
|
327
|
+
catch (err) {
|
|
328
|
+
return { status: 503, body: { error: `Embedding model unavailable: ${err.message}. Check that your embedding provider is running.` } };
|
|
329
|
+
}
|
|
330
|
+
const topK = parseInt(params.get("topK") ?? "10", 10);
|
|
331
|
+
const minScore = parseFloat(params.get("minScore") ?? "0.35");
|
|
332
|
+
const keywordWeight = parseFloat(params.get("keywordWeight") ?? "0.4");
|
|
333
|
+
const hybrid = params.get("hybrid") !== "false";
|
|
334
|
+
const explain = params.get("explain") !== "false";
|
|
335
|
+
const pathFilter = params.get("path") ?? undefined;
|
|
336
|
+
const langFilter = params.get("lang") ?? undefined;
|
|
337
|
+
try {
|
|
338
|
+
const results = await retrieve(q, embedder, store, {
|
|
339
|
+
topK,
|
|
340
|
+
minScore,
|
|
341
|
+
keywordIndex,
|
|
342
|
+
keywordWeight,
|
|
343
|
+
hybridEnabled: hybrid,
|
|
344
|
+
queryPrefix: cfg.embedding.queryPrefix,
|
|
345
|
+
explain,
|
|
346
|
+
filter: {
|
|
347
|
+
pathPatterns: pathFilter ? pathFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
|
|
348
|
+
languages: langFilter ? langFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
|
|
349
|
+
},
|
|
350
|
+
});
|
|
351
|
+
return {
|
|
352
|
+
status: 200,
|
|
353
|
+
body: {
|
|
354
|
+
query: q,
|
|
355
|
+
params: { topK, minScore, keywordWeight, hybrid, queryPrefix: cfg.embedding.queryPrefix },
|
|
356
|
+
results: results.map((r) => ({
|
|
357
|
+
chunk: {
|
|
358
|
+
id: r.chunk.id,
|
|
359
|
+
filePath: r.chunk.metadata.filePath,
|
|
360
|
+
startLine: r.chunk.metadata.startLine,
|
|
361
|
+
endLine: r.chunk.metadata.endLine,
|
|
362
|
+
language: r.chunk.metadata.language,
|
|
363
|
+
content: r.chunk.content,
|
|
364
|
+
description: r.chunk.description,
|
|
365
|
+
},
|
|
366
|
+
score: Math.round(r.score * 1000) / 1000,
|
|
367
|
+
explanation: r.explanation
|
|
368
|
+
? {
|
|
369
|
+
scoreBreakdown: {
|
|
370
|
+
vectorScore: r.explanation.scoreBreakdown.vectorScore,
|
|
371
|
+
keywordScore: r.explanation.scoreBreakdown.keywordScore,
|
|
372
|
+
rawVectorScore: r.explanation.scoreBreakdown.rawVectorScore,
|
|
373
|
+
rawKeywordScore: r.explanation.scoreBreakdown.rawKeywordScore,
|
|
374
|
+
keywordWeight: r.explanation.scoreBreakdown.keywordWeight,
|
|
375
|
+
vectorRank: r.explanation.scoreBreakdown.vectorRank,
|
|
376
|
+
keywordRank: r.explanation.scoreBreakdown.keywordRank,
|
|
377
|
+
},
|
|
378
|
+
matchedTerms: r.explanation.matchedTerms,
|
|
379
|
+
}
|
|
380
|
+
: undefined,
|
|
381
|
+
})),
|
|
382
|
+
},
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
catch (err) {
|
|
386
|
+
return { status: 500, body: { error: `Retrieval failed: ${err.message}` } };
|
|
387
|
+
}
|
|
388
|
+
}
|
|
293
389
|
/** Fetch multiple chunks by their comma-separated IDs (`ids` query param) for side-by-side comparison. */
|
|
294
390
|
async function handleCompare(store, params) {
|
|
295
391
|
const idsParam = params.get("ids") ?? "";
|
|
@@ -301,6 +397,126 @@ async function handleCompare(store, params) {
|
|
|
301
397
|
const chunks = allChunks.filter((c) => ids.includes(c.id ?? ""));
|
|
302
398
|
return { status: 200, body: { chunks } };
|
|
303
399
|
}
|
|
400
|
+
/**
|
|
401
|
+
* Return indexing status — manifest stats, staleness, and a placeholder for watcher state.
|
|
402
|
+
*/
|
|
403
|
+
async function handleIndexingStatus(storePath, cwd) {
|
|
404
|
+
const manifestPath = join(storePath, "manifest.json");
|
|
405
|
+
let manifest = null;
|
|
406
|
+
try {
|
|
407
|
+
manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
|
|
408
|
+
}
|
|
409
|
+
catch { /* no manifest yet */ }
|
|
410
|
+
let staleFileCount = 0;
|
|
411
|
+
let totalChunks = 0;
|
|
412
|
+
let totalFiles = 0;
|
|
413
|
+
let lastIndexedAt = null;
|
|
414
|
+
let schemaVersion = manifest?.schemaVersion ?? 0;
|
|
415
|
+
if (manifest?.files) {
|
|
416
|
+
const storedFiles = manifest.files;
|
|
417
|
+
totalFiles = Object.keys(storedFiles).length;
|
|
418
|
+
totalChunks = Object.values(storedFiles).reduce((sum, f) => sum + (f.chunkCount ?? 0), 0);
|
|
419
|
+
}
|
|
420
|
+
if (manifest?.lastIndexedAt) {
|
|
421
|
+
lastIndexedAt = new Date(manifest.lastIndexedAt).toISOString();
|
|
422
|
+
}
|
|
423
|
+
// Count stale files by comparing manifest file list against current disk state
|
|
424
|
+
if (cwd && manifest?.files) {
|
|
425
|
+
const storedFiles = manifest.files;
|
|
426
|
+
for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
|
|
427
|
+
try {
|
|
428
|
+
const fullPath = filePath;
|
|
429
|
+
const content = readFileSync(fullPath, "utf-8");
|
|
430
|
+
const hash = createHash("sha256").update(content).digest("hex");
|
|
431
|
+
if (hash !== fileMeta.hash)
|
|
432
|
+
staleFileCount++;
|
|
433
|
+
}
|
|
434
|
+
catch {
|
|
435
|
+
staleFileCount++; // file was deleted or unreadable
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
return {
|
|
440
|
+
status: 200,
|
|
441
|
+
body: {
|
|
442
|
+
manifest: { totalChunks, totalFiles, schemaVersion, lastIndexedAt },
|
|
443
|
+
staleFileCount,
|
|
444
|
+
watcherActive: false,
|
|
445
|
+
},
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Trigger a one-shot reindex pass in the background.
|
|
450
|
+
*/
|
|
451
|
+
async function handleReindex(cwd, cfg, storePath, store, getEmbedder) {
|
|
452
|
+
try {
|
|
453
|
+
const { runIndexPass } = await import("../indexer.js");
|
|
454
|
+
const embedder = getEmbedder ? await getEmbedder() : undefined;
|
|
455
|
+
if (!embedder) {
|
|
456
|
+
return { status: 503, body: { error: "Embedder not available" } };
|
|
457
|
+
}
|
|
458
|
+
runIndexPass({ cwd, storePath, config: cfg, store, embedder }).catch((err) => {
|
|
459
|
+
console.error("Background reindex failed:", err);
|
|
460
|
+
});
|
|
461
|
+
return { status: 200, body: { started: true } };
|
|
462
|
+
}
|
|
463
|
+
catch (err) {
|
|
464
|
+
return { status: 500, body: { error: `Failed to start reindex: ${err.message}` } };
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Return the effective configuration with API keys redacted.
|
|
469
|
+
*/
|
|
470
|
+
function handleConfig(cfg) {
|
|
471
|
+
const redacted = JSON.parse(JSON.stringify(cfg));
|
|
472
|
+
redactKeys(redacted);
|
|
473
|
+
return { status: 200, body: { config: redacted } };
|
|
474
|
+
}
|
|
475
|
+
function redactKeys(obj) {
|
|
476
|
+
for (const key of Object.keys(obj)) {
|
|
477
|
+
if (key.toLowerCase().includes("apikey") || key === "apiKey") {
|
|
478
|
+
obj[key] = "***";
|
|
479
|
+
}
|
|
480
|
+
else if (typeof obj[key] === "object" && obj[key] !== null) {
|
|
481
|
+
redactKeys(obj[key]);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Project chunk embeddings to 2D via PCA for the Embedding Space Explorer.
|
|
487
|
+
*/
|
|
488
|
+
async function handleEmbeddingProjection(store, params) {
|
|
489
|
+
const maxChunks = parseInt(params.get("maxChunks") ?? "5000", 10);
|
|
490
|
+
try {
|
|
491
|
+
const chunks = await store.getChunksWithEmbeddings(maxChunks);
|
|
492
|
+
if (chunks.length === 0) {
|
|
493
|
+
return { status: 200, body: { points: [], totalChunks: 0 } };
|
|
494
|
+
}
|
|
495
|
+
if (chunks.length === 1) {
|
|
496
|
+
return { status: 200, body: { points: [{ id: chunks[0].id, x: 0.5, y: 0.5, filePath: chunks[0].filePath, startLine: chunks[0].startLine, endLine: chunks[0].endLine, language: chunks[0].language, description: chunks[0].description }], totalChunks: 1, displayedChunks: 1 } };
|
|
497
|
+
}
|
|
498
|
+
const { computePCA } = await import("./pca.js");
|
|
499
|
+
const vectors = chunks.map(c => c.embedding);
|
|
500
|
+
const projected = computePCA(vectors);
|
|
501
|
+
const points = chunks.map((c, i) => ({
|
|
502
|
+
id: c.id,
|
|
503
|
+
x: projected[i].x,
|
|
504
|
+
y: projected[i].y,
|
|
505
|
+
filePath: c.filePath,
|
|
506
|
+
startLine: c.startLine,
|
|
507
|
+
endLine: c.endLine,
|
|
508
|
+
language: c.language,
|
|
509
|
+
description: c.description,
|
|
510
|
+
}));
|
|
511
|
+
return {
|
|
512
|
+
status: 200,
|
|
513
|
+
body: { points, totalChunks: chunks.length, displayedChunks: points.length },
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
catch (err) {
|
|
517
|
+
return { status: 500, body: { error: `Projection failed: ${err.message}` } };
|
|
518
|
+
}
|
|
519
|
+
}
|
|
304
520
|
// ── File Content API ──────────────────────────────────────────────────
|
|
305
521
|
/** Resolve a user-supplied file path against the workspace root, preventing directory traversal outside `cwd`. Returns `null` when the path escapes the workspace. */
|
|
306
522
|
function resolvePath(cwd, filePath) {
|
package/dist/web/pca.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-contained, zero-dependency PCA implementation for 2D embedding projection.
|
|
3
|
+
*/
|
|
4
|
+
export function computePCA(vectors) {
|
|
5
|
+
const n = vectors.length;
|
|
6
|
+
if (n === 0)
|
|
7
|
+
return [];
|
|
8
|
+
const dim = vectors[0].length;
|
|
9
|
+
if (n === 1)
|
|
10
|
+
return [{ x: 0.5, y: 0.5 }];
|
|
11
|
+
// 1. Compute column means
|
|
12
|
+
const means = new Array(dim).fill(0);
|
|
13
|
+
for (let i = 0; i < n; i++) {
|
|
14
|
+
for (let j = 0; j < dim; j++) {
|
|
15
|
+
means[j] += vectors[i][j];
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
for (let j = 0; j < dim; j++)
|
|
19
|
+
means[j] /= n;
|
|
20
|
+
// 2. Center data
|
|
21
|
+
const centered = vectors.map(v => v.map((val, j) => val - means[j]));
|
|
22
|
+
// 3. Compute covariance matrix (dim x dim), upper triangle
|
|
23
|
+
const cov = Array.from({ length: dim }, () => new Array(dim).fill(0));
|
|
24
|
+
for (let i = 0; i < n; i++) {
|
|
25
|
+
for (let j = 0; j < dim; j++) {
|
|
26
|
+
for (let k = j; k < dim; k++) {
|
|
27
|
+
cov[j][k] += centered[i][j] * centered[i][k];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
for (let j = 0; j < dim; j++) {
|
|
32
|
+
for (let k = j; k < dim; k++) {
|
|
33
|
+
cov[j][k] /= n - 1;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// 4. Power iteration to find top-2 eigenvectors
|
|
37
|
+
const pc1 = powerIteration(cov, dim, 50);
|
|
38
|
+
// Deflate: subtract PC1's contribution to find PC2
|
|
39
|
+
const deflated = cov.map((row, i) => {
|
|
40
|
+
const pc1DotRow = pc1.reduce((sum, v, idx) => sum + v * cov[i][idx], 0);
|
|
41
|
+
const pc1NormSq = pc1.reduce((sum, v) => sum + v * v, 0);
|
|
42
|
+
return row.map((val, j) => val - (pc1DotRow / pc1NormSq) * pc1[j]);
|
|
43
|
+
});
|
|
44
|
+
const pc2 = powerIteration(deflated, dim, 50);
|
|
45
|
+
// 5. Project centered data onto PCs
|
|
46
|
+
const projected = centered.map(v => ({
|
|
47
|
+
x: v.reduce((sum, val, j) => sum + val * pc1[j], 0),
|
|
48
|
+
y: v.reduce((sum, val, j) => sum + val * pc2[j], 0),
|
|
49
|
+
}));
|
|
50
|
+
// 6. Normalize to [0, 1]
|
|
51
|
+
const xs = projected.map(p => p.x);
|
|
52
|
+
const ys = projected.map(p => p.y);
|
|
53
|
+
const minX = Math.min(...xs);
|
|
54
|
+
const maxX = Math.max(...xs);
|
|
55
|
+
const minY = Math.min(...ys);
|
|
56
|
+
const maxY = Math.max(...ys);
|
|
57
|
+
const rangeX = maxX - minX || 1;
|
|
58
|
+
const rangeY = maxY - minY || 1;
|
|
59
|
+
return projected.map(p => ({
|
|
60
|
+
x: (p.x - minX) / rangeX,
|
|
61
|
+
y: (p.y - minY) / rangeY,
|
|
62
|
+
}));
|
|
63
|
+
}
|
|
64
|
+
/** Power iteration to find the dominant eigenvector of a symmetric matrix. */
|
|
65
|
+
function powerIteration(matrix, dim, maxIter) {
|
|
66
|
+
let v = new Array(dim).fill(0).map(() => Math.random() * 2 - 1);
|
|
67
|
+
const normalize = (vec) => {
|
|
68
|
+
const len = Math.sqrt(vec.reduce((s, val) => s + val * val, 0));
|
|
69
|
+
return len > 1e-10 ? vec.map(val => val / len) : vec;
|
|
70
|
+
};
|
|
71
|
+
for (let iter = 0; iter < maxIter; iter++) {
|
|
72
|
+
const w = new Array(dim).fill(0);
|
|
73
|
+
for (let i = 0; i < dim; i++) {
|
|
74
|
+
for (let j = 0; j < dim; j++) {
|
|
75
|
+
w[i] += matrix[i][j] * v[j];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
v = normalize(w);
|
|
79
|
+
if (iter > 5) {
|
|
80
|
+
const change = Math.sqrt(v.reduce((s, val, i) => s + (val - w[i]) * (val - w[i]), 0));
|
|
81
|
+
if (change < 1e-6)
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return v;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=pca.js.map
|