opencode-rag-plugin 1.18.1 → 1.19.0

package/dist/web/api.js

@@ -1,8 +1,10 @@
 import { readFileSync } from "node:fs";
-import { extname, resolve as resolvePathModule } from "node:path";
+import { extname, join, resolve as resolvePathModule } from "node:path";
+import { createHash } from "node:crypto";
 import { listSessions, getSession, deleteSession, compareSessions, validateSessionID } from "../eval/storage.js";
 import { analyzeTokenUsage, compareTokenAnalyses, projectTokenSavings } from "../eval/token-analysis.js";
 import { listQuirks, lintQuirks, removeQuirk } from "../quirks/quirk-store.js";
+import { retrieve } from "../retriever/retriever.js";
 const FILE_MIME_TYPES = {
     ".png": "image/png",
     ".jpg": "image/jpeg",
@@ -54,7 +56,7 @@ function sendJson(res, response) {
  * @param cfg          - Active RAG configuration (used by quirk endpoints).
  * @returns An async handler that returns `true` when a route matched or `false` otherwise.
  */
-export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
+export function createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder) {
     return async (req, res) => {
         const url = req.url ?? "/";
         const method = req.method ?? "GET";
@@ -71,7 +73,7 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
         if (method === "OPTIONS") {
             res.writeHead(204, {
                 "Access-Control-Allow-Origin": "*",
-                "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
+                "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
                 "Access-Control-Allow-Headers": "Content-Type",
             });
             res.end();
@@ -99,6 +101,21 @@ export function createApiHandler(store, keywordIndex, storePath, cwd, cfg) {
             else if (path === "/api/compare") {
                 response = await handleCompare(store, params);
             }
+            else if (path === "/api/retrieve") {
+                response = await handleRetrieve(store, keywordIndex, getEmbedder, cfg, params);
+            }
+            else if (path === "/api/indexing/status") {
+                response = await handleIndexingStatus(storePath, cwd);
+            }
+            else if (path === "/api/indexing/reindex" && method === "POST") {
+                response = await handleReindex(cwd, cfg, storePath, store, getEmbedder);
+            }
+            else if (path === "/api/config") {
+                response = await handleConfig(cfg);
+            }
+            else if (path === "/api/embeddings/projection") {
+                response = await handleEmbeddingProjection(store, params);
+            }
             // File content endpoint (for serving images)
             else if (path === "/api/file" && method === "GET") {
                 if (!cwd) {
@@ -290,6 +307,85 @@ async function handleSearch(keywordIndex, params) {
         },
     };
 }
+/**
+ * Perform a full vector+hybrid semantic search via the retrieve() pipeline.
+ * Accepts GET or POST with query parameters: q, topK, minScore, keywordWeight, hybrid, path, lang, explain.
+ * The embedder is lazily initialized on the first call; returns 202 if still initializing.
+ */
+async function handleRetrieve(store, keywordIndex, getEmbedder, cfg, params) {
+    if (!getEmbedder) {
+        return { status: 503, body: { error: "Embedder not configured for this server instance" } };
+    }
+    const q = params.get("q") ?? "";
+    if (!q.trim()) {
+        return { status: 400, body: { error: "Missing 'q' query parameter" } };
+    }
+    let embedder;
+    try {
+        embedder = await getEmbedder();
+    }
+    catch (err) {
+        return { status: 503, body: { error: `Embedding model unavailable: ${err.message}. Check that your embedding provider is running.` } };
+    }
+    const topK = parseInt(params.get("topK") ?? "10", 10);
+    const minScore = parseFloat(params.get("minScore") ?? "0.35");
+    const keywordWeight = parseFloat(params.get("keywordWeight") ?? "0.4");
+    const hybrid = params.get("hybrid") !== "false";
+    const explain = params.get("explain") !== "false";
+    const pathFilter = params.get("path") ?? undefined;
+    const langFilter = params.get("lang") ?? undefined;
+    try {
+        const results = await retrieve(q, embedder, store, {
+            topK,
+            minScore,
+            keywordIndex,
+            keywordWeight,
+            hybridEnabled: hybrid,
+            queryPrefix: cfg.embedding.queryPrefix,
+            explain,
+            filter: {
+                pathPatterns: pathFilter ? pathFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
+                languages: langFilter ? langFilter.split(",").map((s) => s.trim()).filter(Boolean) : undefined,
+            },
+        });
+        return {
+            status: 200,
+            body: {
+                query: q,
+                params: { topK, minScore, keywordWeight, hybrid, queryPrefix: cfg.embedding.queryPrefix },
+                results: results.map((r) => ({
+                    chunk: {
+                        id: r.chunk.id,
+                        filePath: r.chunk.metadata.filePath,
+                        startLine: r.chunk.metadata.startLine,
+                        endLine: r.chunk.metadata.endLine,
+                        language: r.chunk.metadata.language,
+                        content: r.chunk.content,
+                        description: r.chunk.description,
+                    },
+                    score: Math.round(r.score * 1000) / 1000,
+                    explanation: r.explanation
+                        ? {
+                            scoreBreakdown: {
+                                vectorScore: r.explanation.scoreBreakdown.vectorScore,
+                                keywordScore: r.explanation.scoreBreakdown.keywordScore,
+                                rawVectorScore: r.explanation.scoreBreakdown.rawVectorScore,
+                                rawKeywordScore: r.explanation.scoreBreakdown.rawKeywordScore,
+                                keywordWeight: r.explanation.scoreBreakdown.keywordWeight,
+                                vectorRank: r.explanation.scoreBreakdown.vectorRank,
+                                keywordRank: r.explanation.scoreBreakdown.keywordRank,
+                            },
+                            matchedTerms: r.explanation.matchedTerms,
+                        }
+                        : undefined,
+                })),
+            },
+        };
+    }
+    catch (err) {
+        return { status: 500, body: { error: `Retrieval failed: ${err.message}` } };
+    }
+}
 /** Fetch multiple chunks by their comma-separated IDs (`ids` query param) for side-by-side comparison. */
 async function handleCompare(store, params) {
     const idsParam = params.get("ids") ?? "";
@@ -301,6 +397,126 @@ async function handleCompare(store, params) {
     const chunks = allChunks.filter((c) => ids.includes(c.id ?? ""));
     return { status: 200, body: { chunks } };
 }
+/**
+ * Return indexing status — manifest stats, staleness, and a placeholder for watcher state.
+ */
+async function handleIndexingStatus(storePath, cwd) {
+    const manifestPath = join(storePath, "manifest.json");
+    let manifest = null;
+    try {
+        manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
+    }
+    catch { /* no manifest yet */ }
+    let staleFileCount = 0;
+    let totalChunks = 0;
+    let totalFiles = 0;
+    let lastIndexedAt = null;
+    let schemaVersion = manifest?.schemaVersion ?? 0;
+    if (manifest?.files) {
+        const storedFiles = manifest.files;
+        totalFiles = Object.keys(storedFiles).length;
+        totalChunks = Object.values(storedFiles).reduce((sum, f) => sum + (f.chunkCount ?? 0), 0);
+    }
+    if (manifest?.lastIndexedAt) {
+        lastIndexedAt = new Date(manifest.lastIndexedAt).toISOString();
+    }
+    // Count stale files by comparing manifest file list against current disk state
+    if (cwd && manifest?.files) {
+        const storedFiles = manifest.files;
+        for (const [filePath, fileMeta] of Object.entries(storedFiles)) {
+            try {
+                const fullPath = filePath;
+                const content = readFileSync(fullPath, "utf-8");
+                const hash = createHash("sha256").update(content).digest("hex");
+                if (hash !== fileMeta.hash)
+                    staleFileCount++;
+            }
+            catch {
+                staleFileCount++; // file was deleted or unreadable
+            }
+        }
+    }
+    return {
+        status: 200,
+        body: {
+            manifest: { totalChunks, totalFiles, schemaVersion, lastIndexedAt },
+            staleFileCount,
+            watcherActive: false,
+        },
+    };
+}
+/**
+ * Trigger a one-shot reindex pass in the background.
+ */
+async function handleReindex(cwd, cfg, storePath, store, getEmbedder) {
+    try {
+        const { runIndexPass } = await import("../indexer.js");
+        const embedder = getEmbedder ? await getEmbedder() : undefined;
+        if (!embedder) {
+            return { status: 503, body: { error: "Embedder not available" } };
+        }
+        runIndexPass({ cwd, storePath, config: cfg, store, embedder }).catch((err) => {
+            console.error("Background reindex failed:", err);
+        });
+        return { status: 200, body: { started: true } };
+    }
+    catch (err) {
+        return { status: 500, body: { error: `Failed to start reindex: ${err.message}` } };
+    }
+}
+/**
+ * Return the effective configuration with API keys redacted.
+ */
+function handleConfig(cfg) {
+    const redacted = JSON.parse(JSON.stringify(cfg));
+    redactKeys(redacted);
+    return { status: 200, body: { config: redacted } };
+}
+function redactKeys(obj) {
+    for (const key of Object.keys(obj)) {
+        if (key.toLowerCase().includes("apikey") || key === "apiKey") {
+            obj[key] = "***";
+        }
+        else if (typeof obj[key] === "object" && obj[key] !== null) {
+            redactKeys(obj[key]);
+        }
+    }
+}
+/**
+ * Project chunk embeddings to 2D via PCA for the Embedding Space Explorer.
+ */
+async function handleEmbeddingProjection(store, params) {
+    const maxChunks = parseInt(params.get("maxChunks") ?? "5000", 10);
+    try {
+        const chunks = await store.getChunksWithEmbeddings(maxChunks);
+        if (chunks.length === 0) {
+            return { status: 200, body: { points: [], totalChunks: 0 } };
+        }
+        if (chunks.length === 1) {
+            return { status: 200, body: { points: [{ id: chunks[0].id, x: 0.5, y: 0.5, filePath: chunks[0].filePath, startLine: chunks[0].startLine, endLine: chunks[0].endLine, language: chunks[0].language, description: chunks[0].description }], totalChunks: 1, displayedChunks: 1 } };
+        }
+        const { computePCA } = await import("./pca.js");
+        const vectors = chunks.map(c => c.embedding);
+        const projected = computePCA(vectors);
+        const points = chunks.map((c, i) => ({
+            id: c.id,
+            x: projected[i].x,
+            y: projected[i].y,
+            filePath: c.filePath,
+            startLine: c.startLine,
+            endLine: c.endLine,
+            language: c.language,
+            description: c.description,
+        }));
+        return {
+            status: 200,
+            body: { points, totalChunks: chunks.length, displayedChunks: points.length },
+        };
+    }
+    catch (err) {
+        return { status: 500, body: { error: `Projection failed: ${err.message}` } };
+    }
+}
 // ── File Content API ──────────────────────────────────────────────────
 /** Resolve a user-supplied file path against the workspace root, preventing directory traversal outside `cwd`. Returns `null` when the path escapes the workspace. */
 function resolvePath(cwd, filePath) {

package/dist/web/pca.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Self-contained, zero-dependency PCA implementation for 2D embedding projection.
+ */
+export declare function computePCA(vectors: number[][]): {
+    x: number;
+    y: number;
+}[];

package/dist/web/pca.js

@@ -0,0 +1,87 @@
+/**
+ * Self-contained, zero-dependency PCA implementation for 2D embedding projection.
+ */
+export function computePCA(vectors) {
+    const n = vectors.length;
+    if (n === 0)
+        return [];
+    const dim = vectors[0].length;
+    if (n === 1)
+        return [{ x: 0.5, y: 0.5 }];
+    // 1. Compute column means
+    const means = new Array(dim).fill(0);
+    for (let i = 0; i < n; i++) {
+        for (let j = 0; j < dim; j++) {
+            means[j] += vectors[i][j];
+        }
+    }
+    for (let j = 0; j < dim; j++)
+        means[j] /= n;
+    // 2. Center data
+    const centered = vectors.map(v => v.map((val, j) => val - means[j]));
+    // 3. Compute covariance matrix (dim x dim), upper triangle
+    const cov = Array.from({ length: dim }, () => new Array(dim).fill(0));
+    for (let i = 0; i < n; i++) {
+        for (let j = 0; j < dim; j++) {
+            for (let k = j; k < dim; k++) {
+                cov[j][k] += centered[i][j] * centered[i][k];
+            }
+        }
+    }
+    for (let j = 0; j < dim; j++) {
+        for (let k = j; k < dim; k++) {
+            cov[j][k] /= n - 1;
+        }
+    }
+    // 4. Power iteration to find top-2 eigenvectors
+    const pc1 = powerIteration(cov, dim, 50);
+    // Deflate: subtract PC1's contribution to find PC2
+    const deflated = cov.map((row, i) => {
+        const pc1DotRow = pc1.reduce((sum, v, idx) => sum + v * cov[i][idx], 0);
+        const pc1NormSq = pc1.reduce((sum, v) => sum + v * v, 0);
+        return row.map((val, j) => val - (pc1DotRow / pc1NormSq) * pc1[j]);
+    });
+    const pc2 = powerIteration(deflated, dim, 50);
+    // 5. Project centered data onto PCs
+    const projected = centered.map(v => ({
+        x: v.reduce((sum, val, j) => sum + val * pc1[j], 0),
+        y: v.reduce((sum, val, j) => sum + val * pc2[j], 0),
+    }));
+    // 6. Normalize to [0, 1]
+    const xs = projected.map(p => p.x);
+    const ys = projected.map(p => p.y);
+    const minX = Math.min(...xs);
+    const maxX = Math.max(...xs);
+    const minY = Math.min(...ys);
+    const maxY = Math.max(...ys);
+    const rangeX = maxX - minX || 1;
+    const rangeY = maxY - minY || 1;
+    return projected.map(p => ({
+        x: (p.x - minX) / rangeX,
+        y: (p.y - minY) / rangeY,
+    }));
+}
+/** Power iteration to find the dominant eigenvector of a symmetric matrix. */
+function powerIteration(matrix, dim, maxIter) {
+    let v = new Array(dim).fill(0).map(() => Math.random() * 2 - 1);
+    const normalize = (vec) => {
+        const len = Math.sqrt(vec.reduce((s, val) => s + val * val, 0));
+        return len > 1e-10 ? vec.map(val => val / len) : vec;
+    };
+    for (let iter = 0; iter < maxIter; iter++) {
+        const w = new Array(dim).fill(0);
+        for (let i = 0; i < dim; i++) {
+            for (let j = 0; j < dim; j++) {
+                w[i] += matrix[i][j] * v[j];
+            }
+        }
+        v = normalize(w);
+        if (iter > 5) {
+            const change = Math.sqrt(v.reduce((s, val, i) => s + (val - w[i]) * (val - w[i]), 0));
+            if (change < 1e-6)
+                break;
+        }
+    }
+    return v;
+}
+//# sourceMappingURL=pca.js.map

package/dist/web/server.js

@@ -2,13 +2,13 @@
  * @fileoverview HTTP server for the OpenCodeRAG Web UI dashboard with static asset serving and REST API routing.
  */
 import { createServer } from "node:http";
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { dirname, extname, join, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { LanceDbStore } from "../vectorstore/lancedb.js";
 import { KeywordIndex } from "../retriever/keyword-index.js";
 import { createApiHandler } from "./api.js";
-import { getStaticHtml } from "./static.js";
+import { getStaticHtml, resolveDistAsset } from "./static.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const uiDir = join(__dirname, "ui");
 const MIME_TYPES = {
@@ -59,8 +59,17 @@ function serveUiAsset(res, filePath) {
 export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cfg) {
     const store = new LanceDbStore(storePath, vectorDimension);
     const keywordIndex = await KeywordIndex.load(storePath);
+    // Lazy embedder for /api/retrieve — initialized on first use
+    let embedderPromise = null;
+    async function getEmbedder() {
+        if (!embedderPromise) {
+            const { createEmbedder } = await import("../embedder/factory.js");
+            embedderPromise = Promise.resolve(createEmbedder(cfg));
+        }
+        return embedderPromise;
+    }
     const html = getStaticHtml();
-    const apiHandler = createApiHandler(store, keywordIndex, storePath, cwd, cfg);
+    const apiHandler = createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder);
     const server = createServer(async (req, res) => {
         const url = req.url ?? "/";
         if (url === "/" || url === "/index.html") {
@@ -74,13 +83,19 @@ export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cf
                 res.end("Forbidden");
                 return;
             }
-            const assetPath = join(uiDir, decoded);
-            if (!assetPath.startsWith(uiDir + sep)) {
-                res.writeHead(403, { "Content-Type": "text/plain" });
-                res.end("Forbidden");
+            // Try production build output first, fall back to dev source
+            const distAsset = resolveDistAsset(decoded);
+            if (distAsset) {
+                serveUiAsset(res, distAsset);
+                return;
+            }
+            const devPath = join(uiDir, decoded);
+            if (devPath.startsWith(uiDir + sep) && existsSync(devPath)) {
+                serveUiAsset(res, devPath);
                 return;
             }
-            serveUiAsset(res, assetPath);
+            res.writeHead(404, { "Content-Type": "text/plain" });
+            res.end("Not Found");
             return;
         }
         if (url.startsWith("/api/")) {

package/dist/web/static.d.ts

@@ -1,9 +1,14 @@
 /**
  * Read and cache the Web UI `index.html` from disk.
  *
- * The HTML is read once from the `ui/` directory adjacent to this module
- * and cached in memory for subsequent calls.
+ * Reads from the Vite production build output (`dist/web/ui/index.html`) when available,
+ * falling back to the development source (`src/web/ui/index.html`).
  *
  * @returns The full HTML string of the Web UI entry page.
  */
 export declare function getStaticHtml(): string;
+/**
+ * Resolve the path to a built UI asset from the Vite output directory.
+ * Returns null if the path escapes the dist directory or does not exist.
+ */
+export declare function resolveDistAsset(path: string): string | null;

package/dist/web/static.js

@@ -1,24 +1,48 @@
 /**
  * @fileoverview Static HTML file reader and cache for the Web UI entry page.
+ * Supports both the Vite production build output and the development source.
  */
-import { readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+/**
+ * Resolve the project root directory regardless of whether running from source (tsx)
+ * or compiled output (dist/). The static.ts module lives at src/web/static.ts or
+ * dist/web/static.ts, so going up two directories reaches the project root.
+ */
+function projectRoot() {
+    return resolve(__dirname, "..", "..");
+}
 let cachedHtml = null;
 /**
  * Read and cache the Web UI `index.html` from disk.
  *
- * The HTML is read once from the `ui/` directory adjacent to this module
- * and cached in memory for subsequent calls.
+ * Reads from the Vite production build output (`dist/web/ui/index.html`) when available,
+ * falling back to the development source (`src/web/ui/index.html`).
  *
  * @returns The full HTML string of the Web UI entry page.
  */
 export function getStaticHtml() {
     if (cachedHtml)
         return cachedHtml;
-    const __dirname = dirname(fileURLToPath(import.meta.url));
-    const htmlPath = join(__dirname, "ui", "index.html");
-    cachedHtml = readFileSync(htmlPath, "utf-8");
+    const root = projectRoot();
+    const prodPath = join(root, "dist", "web", "ui", "index.html");
+    const devPath = join(root, "src", "web", "ui", "index.html");
+    const targetPath = existsSync(prodPath) ? prodPath : devPath;
+    cachedHtml = readFileSync(targetPath, "utf-8");
     return cachedHtml;
 }
+/**
+ * Resolve the path to a built UI asset from the Vite output directory.
+ * Returns null if the path escapes the dist directory or does not exist.
+ */
+export function resolveDistAsset(path) {
+    const root = projectRoot();
+    const distDir = join(root, "dist", "web", "ui").replace(/\\/g, "/");
+    const resolved = join(distDir, path).replace(/\\/g, "/");
+    if (!resolved.startsWith(distDir + "/"))
+        return null;
+    return existsSync(resolved) ? resolved : null;
+}
 //# sourceMappingURL=static.js.map