@jagilber-org/index-server 1.22.0

package/dist/dashboard/server/routes/embeddings.routes.js

@@ -0,0 +1,246 @@
+"use strict";
+/**
+ * Embeddings Routes
+ * Route: GET /embeddings/projection — PCA-project embeddings to 2D for visualization.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEmbeddingsRoutes = createEmbeddingsRoutes;
+const express_1 = require("express");
+const node_fs_1 = __importDefault(require("node:fs"));
+const runtimeConfig_js_1 = require("../../../config/runtimeConfig.js");
+const embeddingService_js_1 = require("../../../services/embeddingService.js");
+const adminAuth_js_1 = require("./adminAuth.js");
+// ---------------------------------------------------------------------------
+// Category derivation from instruction ID (shared rules)
+// ---------------------------------------------------------------------------
+const categoryRules_js_1 = require("../../../services/categoryRules.js");
+// ---------------------------------------------------------------------------
+// PCA via power iteration (no dependencies)
+// ---------------------------------------------------------------------------
+function mean(vectors) {
+    const n = vectors.length;
+    const d = vectors[0].length;
+    const m = new Array(d).fill(0);
+    for (const v of vectors)
+        for (let i = 0; i < d; i++)
+            m[i] += v[i];
+    for (let i = 0; i < d; i++)
+        m[i] /= n;
+    return m;
+}
+function subtract(a, b) {
+    return a.map((v, i) => v - b[i]);
+}
+function dot(a, b) {
+    let s = 0;
+    for (let i = 0; i < a.length; i++)
+        s += a[i] * b[i];
+    return s;
+}
+function norm(v) {
+    return Math.sqrt(dot(v, v));
+}
+function scale(v, s) {
+    return v.map(x => x * s);
+}
+/** Compute the top principal component via power iteration. */
+function powerIteration(centered, dims, iterations = 100) {
+    // random init
+    let pc = Array.from({ length: dims }, () => Math.random() - 0.5);
+    const n = norm(pc);
+    if (n > 0)
+        pc = scale(pc, 1 / n);
+    for (let iter = 0; iter < iterations; iter++) {
+        const newPc = new Array(dims).fill(0);
+        for (const row of centered) {
+            const d = dot(row, pc);
+            for (let j = 0; j < dims; j++)
+                newPc[j] += d * row[j];
+        }
+        const len = norm(newPc);
+        if (len === 0)
+            break;
+        pc = scale(newPc, 1 / len);
+    }
+    return pc;
+}
+/** Deflate: remove component along given direction */
+function deflate(centered, pc) {
+    return centered.map(row => {
+        const proj = dot(row, pc);
+        return row.map((v, i) => v - proj * pc[i]);
+    });
+}
+/** PCA project to 2D */
+function pcaProject(vectors) {
+    if (vectors.length === 0)
+        return [];
+    const dims = vectors[0].length;
+    if (vectors.length === 1) {
+        return [{ x: 0, y: 0 }];
+    }
+    const mu = mean(vectors);
+    let centered = vectors.map(v => subtract(v, mu));
+    const pc1 = powerIteration(centered, dims);
+    centered = deflate(centered, pc1);
+    const pc2 = powerIteration(centered, dims);
+    return vectors.map(v => {
+        const c = subtract(v, mu);
+        return { x: dot(c, pc1), y: dot(c, pc2) };
+    });
+}
+// ---------------------------------------------------------------------------
+// Cosine similarity stats
+// ---------------------------------------------------------------------------
+function cosineSimilarity(a, b) {
+    const na = norm(a);
+    const nb = norm(b);
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot(a, b) / (na * nb);
+}
+function computeStats(ids, vectors, topK = 10) {
+    let sumSim = 0;
+    let minSim = 1;
+    let maxSim = -1;
+    let pairCount = 0;
+    let sumNorm = 0;
+    const topPairs = [];
+    for (let i = 0; i < vectors.length; i++) {
+        sumNorm += norm(vectors[i]);
+    }
+    // Sample up to 5000 pairs for large collections
+    const maxPairs = 5000;
+    const allPairs = (vectors.length * (vectors.length - 1)) / 2;
+    const sampleRate = allPairs > maxPairs ? maxPairs / allPairs : 1;
+    for (let i = 0; i < vectors.length; i++) {
+        for (let j = i + 1; j < vectors.length; j++) {
+            if (sampleRate < 1 && Math.random() > sampleRate)
+                continue;
+            const sim = cosineSimilarity(vectors[i], vectors[j]);
+            sumSim += sim;
+            if (sim < minSim)
+                minSim = sim;
+            if (sim > maxSim)
+                maxSim = sim;
+            pairCount++;
+            if (topPairs.length < topK || sim > topPairs[topPairs.length - 1].similarity) {
+                topPairs.push({ a: ids[i], b: ids[j], similarity: Math.round(sim * 1000) / 1000 });
+                topPairs.sort((a, b) => b.similarity - a.similarity);
+                if (topPairs.length > topK)
+                    topPairs.pop();
+            }
+        }
+    }
+    return {
+        stats: {
+            avgCosineSim: pairCount > 0 ? Math.round((sumSim / pairCount) * 1000) / 1000 : 0,
+            minCosineSim: pairCount > 0 ? Math.round(minSim * 1000) / 1000 : 0,
+            maxCosineSim: pairCount > 0 ? Math.round(maxSim * 1000) / 1000 : 0,
+            avgNorm: vectors.length > 0 ? Math.round((sumNorm / vectors.length) * 1000) / 1000 : 0,
+        },
+        similarPairs: topPairs,
+    };
+}
+// ---------------------------------------------------------------------------
+// Route factory
+// ---------------------------------------------------------------------------
+function createEmbeddingsRoutes(embeddingPathOverride) {
+    const router = (0, express_1.Router)();
+    router.get('/embeddings/projection', (_req, res) => {
+        try {
+            const embeddingPath = embeddingPathOverride ?? (0, runtimeConfig_js_1.getRuntimeConfig)().semantic.embeddingPath ?? '';
+            if (!embeddingPath || !node_fs_1.default.existsSync(embeddingPath)) {
+                return res.status(404).json({
+                    success: false,
+                    error: 'Embeddings file not found',
+                });
+            }
+            const raw = node_fs_1.default.readFileSync(embeddingPath, 'utf-8');
+            const data = JSON.parse(raw);
+            const ids = Object.keys(data.embeddings);
+            const vectors = ids.map(id => data.embeddings[id]);
+            if (ids.length === 0) {
+                return res.json({
+                    success: true,
+                    count: 0,
+                    dimensions: 0,
+                    model: data.modelName,
+                    points: [],
+                    stats: { avgCosineSim: 0, minCosineSim: 0, maxCosineSim: 0, avgNorm: 0 },
+                    similarPairs: [],
+                });
+            }
+            const dims = vectors[0].length;
+            const projected = pcaProject(vectors);
+            const points = ids.map((id, i) => ({
+                id,
+                x: Math.round(projected[i].x * 10000) / 10000,
+                y: Math.round(projected[i].y * 10000) / 10000,
+                category: (0, categoryRules_js_1.deriveCategory)(id),
+                norm: Math.round(norm(vectors[i]) * 10000) / 10000,
+            }));
+            const { stats, similarPairs } = computeStats(ids, vectors);
+            return res.json({
+                success: true,
+                count: ids.length,
+                dimensions: dims,
+                model: data.modelName,
+                points,
+                stats,
+                similarPairs,
+            });
+        }
+        catch (err) {
+            return res.status(500).json({
+                success: false,
+                error: 'Failed to compute projection',
+                message: err.message,
+            });
+        }
+    });
+    // POST /embeddings/compute — trigger embedding computation for all instructions
+    router.post('/embeddings/compute', adminAuth_js_1.dashboardAdminAuth, async (_req, res) => {
+        try {
+            const config = (0, runtimeConfig_js_1.getRuntimeConfig)();
+            const sem = config.semantic;
+            if (!sem.enabled) {
+                return res.status(400).json({
+                    success: false,
+                    error: 'Semantic embeddings are disabled',
+                    hint: 'Set INDEX_SERVER_SEMANTIC_ENABLED=1 and restart the server',
+                });
+            }
+            const state = res.locals.indexState;
+            if (!state || !state.list || state.list.length === 0) {
+                return res.status(400).json({
+                    success: false,
+                    error: 'No instructions loaded in index',
+                });
+            }
+            const start = performance.now();
+            const embeddings = await (0, embeddingService_js_1.getInstructionEmbeddings)(state.list, state.hash, sem.embeddingPath, sem.model, sem.cacheDir, sem.device, sem.localOnly);
+            const elapsed = (performance.now() - start).toFixed(0);
+            const count = Object.keys(embeddings).length;
+            return res.json({
+                success: true,
+                count,
+                model: sem.model,
+                device: sem.device,
+                elapsedMs: Number(elapsed),
+                embeddingPath: sem.embeddingPath,
+            });
+        }
+        catch (err) {
+            return res.status(500).json({
+                success: false,
+                error: 'Failed to compute embeddings',
+                message: err.message,
+            });
+        }
+    });
+    return router;
+}

package/dist/dashboard/server/routes/graph.routes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Graph Routes
+ * Routes: GET /graph/mermaid, GET /graph/categories, GET /graph/instructions, GET /graph/relations
+ */
+import { Router } from 'express';
+export declare function createGraphRoutes(): Router;

package/dist/dashboard/server/routes/graph.routes.js

@@ -0,0 +1,279 @@
+"use strict";
+/**
+ * Graph Routes
+ * Routes: GET /graph/mermaid, GET /graph/categories, GET /graph/instructions, GET /graph/relations
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createGraphRoutes = createGraphRoutes;
+const express_1 = require("express");
+const handlers_graph_js_1 = require("../../../services/handlers.graph.js");
+const runtimeConfig_js_1 = require("../../../config/runtimeConfig.js");
+function createGraphRoutes() {
+    const router = (0, express_1.Router)();
+    /**
+     * GET /api/graph/mermaid - Returns mermaid representation of the instruction graph.
+     * Query params:
+     *   enrich=1        -> include enriched schema (v2) data generation path
+     *   categories=1    -> include explicit category nodes
+     *   usage=1         -> include usageCount when available
+     *   edgeTypes=a,b   -> restrict edge types (comma separated)
+     */
+    router.get('/graph/mermaid', (req, res) => {
+        try {
+            const { enrich, categories, usage, edgeTypes, selectedCategories, selectedIds } = req.query;
+            const includeEdgeTypes = edgeTypes ? edgeTypes.split(',').filter(Boolean) : undefined;
+            const t0 = Date.now();
+            try {
+                // eslint-disable-next-line no-console
+                console.debug('[graph/mermaid][start]', `enrich=${enrich}`, `categories=${categories}`, `usage=${usage}`, `edgeTypes=${edgeTypes || ''}`, `selCats=${selectedCategories || ''}`, `selIds=${selectedIds || ''}`); // lgtm[js/log-injection] — query params for debugging
+            }
+            catch { /* ignore diag logging errors */ }
+            const graph = (0, handlers_graph_js_1.buildGraph)({
+                enrich: enrich === '1' || enrich === 'true',
+                includeCategoryNodes: categories === '1' || categories === 'true',
+                includeUsage: usage === '1' || usage === 'true',
+                includeEdgeTypes,
+                format: 'mermaid'
+            });
+            if (!graph.mermaid) {
+                return res.status(500).json({ success: false, error: 'failed_to_generate_mermaid' });
+            }
+            let mermaidSource = graph.mermaid;
+            const catFilter = selectedCategories?.split(',').filter(Boolean) || [];
+            const idFilter = selectedIds?.split(',').filter(Boolean) || [];
+            let filteredNodeCount;
+            let filteredEdgeCount;
+            let scoped = false;
+            let keptIdsSize = 0;
+            if ((catFilter.length || idFilter.length) && mermaidSource) {
+                try {
+                    const keepIds = new Set();
+                    for (const id of idFilter)
+                        keepIds.add(id);
+                    const wantCategoryNodes = (categories === '1' || categories === 'true');
+                    if (catFilter.length) {
+                        const catSet = new Set(catFilter.map(c => c.toLowerCase()));
+                        if (wantCategoryNodes) {
+                            for (const c of catFilter) {
+                                keepIds.add(`category:${c}`);
+                            }
+                        }
+                        if (graph.meta.graphSchemaVersion === 2) {
+                            for (const nodeRaw of graph.nodes) {
+                                const nodeCats = nodeRaw.categories;
+                                if (Array.isArray(nodeCats) && nodeCats.some(c => catSet.has(c.toLowerCase()))) {
+                                    keepIds.add(nodeRaw.id);
+                                    if (wantCategoryNodes) {
+                                        for (const c of nodeCats) {
+                                            if (catSet.has(c.toLowerCase()))
+                                                keepIds.add(`category:${c}`);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    if (keepIds.size) {
+                        let frontmatterBlock = '';
+                        let remainder = mermaidSource;
+                        if (remainder.startsWith('---\n')) {
+                            const fmMatch = /^---\n[\s\S]*?\n---\n/.exec(remainder);
+                            if (fmMatch) {
+                                frontmatterBlock = fmMatch[0];
+                                remainder = remainder.slice(fmMatch[0].length);
+                            }
+                        }
+                        const lines = remainder.split(/\r?\n/);
+                        const directiveIdx = lines.findIndex(l => /^\s*(flowchart|graph)\b/i.test(l));
+                        let directiveLine = '';
+                        if (directiveIdx >= 0) {
+                            directiveLine = lines[directiveIdx];
+                        }
+                        const nodeIdPattern = Array.from(keepIds).map(id => id.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')).join('|');
+                        const nodeRegex = new RegExp(`^(\\s*)(${nodeIdPattern})\\[`);
+                        const edgeRegex = new RegExp(`^(.*)(${nodeIdPattern})(.*)(${nodeIdPattern})(.*)$`);
+                        const filtered = [];
+                        const styleRegexes = [
+                            /^\s*classDef\s+/i,
+                            /^\s*style\s+[A-Za-z0-9:_-]+\s+/i,
+                            /^\s*class\s+[A-Za-z0-9:_-]+\s+/i,
+                            /^\s*linkStyle\s+\d+/i
+                        ];
+                        for (const ln of lines) {
+                            if (!ln)
+                                continue;
+                            if (directiveLine && ln === directiveLine) {
+                                continue;
+                            }
+                            const trimmed = ln.trim();
+                            if (trimmed.startsWith('%%')) {
+                                filtered.push(ln);
+                                continue;
+                            }
+                            if (styleRegexes.some(r => r.test(trimmed))) {
+                                filtered.push(ln);
+                                continue;
+                            }
+                            if (nodeRegex.test(ln) || edgeRegex.test(ln))
+                                filtered.push(ln);
+                        }
+                        const parts = [];
+                        if (frontmatterBlock)
+                            parts.push(frontmatterBlock.trimEnd());
+                        if (directiveLine)
+                            parts.push(directiveLine);
+                        parts.push(...filtered);
+                        mermaidSource = parts.join('\n');
+                        if ((0, runtimeConfig_js_1.getRuntimeConfig)().logging.verbose) {
+                            // eslint-disable-next-line no-console
+                            console.debug('[graph/mermaid][filter:new]', { selectedIds: idFilter.length, selectedCategories: catFilter.length, kept: keepIds.size, totalLines: lines.length, emittedLines: parts.length });
+                        }
+                        keptIdsSize = keepIds.size;
+                        scoped = true;
+                        try {
+                            const nodeLineRegex = /^(\s*)([A-Za-z0-9:_-]+)\[[^\]]*\]/; // lgtm[js/bad-tag-filter] — IDs escaped with regex special char replacement
+                            const edgeLineRegex = /-->|===|~~>|\|-/;
+                            let n = 0, eCnt = 0;
+                            for (const ln of filtered) {
+                                if (nodeLineRegex.test(ln))
+                                    n++;
+                                else if (edgeLineRegex.test(ln))
+                                    eCnt++;
+                            }
+                            filteredNodeCount = n;
+                            filteredEdgeCount = eCnt;
+                        }
+                        catch { /* ignore count derivation errors */ }
+                    }
+                }
+                catch (filterErr) {
+                    console.warn('[graph/mermaid][filter-failed]', filterErr);
+                }
+            }
+            try {
+                // eslint-disable-next-line no-console
+                console.debug('[graph/mermaid][ok]', { ms: Date.now() - t0, nodes: graph.meta?.nodeCount, edges: graph.meta?.edgeCount, bytes: mermaidSource.length });
+            }
+            catch { /* ignore diag logging errors */ }
+            let metaOut = graph.meta;
+            if (scoped && graph.meta) {
+                try {
+                    const base = { ...graph.meta };
+                    const augmented = base;
+                    if (typeof filteredNodeCount === 'number')
+                        augmented.nodeCount = filteredNodeCount;
+                    if (typeof filteredEdgeCount === 'number')
+                        augmented.edgeCount = filteredEdgeCount;
+                    augmented.scoped = true;
+                    augmented.keptIds = keptIdsSize;
+                    metaOut = augmented;
+                }
+                catch { /* ignore meta cloning issues */ }
+            }
+            res.json({ success: true, meta: metaOut, mermaid: mermaidSource });
+        }
+        catch (err) {
+            const e = err;
+            try {
+                // eslint-disable-next-line no-console
+                console.warn('[graph/mermaid][error]', e.message);
+            }
+            catch { /* ignore diag logging errors */ }
+            res.status(500).json({ success: false, error: 'Failed to generate mermaid graph' });
+        }
+    });
+    /**
+     * GET /api/graph/categories
+     * Returns list of unique categories with instruction counts.
+     */
+    router.get('/graph/categories', (_req, res) => {
+        try {
+            const st = res.locals.indexState;
+            const map = new Map();
+            for (const inst of st.list) {
+                const cats = Array.isArray(inst.categories) ? inst.categories : [];
+                for (const c of cats) {
+                    map.set(c, (map.get(c) || 0) + 1);
+                }
+            }
+            const categories = [...map.entries()].sort((a, b) => a[0].localeCompare(b[0])).map(([id, count]) => ({ id, count }));
+            res.json({ success: true, categories, total: categories.length, timestamp: Date.now() });
+        }
+        catch (err) {
+            console.error('[graph/categories] Error:', err);
+            res.status(500).json({ success: false, error: 'Failed to get categories' });
+        }
+    });
+    /**
+     * GET /api/graph/instructions?categories=a,b&limit=100
+     * Returns lightweight instruction list filtered by categories (OR semantics).
+     */
+    router.get('/graph/instructions', (req, res) => {
+        try {
+            const st = res.locals.indexState;
+            const catsParam = req.query.categories || '';
+            const filterCats = catsParam ? catsParam.split(',').filter(Boolean) : [];
+            const limitRaw = parseInt(req.query.limit || '0', 10);
+            const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? limitRaw : 500;
+            let instructions = st.list.slice().sort((a, b) => a.id.localeCompare(b.id));
+            if (filterCats.length) {
+                const set = new Set(filterCats.map(c => c.toLowerCase()));
+                instructions = instructions.filter(i => (i.categories || []).some(c => set.has(c.toLowerCase())));
+            }
+            instructions = instructions.slice(0, limit);
+            const flat = instructions.map(i => ({ id: i.id, primaryCategory: i.primaryCategory || i.categories?.[0], categories: i.categories || [] }));
+            res.json({ success: true, instructions: flat, count: flat.length, filtered: !!filterCats.length, timestamp: Date.now() });
+        }
+        catch (err) {
+            console.error('[graph/instructions] Error:', err);
+            res.status(500).json({ success: false, error: 'Failed to get instructions' });
+        }
+    });
+    /**
+     * GET /api/graph/relations?instructions=id1,id2
+     * Returns minimal edges among the provided instruction ids plus category linkage.
+     */
+    router.get('/graph/relations', (req, res) => {
+        try {
+            const idsParam = req.query.instructions || '';
+            const ids = idsParam.split(',').filter(Boolean);
+            if (!ids.length) {
+                return res.json({ success: true, nodes: [], edges: [], categories: [], timestamp: Date.now() });
+            }
+            const graph = (0, handlers_graph_js_1.buildGraph)({ enrich: true, includeCategoryNodes: true, includeUsage: false });
+            const expand = (req.query.expand === '1' || req.query.expand === 'true');
+            const selectedSet = new Set(ids);
+            const workingSet = new Set(ids);
+            const firstEdges = graph.edges.filter(e => selectedSet.has(e.from) || selectedSet.has(e.to));
+            if (expand) {
+                const instructionNodeIds = new Set(graph.nodes.filter(n => n.nodeType === 'instruction').map(n => n.id));
+                for (const e of firstEdges) {
+                    if (instructionNodeIds.has(e.from) && !workingSet.has(e.from))
+                        workingSet.add(e.from);
+                    if (instructionNodeIds.has(e.to) && !workingSet.has(e.to))
+                        workingSet.add(e.to);
+                }
+            }
+            const nodesAll = graph.nodes.filter(n => workingSet.has(n.id) || n.nodeType === 'category');
+            const edges = graph.edges.filter(e => workingSet.has(e.from) && workingSet.has(e.to) && (selectedSet.has(e.from) || selectedSet.has(e.to)));
+            const categoryRefs = new Set();
+            for (const e of edges) {
+                if (e.type === 'belongs' || e.type === 'primary') {
+                    if (e.to.startsWith('category:'))
+                        categoryRefs.add(e.to);
+                    if (e.from.startsWith('category:'))
+                        categoryRefs.add(e.from);
+                }
+            }
+            const finalNodes = nodesAll.filter(n => !('nodeType' in n && n.nodeType === 'category') || categoryRefs.has(n.id));
+            const categories = [...categoryRefs].map(id => ({ id: id.replace(/^category:/, '') }));
+            const expandedCount = workingSet.size - selectedSet.size;
+            res.json({ success: true, nodes: finalNodes, edges, categories, expanded: expand ? expandedCount : 0, timestamp: Date.now() });
+        }
+        catch (err) {
+            console.error('[graph/relations] Error:', err);
+            res.status(500).json({ success: false, error: 'Failed to get relations' });
+        }
+    });
+    return router;
+}

package/dist/dashboard/server/routes/index.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Route module index — re-exports all route factories and mounts dashboard routes.
+ */
+import { Express } from 'express';
+import { MetricsCollector } from '../MetricsCollector.js';
+export { createStatusRoutes } from './status.routes.js';
+export { createMetricsRoutes } from './metrics.routes.js';
+export { createAdminRoutes } from './admin.routes.js';
+export { createGraphRoutes } from './graph.routes.js';
+export { createInstructionsRoutes } from './instructions.routes.js';
+export { createKnowledgeRoutes } from './knowledge.routes.js';
+export { createAlertsRoutes } from './alerts.routes.js';
+export { createLogsRoutes } from './logs.routes.js';
+export { createSyntheticRoutes } from './synthetic.routes.js';
+export { createInstancesRoutes } from './instances.routes.js';
+export { createToolsRoutes } from './tools.routes.js';
+export { createEmbeddingsRoutes } from './embeddings.routes.js';
+export { createUsageRoutes } from './usage.routes.js';
+export { createScriptsRoutes } from './scripts.routes.js';
+export { createMessagingRoutes } from './messaging.routes.js';
+export { createSqliteRoutes } from './sqlite.routes.js';
+export interface DashboardRoutesContext {
+    metricsCollector: MetricsCollector;
+    enableWebSockets: boolean;
+    enableCors: boolean;
+    graphEnabled: boolean;
+    wsProtocol: 'wss' | 'ws';
+    getServerInfo: () => {
+        port: number;
+        host: string;
+        url: string;
+    } | null;
+}
+export declare function renderPanelMarkdownHtml(name: string, markdown: string): string;
+/**
+ * Registers all top-level dashboard routes on the given Express app.
+ * Called once during DashboardServer construction after middleware is set up.
+ */
+export declare function mountDashboardRoutes(app: Express, ctx: DashboardRoutesContext): void;