@renseiai/agentfactory-code-intelligence 0.8.8 → 0.8.9

package/dist/src/vector/hnsw-store.js

@@ -0,0 +1,437 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+// ── Cosine Similarity ───────────────────────────────────────────────
+function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    if (denom === 0)
+        return 0;
+    return dot / denom;
+}
+// ── HNSW Threshold ──────────────────────────────────────────────────
+/** Minimum chunk count to trigger HNSW graph construction. Below this, brute-force is used. */
+const DEFAULT_HNSW_THRESHOLD = 1000;
+// ── InMemoryVectorStore ─────────────────────────────────────────────
+export class InMemoryVectorStore {
+    M;
+    efConstruction;
+    efSearch;
+    modelConfigHash;
+    hnswThreshold;
+    // Flat storage — always maintained
+    chunks = new Map();
+    vectors = new Map();
+    // HNSW graph — built when size >= threshold
+    nodes = new Map();
+    entryPoint = null;
+    maxLevel = 0;
+    hnswBuilt = false;
+    constructor(options = {}) {
+        this.M = options.M ?? 16;
+        this.efConstruction = options.efConstruction ?? 200;
+        this.efSearch = options.efSearch ?? 100;
+        this.modelConfigHash = options.modelConfigHash ?? '';
+        this.hnswThreshold = DEFAULT_HNSW_THRESHOLD;
+    }
+    /**
+     * Override the HNSW promotion threshold (useful for testing).
+     */
+    setHNSWThreshold(n) {
+        this.hnswThreshold = n;
+    }
+    // ── VectorStore interface ───────────────────────────────────────
+    async insert(chunks) {
+        for (const chunk of chunks) {
+            if (!chunk.embedding || chunk.embedding.length === 0)
+                continue;
+            this.chunks.set(chunk.id, chunk);
+            this.vectors.set(chunk.id, chunk.embedding);
+        }
+        // Check if we should build / update the HNSW graph
+        if (this.chunks.size >= this.hnswThreshold && !this.hnswBuilt) {
+            this.buildHNSW();
+        }
+        else if (this.hnswBuilt) {
+            // Incrementally add new chunks to existing graph
+            for (const chunk of chunks) {
+                if (!chunk.embedding || chunk.embedding.length === 0)
+                    continue;
+                if (!this.nodes.has(chunk.id)) {
+                    this.hnswInsert(chunk.id, chunk, chunk.embedding);
+                }
+            }
+        }
+    }
+    async search(query, topK) {
+        if (this.chunks.size === 0)
+            return [];
+        if (this.hnswBuilt) {
+            return this.hnswSearch(query, topK);
+        }
+        return this.bruteForceSearch(query, topK);
+    }
+    async delete(ids) {
+        for (const id of ids) {
+            this.chunks.delete(id);
+            this.vectors.delete(id);
+            if (this.hnswBuilt) {
+                this.hnswRemove(id);
+            }
+        }
+    }
+    size() {
+        return this.chunks.size;
+    }
+    async save(dirPath) {
+        await mkdir(dirPath, { recursive: true });
+        const meta = {
+            model: 'in-memory',
+            dimensions: this.getFirstDimensions(),
+            chunkCount: this.chunks.size,
+            createdAt: Date.now(),
+            modelConfigHash: this.modelConfigHash,
+        };
+        const index = this.serializeIndex();
+        await writeFile(join(dirPath, 'meta.json'), JSON.stringify(meta, null, 2));
+        await writeFile(join(dirPath, 'index.json'), JSON.stringify(index));
+    }
+    async load(dirPath) {
+        const metaRaw = await readFile(join(dirPath, 'meta.json'), 'utf-8');
+        const meta = JSON.parse(metaRaw);
+        // Validate model config hash
+        if (this.modelConfigHash && meta.modelConfigHash && meta.modelConfigHash !== this.modelConfigHash) {
+            throw new Error(`Index model config hash mismatch: stored="${meta.modelConfigHash}" current="${this.modelConfigHash}"`);
+        }
+        const indexRaw = await readFile(join(dirPath, 'index.json'), 'utf-8');
+        const index = JSON.parse(indexRaw);
+        this.deserializeIndex(index);
+    }
+    async clear() {
+        this.chunks.clear();
+        this.vectors.clear();
+        this.nodes.clear();
+        this.entryPoint = null;
+        this.maxLevel = 0;
+        this.hnswBuilt = false;
+    }
+    // ── Query whether the HNSW graph is active (for testing) ──────
+    get isHNSWActive() {
+        return this.hnswBuilt;
+    }
+    // ── Brute-force search ────────────────────────────────────────
+    bruteForceSearch(query, topK) {
+        const results = [];
+        for (const [id, vector] of this.vectors) {
+            const score = cosineSimilarity(query, vector);
+            const chunk = this.chunks.get(id);
+            results.push({ chunk, score });
+        }
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, topK);
+    }
+    // ── HNSW: Build from scratch ──────────────────────────────────
+    buildHNSW() {
+        this.nodes.clear();
+        this.entryPoint = null;
+        this.maxLevel = 0;
+        for (const [id, vector] of this.vectors) {
+            const chunk = this.chunks.get(id);
+            this.hnswInsert(id, chunk, vector);
+        }
+        this.hnswBuilt = true;
+    }
+    // ── HNSW: Random level assignment ─────────────────────────────
+    randomLevel() {
+        // Level = floor(-ln(uniform) * mL), where mL = 1/ln(M)
+        const mL = 1 / Math.log(this.M);
+        return Math.floor(-Math.log(Math.random()) * mL);
+    }
+    // ── HNSW: Insert a single element ────────────────────────────
+    hnswInsert(id, chunk, vector) {
+        const level = this.randomLevel();
+        const node = {
+            id,
+            chunk,
+            vector,
+            level,
+            neighbors: [],
+        };
+        for (let l = 0; l <= level; l++) {
+            node.neighbors.push(new Map());
+        }
+        this.nodes.set(id, node);
+        if (!this.entryPoint) {
+            this.entryPoint = node;
+            this.maxLevel = level;
+            return;
+        }
+        let currentNode = this.entryPoint;
+        // Traverse from top to node's level+1 — greedy single-hop
+        for (let l = this.maxLevel; l > level; l--) {
+            currentNode = this.greedyClosest(currentNode, vector, l);
+        }
+        // For each layer from min(level, maxLevel) down to 0, do ef-search and connect
+        const topInsertLevel = Math.min(level, this.maxLevel);
+        for (let l = topInsertLevel; l >= 0; l--) {
+            const candidates = this.searchLayer(currentNode, vector, this.efConstruction, l);
+            const neighbors = this.selectNeighbors(candidates, this.M);
+            for (const [neighborId, score] of neighbors) {
+                const neighbor = this.nodes.get(neighborId);
+                // Connect bidirectionally
+                node.neighbors[l].set(neighborId, score);
+                if (l < neighbor.neighbors.length) {
+                    neighbor.neighbors[l].set(id, score);
+                    // Prune neighbor's connections if needed
+                    if (neighbor.neighbors[l].size > this.M) {
+                        this.pruneConnections(neighbor, l);
+                    }
+                }
+            }
+            // Move currentNode to the closest found for next layer
+            if (candidates.length > 0) {
+                currentNode = this.nodes.get(candidates[0][0]);
+            }
+        }
+        // Update entry point if new node has higher level
+        if (level > this.maxLevel) {
+            this.entryPoint = node;
+            this.maxLevel = level;
+        }
+    }
+    // ── HNSW: Remove an element ───────────────────────────────────
+    hnswRemove(id) {
+        const node = this.nodes.get(id);
+        if (!node)
+            return;
+        // Remove all bidirectional edges
+        for (let l = 0; l < node.neighbors.length; l++) {
+            for (const neighborId of node.neighbors[l].keys()) {
+                const neighbor = this.nodes.get(neighborId);
+                if (neighbor && l < neighbor.neighbors.length) {
+                    neighbor.neighbors[l].delete(id);
+                }
+            }
+        }
+        this.nodes.delete(id);
+        // If this was the entry point, pick a new one
+        if (this.entryPoint?.id === id) {
+            if (this.nodes.size === 0) {
+                this.entryPoint = null;
+                this.maxLevel = 0;
+                this.hnswBuilt = false;
+            }
+            else {
+                // Find node with highest level
+                let best = null;
+                for (const n of this.nodes.values()) {
+                    if (!best || n.level > best.level)
+                        best = n;
+                }
+                this.entryPoint = best;
+                this.maxLevel = best?.level ?? 0;
+            }
+        }
+    }
+    // ── HNSW: Search ──────────────────────────────────────────────
+    hnswSearch(query, topK) {
+        if (!this.entryPoint)
+            return [];
+        let currentNode = this.entryPoint;
+        // Greedy descent from top layer to layer 1
+        for (let l = this.maxLevel; l > 0; l--) {
+            currentNode = this.greedyClosest(currentNode, query, l);
+        }
+        // ef-search at layer 0
+        const candidates = this.searchLayer(currentNode, query, Math.max(this.efSearch, topK), 0);
+        const results = candidates
+            .slice(0, topK)
+            .map(([id, score]) => ({
+            chunk: this.chunks.get(id),
+            score,
+        }));
+        return results;
+    }
+    // ── HNSW: Greedy closest at a layer ───────────────────────────
+    greedyClosest(start, query, layer) {
+        let current = start;
+        let bestSim = cosineSimilarity(query, current.vector);
+        let improved = true;
+        while (improved) {
+            improved = false;
+            if (layer >= current.neighbors.length)
+                break;
+            for (const neighborId of current.neighbors[layer].keys()) {
+                const neighbor = this.nodes.get(neighborId);
+                if (!neighbor)
+                    continue;
+                const sim = cosineSimilarity(query, neighbor.vector);
+                if (sim > bestSim) {
+                    bestSim = sim;
+                    current = neighbor;
+                    improved = true;
+                }
+            }
+        }
+        return current;
+    }
+    // ── HNSW: ef-bounded search at a single layer ────────────────
+    searchLayer(entryNode, query, ef, layer) {
+        const visited = new Set();
+        const candidates = []; // [id, similarity]
+        const results = []; // [id, similarity]
+        const entrySim = cosineSimilarity(query, entryNode.vector);
+        candidates.push([entryNode.id, entrySim]);
+        results.push([entryNode.id, entrySim]);
+        visited.add(entryNode.id);
+        while (candidates.length > 0) {
+            // Pick the best candidate (highest similarity)
+            candidates.sort((a, b) => b[1] - a[1]);
+            const [currentId, currentSim] = candidates.shift();
+            // Get the worst result
+            results.sort((a, b) => b[1] - a[1]);
+            const worstResult = results[results.length - 1][1];
+            // If current candidate is worse than worst result and we have enough results, stop
+            if (currentSim < worstResult && results.length >= ef)
+                break;
+            const currentNode = this.nodes.get(currentId);
+            if (!currentNode || layer >= currentNode.neighbors.length)
+                continue;
+            for (const neighborId of currentNode.neighbors[layer].keys()) {
+                if (visited.has(neighborId))
+                    continue;
+                visited.add(neighborId);
+                const neighbor = this.nodes.get(neighborId);
+                if (!neighbor)
+                    continue;
+                const sim = cosineSimilarity(query, neighbor.vector);
+                results.sort((a, b) => b[1] - a[1]);
+                const currentWorst = results[results.length - 1][1];
+                if (results.length < ef || sim > currentWorst) {
+                    candidates.push([neighborId, sim]);
+                    results.push([neighborId, sim]);
+                    if (results.length > ef) {
+                        results.sort((a, b) => b[1] - a[1]);
+                        results.pop();
+                    }
+                }
+            }
+        }
+        results.sort((a, b) => b[1] - a[1]);
+        return results;
+    }
+    // ── HNSW: Select best neighbors ──────────────────────────────
+    selectNeighbors(candidates, maxNeighbors) {
+        // Simple: just take the top-M by similarity
+        const sorted = [...candidates].sort((a, b) => b[1] - a[1]);
+        return sorted.slice(0, maxNeighbors);
+    }
+    // ── HNSW: Prune overloaded connections ────────────────────────
+    pruneConnections(node, layer) {
+        const neighbors = Array.from(node.neighbors[layer].entries())
+            .map(([id]) => {
+            const n = this.nodes.get(id);
+            if (!n)
+                return null;
+            const sim = cosineSimilarity(node.vector, n.vector);
+            return [id, sim];
+        })
+            .filter((x) => x !== null);
+        neighbors.sort((a, b) => b[1] - a[1]);
+        const kept = neighbors.slice(0, this.M);
+        node.neighbors[layer].clear();
+        for (const [id, sim] of kept) {
+            node.neighbors[layer].set(id, sim);
+        }
+    }
+    // ── Serialization ─────────────────────────────────────────────
+    serializeIndex() {
+        const serializedNodes = [];
+        for (const node of this.nodes.values()) {
+            const neighbors = [];
+            for (let l = 0; l < node.neighbors.length; l++) {
+                neighbors.push(Array.from(node.neighbors[l].keys()));
+            }
+            serializedNodes.push({
+                id: node.id,
+                chunk: node.chunk,
+                vector: node.vector,
+                level: node.level,
+                neighbors,
+            });
+        }
+        // If HNSW is not built, serialize from flat storage
+        if (!this.hnswBuilt && this.chunks.size > 0) {
+            for (const [id, chunk] of this.chunks) {
+                const vector = this.vectors.get(id);
+                if (!vector)
+                    continue;
+                serializedNodes.push({
+                    id,
+                    chunk,
+                    vector,
+                    level: 0,
+                    neighbors: [[]],
+                });
+            }
+        }
+        return {
+            entryPointId: this.entryPoint?.id ?? null,
+            maxLevel: this.maxLevel,
+            nodes: serializedNodes,
+        };
+    }
+    deserializeIndex(index) {
+        this.chunks.clear();
+        this.vectors.clear();
+        this.nodes.clear();
+        this.entryPoint = null;
+        this.maxLevel = index.maxLevel;
+        // First pass: create all nodes without neighbors
+        for (const sn of index.nodes) {
+            this.chunks.set(sn.id, sn.chunk);
+            this.vectors.set(sn.id, sn.vector);
+            const node = {
+                id: sn.id,
+                chunk: sn.chunk,
+                vector: sn.vector,
+                level: sn.level,
+                neighbors: [],
+            };
+            for (let l = 0; l <= sn.level; l++) {
+                node.neighbors.push(new Map());
+            }
+            this.nodes.set(sn.id, node);
+        }
+        // Second pass: wire up neighbor connections
+        for (const sn of index.nodes) {
+            const node = this.nodes.get(sn.id);
+            for (let l = 0; l < sn.neighbors.length; l++) {
+                for (const neighborId of sn.neighbors[l]) {
+                    if (this.nodes.has(neighborId)) {
+                        node.neighbors[l].set(neighborId, 0);
+                    }
+                }
+            }
+        }
+        // Set entry point
+        if (index.entryPointId && this.nodes.has(index.entryPointId)) {
+            this.entryPoint = this.nodes.get(index.entryPointId);
+        }
+        // Mark HNSW as built if we have enough nodes
+        this.hnswBuilt = this.nodes.size >= this.hnswThreshold;
+    }
+    // ── Helpers ───────────────────────────────────────────────────
+    getFirstDimensions() {
+        for (const vector of this.vectors.values()) {
+            return vector.length;
+        }
+        return 0;
+    }
+}

package/dist/src/vector/vector-store.d.ts

@@ -0,0 +1,15 @@
+import type { EmbeddingChunk } from '../types.js';
+export interface VectorSearchResult {
+    chunk: EmbeddingChunk;
+    score: number;
+}
+export interface VectorStore {
+    insert(chunks: EmbeddingChunk[]): Promise<void>;
+    search(query: number[], topK: number): Promise<VectorSearchResult[]>;
+    delete(ids: string[]): Promise<void>;
+    size(): number;
+    save(path: string): Promise<void>;
+    load(path: string): Promise<void>;
+    clear(): Promise<void>;
+}
+//# sourceMappingURL=vector-store.d.ts.map

package/dist/src/vector/vector-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../../src/vector/vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,cAAc,CAAA;IACrB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAA;IACpE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,IAAI,IAAI,MAAM,CAAA;IACd,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjC,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB"}

package/dist/src/vector/vector-store.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@renseiai/agentfactory-code-intelligence",
-  "version": "0.8.8",
+  "version": "0.8.9",
   "type": "module",
   "description": "Code intelligence for AgentFactory — tree-sitter AST parsing, BM25 search, incremental indexing, memory deduplication",
   "author": "Rensei AI (https://rensei.ai)",