@andespindola/brainlink 1.0.7 → 1.2.0

package/dist/infrastructure/knowledge-store/binary-store.js

@@ -0,0 +1,333 @@
+// Binary storage backend. Stores chunk vectors as raw little-endian Float32 in a
+// dense `vectors.f32` file (~4x smaller than the JSON backend's decimal text and
+// far cheaper to load) and the document/chunk/link metadata as Brotli-compressed
+// JSON. Reads reconstruct the shared `StoredIndex` shape and answer through the
+// same query core as the JSON backend, so results are behavior-identical (vector
+// values are stored at Float32 precision, the standard for embeddings).
+//
+// Migration is seamless: until the first index run materializes the binary store,
+// reads transparently fall back to a legacy `index.json`; the first save writes
+// the binary store and removes the legacy file.
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
+import { mkdir, open, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createEmbeddingBuckets } from '../../domain/embeddings.js';
+import { runSearch } from '../../application/ranking/run-search.js';
+import { indexStoragePath, readJsonStoredIndex } from '../file-index.js';
+import { selectSemanticCandidates, semanticPrefilterMinChunks } from '../semantic-prefilter.js';
+import { createStoredIndexQueries, toStoredIndex } from './stored-index.js';
+const storeVersion = 1;
+export const binaryStoreDirectory = (vaultPath) => join(vaultPath, '.brainlink', 'store');
+const storeFile = (vaultPath, name) => join(binaryStoreDirectory(vaultPath), name);
+const metaPath = (vaultPath) => storeFile(vaultPath, 'meta.json');
+const brotli = (value) => brotliCompressSync(Buffer.from(value, 'utf8'), {
+    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 5, [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT }
+});
+const unbrotli = (buffer) => brotliDecompressSync(buffer).toString('utf8');
+export const binaryStoreExists = async (vaultPath) => {
+    try {
+        await stat(metaPath(vaultPath));
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+// Pack chunk embeddings into one dense Float32 buffer (chunk i at i*dims floats).
+const encodeVectors = (chunks, dims) => {
+    if (dims === 0) {
+        return Buffer.alloc(0);
+    }
+    const buffer = Buffer.alloc(chunks.length * dims * 4);
+    chunks.forEach((chunk, index) => {
+        const base = index * dims * 4;
+        for (let dimension = 0; dimension < dims; dimension += 1) {
+            buffer.writeFloatLE(chunk.embedding[dimension] ?? 0, base + dimension * 4);
+        }
+    });
+    return buffer;
+};
+const decodeVector = (vectors, index, dims) => {
+    const out = new Array(dims);
+    const base = index * dims * 4;
+    for (let dimension = 0; dimension < dims; dimension += 1) {
+        out[dimension] = vectors.readFloatLE(base + dimension * 4);
+    }
+    return out;
+};
+// int8 quantization of the (L2-normalized, so ~[-1, 1]) embeddings: one byte per
+// dimension, `round(v * 127)` clamped to the signed range. The scale is uniform,
+// so int8·int8 preserves the ordering of the exact dot product — enough to pick
+// approximate nearest neighbours cheaply before an exact Float32 rerank.
+const clampInt8 = (value) => (value > 127 ? 127 : value < -127 ? -127 : value);
+const encodeVectorsI8 = (chunks, dims) => {
+    if (dims === 0) {
+        return Buffer.alloc(0);
+    }
+    const buffer = Buffer.alloc(chunks.length * dims);
+    chunks.forEach((chunk, index) => {
+        const base = index * dims;
+        for (let dimension = 0; dimension < dims; dimension += 1) {
+            buffer.writeInt8(clampInt8(Math.round((chunk.embedding[dimension] ?? 0) * 127)), base + dimension);
+        }
+    });
+    return buffer;
+};
+const writeAtomic = async (path, data) => {
+    const temp = `${path}.tmp`;
+    await writeFile(temp, data, { mode: 0o600 });
+    await rename(temp, path);
+};
+const persistStoredIndex = async (vaultPath, stored) => {
+    const dims = stored.chunks.reduce((max, chunk) => Math.max(max, chunk.embedding.length), 0);
+    const chunkMeta = stored.chunks.map((chunk) => ({
+        id: chunk.id,
+        documentId: chunk.documentId,
+        ordinal: chunk.ordinal,
+        content: chunk.content,
+        tokenCount: chunk.tokenCount,
+        embeddingProvider: chunk.embeddingProvider,
+        ...(chunk.buckets ? { buckets: chunk.buckets } : {}),
+        hasEmbedding: chunk.embedding.length > 0
+    }));
+    const meta = {
+        version: storeVersion,
+        updatedAt: stored.updatedAt,
+        dims,
+        documentCount: stored.documents.length,
+        chunkCount: stored.chunks.length
+    };
+    await mkdir(binaryStoreDirectory(vaultPath), { recursive: true, mode: 0o700 });
+    // Write data files first, then meta.json last as the commit point: a reader
+    // checks meta first, so a crash mid-write leaves the previous meta (and its
+    // matching data) in place rather than a torn state.
+    await writeAtomic(storeFile(vaultPath, 'documents.json.br'), brotli(JSON.stringify(stored.documents)));
+    await writeAtomic(storeFile(vaultPath, 'chunks.json.br'), brotli(JSON.stringify(chunkMeta)));
+    await writeAtomic(storeFile(vaultPath, 'links.json.br'), brotli(JSON.stringify(stored.links)));
+    await writeAtomic(storeFile(vaultPath, 'vectors.f32'), encodeVectors(stored.chunks, dims));
+    // int8 vectors: a compact (1 byte/dim) approximate-nearest-neighbour scan index
+    // read fully during search; the exact Float32 vectors are read by offset only
+    // for the shortlist the int8 scan produces.
+    await writeAtomic(storeFile(vaultPath, 'vectors.i8'), encodeVectorsI8(stored.chunks, dims));
+    await writeAtomic(metaPath(vaultPath), Buffer.from(`${JSON.stringify(meta)}\n`, 'utf8'));
+    // Migration complete: the binary store now holds everything, so drop the
+    // legacy JSON index. Markdown remains the canonical source either way.
+    await rm(indexStoragePath(vaultPath), { force: true });
+};
+const persist = (vaultPath, documents) => persistStoredIndex(vaultPath, toStoredIndex(documents));
+// One-time, lossless format conversion: if the binary store is absent but a
+// legacy index.json exists, rewrite its contents (embeddings included, no
+// re-embedding) into the binary store and drop the legacy file. Deduped per
+// vault so concurrent operations convert once.
+const inFlightMigrations = new Map();
+export const migrateLegacyIndexToBinary = async (vaultPath) => {
+    if (await binaryStoreExists(vaultPath)) {
+        return;
+    }
+    const existing = inFlightMigrations.get(vaultPath);
+    if (existing) {
+        return existing;
+    }
+    const run = (async () => {
+        const legacy = await readJsonStoredIndex(vaultPath, false);
+        if (legacy.documents.length === 0 && legacy.chunks.length === 0) {
+            return;
+        }
+        await persistStoredIndex(vaultPath, legacy);
+    })().finally(() => inFlightMigrations.delete(vaultPath));
+    inFlightMigrations.set(vaultPath, run);
+    return run;
+};
+const loadBinary = async (vaultPath) => {
+    const meta = JSON.parse(await readFile(metaPath(vaultPath), 'utf8'));
+    const [documentsRaw, chunkMetaRaw, linksRaw, vectors] = await Promise.all([
+        readFile(storeFile(vaultPath, 'documents.json.br')),
+        readFile(storeFile(vaultPath, 'chunks.json.br')),
+        readFile(storeFile(vaultPath, 'links.json.br')),
+        readFile(storeFile(vaultPath, 'vectors.f32'))
+    ]);
+    const documents = JSON.parse(unbrotli(documentsRaw));
+    const chunkMeta = JSON.parse(unbrotli(chunkMetaRaw));
+    const links = JSON.parse(unbrotli(linksRaw));
+    const chunks = chunkMeta.map((chunk, index) => ({
+        id: chunk.id,
+        documentId: chunk.documentId,
+        ordinal: chunk.ordinal,
+        content: chunk.content,
+        tokenCount: chunk.tokenCount,
+        embeddingProvider: chunk.embeddingProvider,
+        embedding: chunk.hasEmbedding && meta.dims > 0 ? decodeVector(vectors, index, meta.dims) : [],
+        ...(chunk.buckets ? { buckets: chunk.buckets } : {})
+    }));
+    return {
+        version: 1,
+        updatedAt: typeof meta.updatedAt === 'string' ? meta.updatedAt : new Date().toISOString(),
+        documents,
+        chunks,
+        links
+    };
+};
+const readInt8Vectors = async (vaultPath) => {
+    try {
+        const raw = await readFile(storeFile(vaultPath, 'vectors.i8'));
+        return new Int8Array(raw.buffer, raw.byteOffset, raw.byteLength);
+    }
+    catch {
+        // A store written before the int8 index existed (e.g. a 1.1.0 vault not yet
+        // reindexed): fall back to exact scoring over the shortlist pool.
+        return null;
+    }
+};
+const loadForSearch = async (vaultPath) => {
+    const meta = JSON.parse(await readFile(metaPath(vaultPath), 'utf8'));
+    const [documentsRaw, chunkMetaRaw, linksRaw] = await Promise.all([
+        readFile(storeFile(vaultPath, 'documents.json.br')),
+        readFile(storeFile(vaultPath, 'chunks.json.br')),
+        readFile(storeFile(vaultPath, 'links.json.br'))
+    ]);
+    const documents = JSON.parse(unbrotli(documentsRaw));
+    const chunkMeta = JSON.parse(unbrotli(chunkMetaRaw));
+    const links = JSON.parse(unbrotli(linksRaw));
+    const chunks = chunkMeta.map((chunk) => ({
+        id: chunk.id,
+        documentId: chunk.documentId,
+        ordinal: chunk.ordinal,
+        content: chunk.content,
+        tokenCount: chunk.tokenCount,
+        embeddingProvider: chunk.embeddingProvider,
+        embedding: [],
+        ...(chunk.buckets ? { buckets: chunk.buckets } : {})
+    }));
+    const int8 = meta.dims > 0 ? await readInt8Vectors(vaultPath) : null;
+    return {
+        index: {
+            version: 1,
+            updatedAt: typeof meta.updatedAt === 'string' ? meta.updatedAt : new Date().toISOString(),
+            documents,
+            chunks,
+            links
+        },
+        int8,
+        dims: meta.dims
+    };
+};
+// Approximate dot product of the Float32 query against a chunk's int8 vector.
+// The uniform int8 scale cancels out of the ranking, so this preserves the exact
+// dot product's ordering closely enough to shortlist candidates.
+const approxScoreI8 = (query, int8, index, dims) => {
+    const length = Math.min(query.length, dims);
+    const base = index * dims;
+    let total = 0;
+    for (let dimension = 0; dimension < length; dimension += 1) {
+        total += query[dimension] * int8[base + dimension];
+    }
+    return total;
+};
+// Read the exact Float32 vectors for a shortlist of chunk indices by offset,
+// through a single file handle. This is the only place the exact vectors are
+// touched during search, and only for the shortlist (not the whole corpus).
+const readExactVectors = async (vaultPath, indices, dims) => {
+    const vectors = new Map();
+    if (indices.length === 0 || dims === 0) {
+        return vectors;
+    }
+    const handle = await open(storeFile(vaultPath, 'vectors.f32'), 'r');
+    try {
+        const byteLength = dims * 4;
+        const buffer = Buffer.alloc(byteLength);
+        for (const index of indices) {
+            await handle.read(buffer, 0, byteLength, index * byteLength);
+            const vector = new Array(dims);
+            for (let dimension = 0; dimension < dims; dimension += 1) {
+                vector[dimension] = buffer.readFloatLE(dimension * 4);
+            }
+            vectors.set(index, vector);
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    return vectors;
+};
+// ANN search over the materialized binary store. Candidate selection is layered:
+// embedding buckets prune coarsely on large vaults (IVF-style), the int8 scan
+// ranks the survivors, and the exact Float32 vectors are read only for the
+// resulting shortlist and reranked by the shared ranker — so the final ordering
+// is produced by the same deterministic scorer as the JSON backend. Small vaults
+// (or a store without an int8 index) rerank the whole pool exactly, staying
+// bit-identical to a full scan.
+const annSearch = async (vaultPath, options, query, limit, agentId, mode, queryEmbedding) => {
+    const loaded = await loadForSearch(vaultPath);
+    const documentsById = new Map(loaded.index.documents.map((document) => [document.id, document]));
+    const eligible = loaded.index.chunks.flatMap((chunk, index) => {
+        const document = documentsById.get(chunk.documentId);
+        if (!document || (agentId && document.agentId !== agentId)) {
+            return [];
+        }
+        return [{ index, chunk, document, buckets: chunk.buckets }];
+    });
+    const toRow = (entry, embedding) => ({
+        documentId: entry.document.id,
+        agentId: entry.document.agentId,
+        title: entry.document.title,
+        path: entry.document.path,
+        chunkId: entry.chunk.id,
+        chunkOrdinal: entry.chunk.ordinal,
+        content: entry.chunk.content,
+        tokenCount: entry.chunk.tokenCount,
+        tags: entry.document.tags,
+        embedding,
+        buckets: entry.chunk.buckets
+    });
+    // Lexical-only modes never need vectors, so no exact vector is ever read.
+    if (mode === 'fts' || queryEmbedding.length === 0 || loaded.dims === 0) {
+        return runSearch(eligible.map((entry) => toRow(entry, [])), query, limit, mode, queryEmbedding, options.hybridFusion);
+    }
+    // Small vaults (a full exact scan is already sub-millisecond) and stores
+    // written before the int8 index existed rerank every candidate exactly, read
+    // from a single bulk pass over the exact vectors — bit-identical to a full
+    // scan, and no slower than the legacy load.
+    const useAnn = loaded.int8 !== null && loaded.index.chunks.length > semanticPrefilterMinChunks;
+    if (!useAnn) {
+        const buffer = await readFile(storeFile(vaultPath, 'vectors.f32'));
+        return runSearch(eligible.map((entry) => toRow(entry, decodeVector(buffer, entry.index, loaded.dims))), query, limit, mode, queryEmbedding, options.hybridFusion);
+    }
+    // Large vault: coarse-prune with embedding buckets, rank the survivors with the
+    // int8 scan, then read the exact Float32 vectors by offset only for the
+    // shortlist and rerank with the shared scorer.
+    const int8 = loaded.int8;
+    const pool = selectSemanticCandidates(eligible, createEmbeddingBuckets(queryEmbedding), limit);
+    const recallFloor = Math.max(limit * 4, 64);
+    const shortlist = [...pool]
+        .map((entry) => ({ entry, score: approxScoreI8(queryEmbedding, int8, entry.index, loaded.dims) }))
+        .sort((left, right) => right.score - left.score || left.entry.chunk.id.localeCompare(right.entry.chunk.id))
+        .slice(0, recallFloor)
+        .map((scored) => scored.entry);
+    const exactVectors = await readExactVectors(vaultPath, shortlist.map((entry) => entry.index), loaded.dims);
+    return runSearch(eligible.map((entry) => toRow(entry, exactVectors.get(entry.index) ?? [])), query, limit, mode, queryEmbedding, options.hybridFusion);
+};
+// Binary backend. While its store does not yet exist, reads fall back to the
+// legacy JSON index so search/graph stay available mid-migration; the first save
+// materializes the binary store and removes the legacy file.
+export const openBinaryStore = (vaultPath, options = {}) => {
+    const load = async () => (await binaryStoreExists(vaultPath)) ? loadBinary(vaultPath) : readJsonStoredIndex(vaultPath, false);
+    const loadStrict = async () => (await binaryStoreExists(vaultPath)) ? loadBinary(vaultPath) : readJsonStoredIndex(vaultPath, true);
+    const shared = createStoredIndexQueries({ load, loadStrict }, options);
+    return {
+        reset: async () => {
+            await persist(vaultPath, []);
+        },
+        saveDocuments: async (documents) => {
+            await persist(vaultPath, documents);
+        },
+        ...shared,
+        // Override search with the ANN path once the binary store is materialized;
+        // until then defer to the shared full-scan search over the legacy fallback.
+        search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => (await binaryStoreExists(vaultPath))
+            ? annSearch(vaultPath, options, query, limit, agentId, mode, queryEmbedding)
+            : shared.search(query, limit, agentId, mode, queryEmbedding),
+        close: () => {
+            // File-based store has no persistent connection.
+        }
+    };
+};

package/dist/infrastructure/knowledge-store/index.js

@@ -0,0 +1,37 @@
+import { loadBrainlinkConfig } from '../config.js';
+import { openFileIndex } from '../file-index.js';
+import { migrateLegacyIndexToBinary, openBinaryStore } from './binary-store.js';
+const resolveBackend = async (vaultPath) => {
+    const config = await loadBrainlinkConfig();
+    const options = { hybridFusion: config.hybridFusion };
+    if (config.storageBackend === 'binary') {
+        await migrateLegacyIndexToBinary(vaultPath);
+        return openBinaryStore(vaultPath, options);
+    }
+    return openFileIndex(vaultPath, options);
+};
+export const openKnowledgeStore = (vaultPath) => {
+    let backend = null;
+    // Resolve (and memoize) the backend once per store instance. A config-load
+    // failure degrades to the always-available JSON backend rather than throwing.
+    const resolve = () => {
+        backend ??= resolveBackend(vaultPath).catch(() => openFileIndex(vaultPath));
+        return backend;
+    };
+    return {
+        reset: async () => (await resolve()).reset(),
+        saveDocuments: async (documents) => (await resolve()).saveDocuments(documents),
+        getIndexedDocuments: async (agentId) => (await resolve()).getIndexedDocuments(agentId),
+        search: async (query, limit, agentId, mode, queryEmbedding) => (await resolve()).search(query, limit, agentId, mode, queryEmbedding),
+        listLinks: async (agentId) => (await resolve()).listLinks(agentId),
+        listBacklinks: async (title, agentId) => (await resolve()).listBacklinks(title, agentId),
+        getGraph: async (agentId) => (await resolve()).getGraph(agentId),
+        getGraphSummary: async (agentId) => (await resolve()).getGraphSummary(agentId),
+        getGraphNode: async (id, agentId) => (await resolve()).getGraphNode(id, agentId),
+        searchGraphNodeIds: async (query, limit, agentId) => (await resolve()).searchGraphNodeIds(query, limit, agentId),
+        listAgents: async () => (await resolve()).listAgents(),
+        close: () => {
+            // Backends hold no persistent connection; nothing to release.
+        }
+    };
+};

package/dist/infrastructure/knowledge-store/stored-index.js

@@ -0,0 +1,216 @@
+// Shared in-memory query core for the knowledge index. Both storage backends
+// (the JSON file and the binary store) load their data into the same
+// `StoredIndex` shape and then expose identical read semantics through these
+// queries. Keeping the graph/search/link logic here — instead of duplicating it
+// per backend — guarantees the two backends answer the same way and that the
+// only thing a backend owns is how bytes are loaded and persisted.
+import { createEmbeddingBuckets } from '../../domain/embeddings.js';
+import { lexicalFieldScore, normalizeToken, runSearch, tokenize } from '../../application/ranking/run-search.js';
+import { selectSemanticCandidates } from '../semantic-prefilter.js';
+export const emptyIndex = () => ({
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    documents: [],
+    chunks: [],
+    links: []
+});
+const toGraphLink = (link, documentsById) => {
+    const source = documentsById.get(link.fromDocumentId);
+    const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
+    return {
+        agentId: source?.agentId ?? 'shared',
+        fromTitle: source?.title ?? 'Unknown',
+        fromPath: source?.path ?? 'Unknown',
+        toTitle: target?.title ?? link.toTitle,
+        toPath: target?.path ?? null,
+        weight: link.weight,
+        priority: link.priority
+    };
+};
+const toGraphNode = (document, includeContent) => ({
+    id: document.id,
+    agentId: document.agentId,
+    title: document.title,
+    path: document.path,
+    content: includeContent ? document.content : '',
+    tags: document.tags,
+    contextLinks: document.contextLinks ?? []
+});
+// Build the shared read methods over a backend's loaders. The returned object is
+// the read half of the KnowledgeStore port; backends add reset/saveDocuments/
+// close around it.
+export const createStoredIndexQueries = ({ load, loadStrict }, options = {}) => ({
+    getIndexedDocuments: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const selectedDocumentIds = new Set(documents.map((document) => document.id));
+        const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
+            if (!selectedDocumentIds.has(chunk.documentId)) {
+                return state;
+            }
+            const current = state.get(chunk.documentId) ?? [];
+            current.push(chunk);
+            state.set(chunk.documentId, current);
+            return state;
+        }, new Map());
+        const linksByDocumentId = index.links.reduce((state, link) => {
+            if (!selectedDocumentIds.has(link.fromDocumentId)) {
+                return state;
+            }
+            const current = state.get(link.fromDocumentId) ?? [];
+            current.push(link);
+            state.set(link.fromDocumentId, current);
+            return state;
+        }, new Map());
+        return documents
+            .map((document) => ({
+            document,
+            chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
+            links: linksByDocumentId.get(document.id) ?? []
+        }))
+            .sort((left, right) => left.document.path.localeCompare(right.document.path));
+    },
+    search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
+        const index = await loadStrict();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        const rows = index.chunks.flatMap((chunk) => {
+            const document = documentsById.get(chunk.documentId);
+            if (!document) {
+                return [];
+            }
+            if (agentId && document.agentId !== agentId) {
+                return [];
+            }
+            return [
+                {
+                    documentId: document.id,
+                    agentId: document.agentId,
+                    title: document.title,
+                    path: document.path,
+                    chunkId: chunk.id,
+                    chunkOrdinal: chunk.ordinal,
+                    content: chunk.content,
+                    tokenCount: chunk.tokenCount,
+                    tags: document.tags,
+                    embedding: chunk.embedding,
+                    buckets: chunk.buckets
+                }
+            ];
+        });
+        // Pure-semantic scoring on a large vault can skip chunks that share no
+        // embedding bucket with the query. Hybrid/fts keep every row so lexical
+        // matches are never dropped; the prefilter also falls back to a full scan
+        // on small or partially-indexed vaults. Candidate selection is the
+        // backend's job; the shared ranker then scores and orders the candidates.
+        const candidates = mode === 'semantic' && queryEmbedding.length > 0
+            ? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
+            : rows;
+        return runSearch(candidates, query, limit, mode, queryEmbedding, options.hybridFusion);
+    },
+    listLinks: async (agentId) => {
+        const index = await load();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        return index.links
+            .filter((link) => {
+            const source = documentsById.get(link.fromDocumentId);
+            return agentId ? source?.agentId === agentId : true;
+        })
+            .map((link) => toGraphLink(link, documentsById))
+            .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
+    },
+    listBacklinks: async (title, agentId) => {
+        const index = await load();
+        const titleKey = title.toLowerCase();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        return index.links
+            .filter((link) => link.toTitle.toLowerCase() === titleKey)
+            .filter((link) => {
+            const source = documentsById.get(link.fromDocumentId);
+            return agentId ? source?.agentId === agentId : true;
+        })
+            .map((link) => toGraphLink(link, documentsById))
+            .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
+    },
+    getGraph: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const documentIds = new Set(documents.map((document) => document.id));
+        const edges = index.links
+            .filter((link) => documentIds.has(link.fromDocumentId))
+            .map((link) => ({
+            source: link.fromDocumentId,
+            target: link.toDocumentId,
+            targetTitle: link.toTitle,
+            weight: link.weight,
+            priority: link.priority
+        }));
+        return { nodes: documents.map((document) => toGraphNode(document, true)), edges };
+    },
+    getGraphSummary: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const documentIds = new Set(documents.map((document) => document.id));
+        const edges = index.links
+            .filter((link) => documentIds.has(link.fromDocumentId))
+            .map((link) => ({
+            source: link.fromDocumentId,
+            target: link.toDocumentId,
+            targetTitle: link.toTitle,
+            weight: link.weight,
+            priority: link.priority
+        }));
+        return { nodes: documents.map((document) => toGraphNode(document, false)), edges };
+    },
+    getGraphNode: async (id, agentId) => {
+        const index = await load();
+        const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
+        return document ? toGraphNode(document, true) : undefined;
+    },
+    searchGraphNodeIds: async (query, limit, agentId) => {
+        const index = await load();
+        const normalized = normalizeToken(query);
+        if (normalized.length === 0 || limit <= 0) {
+            return [];
+        }
+        const tokens = tokenize(query);
+        const scored = index.documents
+            .filter((document) => !agentId || document.agentId === agentId)
+            .map((document) => {
+            const score = lexicalFieldScore({
+                documentId: document.id,
+                agentId: document.agentId,
+                title: document.title,
+                path: document.path,
+                chunkId: document.id,
+                chunkOrdinal: 0,
+                content: document.content,
+                tokenCount: 0,
+                tags: document.tags,
+                embedding: []
+            }, tokens);
+            return { id: document.id, score };
+        })
+            .filter((row) => row.score > 0)
+            .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
+            .slice(0, limit);
+        return scored.map((row) => row.id);
+    },
+    listAgents: async () => {
+        const index = await load();
+        const counts = index.documents.reduce((state, document) => {
+            state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
+            return state;
+        }, new Map());
+        return Array.from(counts.entries())
+            .sort((left, right) => left[0].localeCompare(right[0]))
+            .map(([id, documentCount]) => ({ id, documentCount }));
+    }
+});
+// Flatten indexed documents into the stored shape persisted by both backends.
+export const toStoredIndex = (documents) => ({
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    documents: documents.map((document) => document.document),
+    chunks: documents.flatMap((document) => document.chunks),
+    links: documents.flatMap((document) => document.links)
+});

package/dist/infrastructure/vault-crypto.js

@@ -0,0 +1,78 @@
+// Symmetric encryption for versioned vault content. The per-vault key lives in
+// ~/.brainlink/keys (outside the vault, never committed), so only a Brainlink
+// instance holding that key can decrypt the repository — the ciphertext in git
+// is opaque without it. AES-256-GCM with a content-derived (deterministic)
+// nonce, so identical content always encrypts to identical bytes (no git churn)
+// while distinct content always gets a distinct nonce (no nonce reuse).
+import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { getBrainlinkHomePath } from './paths.js';
+const magic = Buffer.from('BLVE1', 'ascii');
+const nonceLength = 12;
+const authTagLength = 16;
+const algorithm = 'aes-256-gcm';
+const keyFilePath = (vaultPath) => {
+    const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
+    return join(getBrainlinkHomePath(), 'keys', `vault-versioning-${vaultHash}.key`);
+};
+const deriveKeyFromSecret = (secret) => createHash('sha256').update(secret, 'utf8').digest();
+// Load the vault's versioning key, generating a fresh 48-byte secret on first
+// use. BRAINLINK_VAULT_KEY overrides the file (useful to restore an encrypted
+// vault on another machine by carrying just the secret, not the whole repo).
+export const readOrCreateVaultKey = async (vaultPath) => {
+    const envSecret = process.env.BRAINLINK_VAULT_KEY?.trim();
+    if (envSecret && envSecret.length > 0) {
+        return deriveKeyFromSecret(envSecret);
+    }
+    const path = keyFilePath(vaultPath);
+    try {
+        const existing = (await readFile(path, 'utf8')).trim();
+        if (existing.length > 0) {
+            return deriveKeyFromSecret(existing);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+    const secret = randomBytes(48).toString('base64url');
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    await writeFile(path, `${secret}\n`, { encoding: 'utf8', mode: 0o600 });
+    return deriveKeyFromSecret(secret);
+};
+// True when the vault has a versioning key on disk (or via env), so callers can
+// warn before losing access rather than silently minting a new key.
+export const vaultKeyExists = async (vaultPath) => {
+    if (process.env.BRAINLINK_VAULT_KEY?.trim()) {
+        return true;
+    }
+    try {
+        return (await readFile(keyFilePath(vaultPath), 'utf8')).trim().length > 0;
+    }
+    catch {
+        return false;
+    }
+};
+const deriveNonce = (key, plaintext) => createHash('sha256').update(key).update(plaintext).digest().subarray(0, nonceLength);
+export const encryptVaultContent = (key, plaintext) => {
+    const nonce = deriveNonce(key, plaintext);
+    const cipher = createCipheriv(algorithm, key, nonce);
+    const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+    return Buffer.concat([magic, nonce, authTag, ciphertext]);
+};
+export const isEncryptedVaultPayload = (payload) => payload.length >= magic.length && payload.subarray(0, magic.length).equals(magic);
+export const decryptVaultContent = (key, payload) => {
+    if (payload.length < magic.length + nonceLength + authTagLength || !isEncryptedVaultPayload(payload)) {
+        throw new Error('Not a Brainlink encrypted vault file.');
+    }
+    let offset = magic.length;
+    const nonce = payload.subarray(offset, (offset += nonceLength));
+    const authTag = payload.subarray(offset, (offset += authTagLength));
+    const ciphertext = payload.subarray(offset);
+    const decipher = createDecipheriv(algorithm, key, nonce);
+    decipher.setAuthTag(authTag);
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+};