@andespindola/brainlink 1.0.7 → 1.1.0

package/dist/infrastructure/knowledge-store/binary-store.js

@@ -0,0 +1,163 @@
+// Binary storage backend. Stores chunk vectors as raw little-endian Float32 in a
+// dense `vectors.f32` file (~4x smaller than the JSON backend's decimal text and
+// far cheaper to load) and the document/chunk/link metadata as Brotli-compressed
+// JSON. Reads reconstruct the shared `StoredIndex` shape and answer through the
+// same query core as the JSON backend, so results are behavior-identical (vector
+// values are stored at Float32 precision, the standard for embeddings).
+//
+// Migration is seamless: until the first index run materializes the binary store,
+// reads transparently fall back to a legacy `index.json`; the first save writes
+// the binary store and removes the legacy file.
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
+import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { indexStoragePath, readJsonStoredIndex } from '../file-index.js';
+import { createStoredIndexQueries, toStoredIndex } from './stored-index.js';
+const storeVersion = 1;
+export const binaryStoreDirectory = (vaultPath) => join(vaultPath, '.brainlink', 'store');
+const storeFile = (vaultPath, name) => join(binaryStoreDirectory(vaultPath), name);
+const metaPath = (vaultPath) => storeFile(vaultPath, 'meta.json');
+const brotli = (value) => brotliCompressSync(Buffer.from(value, 'utf8'), {
+    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 5, [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT }
+});
+const unbrotli = (buffer) => brotliDecompressSync(buffer).toString('utf8');
+export const binaryStoreExists = async (vaultPath) => {
+    try {
+        await stat(metaPath(vaultPath));
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+// Pack chunk embeddings into one dense Float32 buffer (chunk i at i*dims floats).
+const encodeVectors = (chunks, dims) => {
+    if (dims === 0) {
+        return Buffer.alloc(0);
+    }
+    const buffer = Buffer.alloc(chunks.length * dims * 4);
+    chunks.forEach((chunk, index) => {
+        const base = index * dims * 4;
+        for (let dimension = 0; dimension < dims; dimension += 1) {
+            buffer.writeFloatLE(chunk.embedding[dimension] ?? 0, base + dimension * 4);
+        }
+    });
+    return buffer;
+};
+const decodeVector = (vectors, index, dims) => {
+    const out = new Array(dims);
+    const base = index * dims * 4;
+    for (let dimension = 0; dimension < dims; dimension += 1) {
+        out[dimension] = vectors.readFloatLE(base + dimension * 4);
+    }
+    return out;
+};
+const writeAtomic = async (path, data) => {
+    const temp = `${path}.tmp`;
+    await writeFile(temp, data, { mode: 0o600 });
+    await rename(temp, path);
+};
+const persistStoredIndex = async (vaultPath, stored) => {
+    const dims = stored.chunks.reduce((max, chunk) => Math.max(max, chunk.embedding.length), 0);
+    const chunkMeta = stored.chunks.map((chunk) => ({
+        id: chunk.id,
+        documentId: chunk.documentId,
+        ordinal: chunk.ordinal,
+        content: chunk.content,
+        tokenCount: chunk.tokenCount,
+        embeddingProvider: chunk.embeddingProvider,
+        ...(chunk.buckets ? { buckets: chunk.buckets } : {}),
+        hasEmbedding: chunk.embedding.length > 0
+    }));
+    const meta = {
+        version: storeVersion,
+        updatedAt: stored.updatedAt,
+        dims,
+        documentCount: stored.documents.length,
+        chunkCount: stored.chunks.length
+    };
+    await mkdir(binaryStoreDirectory(vaultPath), { recursive: true, mode: 0o700 });
+    // Write data files first, then meta.json last as the commit point: a reader
+    // checks meta first, so a crash mid-write leaves the previous meta (and its
+    // matching data) in place rather than a torn state.
+    await writeAtomic(storeFile(vaultPath, 'documents.json.br'), brotli(JSON.stringify(stored.documents)));
+    await writeAtomic(storeFile(vaultPath, 'chunks.json.br'), brotli(JSON.stringify(chunkMeta)));
+    await writeAtomic(storeFile(vaultPath, 'links.json.br'), brotli(JSON.stringify(stored.links)));
+    await writeAtomic(storeFile(vaultPath, 'vectors.f32'), encodeVectors(stored.chunks, dims));
+    await writeAtomic(metaPath(vaultPath), Buffer.from(`${JSON.stringify(meta)}\n`, 'utf8'));
+    // Migration complete: the binary store now holds everything, so drop the
+    // legacy JSON index. Markdown remains the canonical source either way.
+    await rm(indexStoragePath(vaultPath), { force: true });
+};
+const persist = (vaultPath, documents) => persistStoredIndex(vaultPath, toStoredIndex(documents));
+// One-time, lossless format conversion: if the binary store is absent but a
+// legacy index.json exists, rewrite its contents (embeddings included, no
+// re-embedding) into the binary store and drop the legacy file. Deduped per
+// vault so concurrent operations convert once.
+const inFlightMigrations = new Map();
+export const migrateLegacyIndexToBinary = async (vaultPath) => {
+    if (await binaryStoreExists(vaultPath)) {
+        return;
+    }
+    const existing = inFlightMigrations.get(vaultPath);
+    if (existing) {
+        return existing;
+    }
+    const run = (async () => {
+        const legacy = await readJsonStoredIndex(vaultPath, false);
+        if (legacy.documents.length === 0 && legacy.chunks.length === 0) {
+            return;
+        }
+        await persistStoredIndex(vaultPath, legacy);
+    })().finally(() => inFlightMigrations.delete(vaultPath));
+    inFlightMigrations.set(vaultPath, run);
+    return run;
+};
+const loadBinary = async (vaultPath) => {
+    const meta = JSON.parse(await readFile(metaPath(vaultPath), 'utf8'));
+    const [documentsRaw, chunkMetaRaw, linksRaw, vectors] = await Promise.all([
+        readFile(storeFile(vaultPath, 'documents.json.br')),
+        readFile(storeFile(vaultPath, 'chunks.json.br')),
+        readFile(storeFile(vaultPath, 'links.json.br')),
+        readFile(storeFile(vaultPath, 'vectors.f32'))
+    ]);
+    const documents = JSON.parse(unbrotli(documentsRaw));
+    const chunkMeta = JSON.parse(unbrotli(chunkMetaRaw));
+    const links = JSON.parse(unbrotli(linksRaw));
+    const chunks = chunkMeta.map((chunk, index) => ({
+        id: chunk.id,
+        documentId: chunk.documentId,
+        ordinal: chunk.ordinal,
+        content: chunk.content,
+        tokenCount: chunk.tokenCount,
+        embeddingProvider: chunk.embeddingProvider,
+        embedding: chunk.hasEmbedding && meta.dims > 0 ? decodeVector(vectors, index, meta.dims) : [],
+        ...(chunk.buckets ? { buckets: chunk.buckets } : {})
+    }));
+    return {
+        version: 1,
+        updatedAt: typeof meta.updatedAt === 'string' ? meta.updatedAt : new Date().toISOString(),
+        documents,
+        chunks,
+        links
+    };
+};
+// Binary backend. While its store does not yet exist, reads fall back to the
+// legacy JSON index so search/graph stay available mid-migration; the first save
+// materializes the binary store and removes the legacy file.
+export const openBinaryStore = (vaultPath) => {
+    const load = async () => (await binaryStoreExists(vaultPath)) ? loadBinary(vaultPath) : readJsonStoredIndex(vaultPath, false);
+    const loadStrict = async () => (await binaryStoreExists(vaultPath)) ? loadBinary(vaultPath) : readJsonStoredIndex(vaultPath, true);
+    return {
+        reset: async () => {
+            await persist(vaultPath, []);
+        },
+        saveDocuments: async (documents) => {
+            await persist(vaultPath, documents);
+        },
+        ...createStoredIndexQueries({ load, loadStrict }),
+        close: () => {
+            // File-based store has no persistent connection.
+        }
+    };
+};

package/dist/infrastructure/knowledge-store/index.js

@@ -0,0 +1,36 @@
+import { loadBrainlinkConfig } from '../config.js';
+import { openFileIndex } from '../file-index.js';
+import { migrateLegacyIndexToBinary, openBinaryStore } from './binary-store.js';
+const resolveBackend = async (vaultPath) => {
+    const config = await loadBrainlinkConfig();
+    if (config.storageBackend === 'binary') {
+        await migrateLegacyIndexToBinary(vaultPath);
+        return openBinaryStore(vaultPath);
+    }
+    return openFileIndex(vaultPath);
+};
+export const openKnowledgeStore = (vaultPath) => {
+    let backend = null;
+    // Resolve (and memoize) the backend once per store instance. A config-load
+    // failure degrades to the always-available JSON backend rather than throwing.
+    const resolve = () => {
+        backend ??= resolveBackend(vaultPath).catch(() => openFileIndex(vaultPath));
+        return backend;
+    };
+    return {
+        reset: async () => (await resolve()).reset(),
+        saveDocuments: async (documents) => (await resolve()).saveDocuments(documents),
+        getIndexedDocuments: async (agentId) => (await resolve()).getIndexedDocuments(agentId),
+        search: async (query, limit, agentId, mode, queryEmbedding) => (await resolve()).search(query, limit, agentId, mode, queryEmbedding),
+        listLinks: async (agentId) => (await resolve()).listLinks(agentId),
+        listBacklinks: async (title, agentId) => (await resolve()).listBacklinks(title, agentId),
+        getGraph: async (agentId) => (await resolve()).getGraph(agentId),
+        getGraphSummary: async (agentId) => (await resolve()).getGraphSummary(agentId),
+        getGraphNode: async (id, agentId) => (await resolve()).getGraphNode(id, agentId),
+        searchGraphNodeIds: async (query, limit, agentId) => (await resolve()).searchGraphNodeIds(query, limit, agentId),
+        listAgents: async () => (await resolve()).listAgents(),
+        close: () => {
+            // Backends hold no persistent connection; nothing to release.
+        }
+    };
+};

package/dist/infrastructure/knowledge-store/stored-index.js

@@ -0,0 +1,216 @@
+// Shared in-memory query core for the knowledge index. Both storage backends
+// (the JSON file and the binary store) load their data into the same
+// `StoredIndex` shape and then expose identical read semantics through these
+// queries. Keeping the graph/search/link logic here — instead of duplicating it
+// per backend — guarantees the two backends answer the same way and that the
+// only thing a backend owns is how bytes are loaded and persisted.
+import { createEmbeddingBuckets } from '../../domain/embeddings.js';
+import { lexicalFieldScore, normalizeToken, runSearch, tokenize } from '../../application/ranking/run-search.js';
+import { selectSemanticCandidates } from '../semantic-prefilter.js';
+export const emptyIndex = () => ({
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    documents: [],
+    chunks: [],
+    links: []
+});
+const toGraphLink = (link, documentsById) => {
+    const source = documentsById.get(link.fromDocumentId);
+    const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
+    return {
+        agentId: source?.agentId ?? 'shared',
+        fromTitle: source?.title ?? 'Unknown',
+        fromPath: source?.path ?? 'Unknown',
+        toTitle: target?.title ?? link.toTitle,
+        toPath: target?.path ?? null,
+        weight: link.weight,
+        priority: link.priority
+    };
+};
+const toGraphNode = (document, includeContent) => ({
+    id: document.id,
+    agentId: document.agentId,
+    title: document.title,
+    path: document.path,
+    content: includeContent ? document.content : '',
+    tags: document.tags,
+    contextLinks: document.contextLinks ?? []
+});
+// Build the shared read methods over a backend's loaders. The returned object is
+// the read half of the KnowledgeStore port; backends add reset/saveDocuments/
+// close around it.
+export const createStoredIndexQueries = ({ load, loadStrict }) => ({
+    getIndexedDocuments: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const selectedDocumentIds = new Set(documents.map((document) => document.id));
+        const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
+            if (!selectedDocumentIds.has(chunk.documentId)) {
+                return state;
+            }
+            const current = state.get(chunk.documentId) ?? [];
+            current.push(chunk);
+            state.set(chunk.documentId, current);
+            return state;
+        }, new Map());
+        const linksByDocumentId = index.links.reduce((state, link) => {
+            if (!selectedDocumentIds.has(link.fromDocumentId)) {
+                return state;
+            }
+            const current = state.get(link.fromDocumentId) ?? [];
+            current.push(link);
+            state.set(link.fromDocumentId, current);
+            return state;
+        }, new Map());
+        return documents
+            .map((document) => ({
+            document,
+            chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
+            links: linksByDocumentId.get(document.id) ?? []
+        }))
+            .sort((left, right) => left.document.path.localeCompare(right.document.path));
+    },
+    search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
+        const index = await loadStrict();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        const rows = index.chunks.flatMap((chunk) => {
+            const document = documentsById.get(chunk.documentId);
+            if (!document) {
+                return [];
+            }
+            if (agentId && document.agentId !== agentId) {
+                return [];
+            }
+            return [
+                {
+                    documentId: document.id,
+                    agentId: document.agentId,
+                    title: document.title,
+                    path: document.path,
+                    chunkId: chunk.id,
+                    chunkOrdinal: chunk.ordinal,
+                    content: chunk.content,
+                    tokenCount: chunk.tokenCount,
+                    tags: document.tags,
+                    embedding: chunk.embedding,
+                    buckets: chunk.buckets
+                }
+            ];
+        });
+        // Pure-semantic scoring on a large vault can skip chunks that share no
+        // embedding bucket with the query. Hybrid/fts keep every row so lexical
+        // matches are never dropped; the prefilter also falls back to a full scan
+        // on small or partially-indexed vaults. Candidate selection is the
+        // backend's job; the shared ranker then scores and orders the candidates.
+        const candidates = mode === 'semantic' && queryEmbedding.length > 0
+            ? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
+            : rows;
+        return runSearch(candidates, query, limit, mode, queryEmbedding);
+    },
+    listLinks: async (agentId) => {
+        const index = await load();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        return index.links
+            .filter((link) => {
+            const source = documentsById.get(link.fromDocumentId);
+            return agentId ? source?.agentId === agentId : true;
+        })
+            .map((link) => toGraphLink(link, documentsById))
+            .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
+    },
+    listBacklinks: async (title, agentId) => {
+        const index = await load();
+        const titleKey = title.toLowerCase();
+        const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+        return index.links
+            .filter((link) => link.toTitle.toLowerCase() === titleKey)
+            .filter((link) => {
+            const source = documentsById.get(link.fromDocumentId);
+            return agentId ? source?.agentId === agentId : true;
+        })
+            .map((link) => toGraphLink(link, documentsById))
+            .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
+    },
+    getGraph: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const documentIds = new Set(documents.map((document) => document.id));
+        const edges = index.links
+            .filter((link) => documentIds.has(link.fromDocumentId))
+            .map((link) => ({
+            source: link.fromDocumentId,
+            target: link.toDocumentId,
+            targetTitle: link.toTitle,
+            weight: link.weight,
+            priority: link.priority
+        }));
+        return { nodes: documents.map((document) => toGraphNode(document, true)), edges };
+    },
+    getGraphSummary: async (agentId) => {
+        const index = await load();
+        const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+        const documentIds = new Set(documents.map((document) => document.id));
+        const edges = index.links
+            .filter((link) => documentIds.has(link.fromDocumentId))
+            .map((link) => ({
+            source: link.fromDocumentId,
+            target: link.toDocumentId,
+            targetTitle: link.toTitle,
+            weight: link.weight,
+            priority: link.priority
+        }));
+        return { nodes: documents.map((document) => toGraphNode(document, false)), edges };
+    },
+    getGraphNode: async (id, agentId) => {
+        const index = await load();
+        const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
+        return document ? toGraphNode(document, true) : undefined;
+    },
+    searchGraphNodeIds: async (query, limit, agentId) => {
+        const index = await load();
+        const normalized = normalizeToken(query);
+        if (normalized.length === 0 || limit <= 0) {
+            return [];
+        }
+        const tokens = tokenize(query);
+        const scored = index.documents
+            .filter((document) => !agentId || document.agentId === agentId)
+            .map((document) => {
+            const score = lexicalFieldScore({
+                documentId: document.id,
+                agentId: document.agentId,
+                title: document.title,
+                path: document.path,
+                chunkId: document.id,
+                chunkOrdinal: 0,
+                content: document.content,
+                tokenCount: 0,
+                tags: document.tags,
+                embedding: []
+            }, tokens);
+            return { id: document.id, score };
+        })
+            .filter((row) => row.score > 0)
+            .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
+            .slice(0, limit);
+        return scored.map((row) => row.id);
+    },
+    listAgents: async () => {
+        const index = await load();
+        const counts = index.documents.reduce((state, document) => {
+            state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
+            return state;
+        }, new Map());
+        return Array.from(counts.entries())
+            .sort((left, right) => left[0].localeCompare(right[0]))
+            .map(([id, documentCount]) => ({ id, documentCount }));
+    }
+});
+// Flatten indexed documents into the stored shape persisted by both backends.
+export const toStoredIndex = (documents) => ({
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    documents: documents.map((document) => document.document),
+    chunks: documents.flatMap((document) => document.chunks),
+    links: documents.flatMap((document) => document.links)
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",
@@ -65,7 +65,8 @@
   },
   "overrides": {
     "hono": "4.12.26",
-    "qs": "6.15.2"
+    "qs": "6.15.2",
+    "fast-uri": "^4.1.0"
   },
   "devDependencies": {
     "@types/node": "^24.9.2",