@andespindola/brainlink 1.0.7 → 1.2.0

package/dist/infrastructure/file-index.js

@@ -1,18 +1,8 @@
 import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { createEmbeddingBuckets, dotProduct } from '../domain/embeddings.js';
-import { bm25TermScore, minMaxNormalize } from '../domain/scoring.js';
-import { selectSemanticCandidates } from './semantic-prefilter.js';
-const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+import { createStoredIndexQueries, emptyIndex, toStoredIndex } from './knowledge-store/stored-index.js';
 const indexCacheMaxEntries = 16;
 const indexCache = new Map();
-const emptyIndex = () => ({
-    version: 1,
-    updatedAt: new Date().toISOString(),
-    documents: [],
-    chunks: [],
-    links: []
-});
 // Raised when index.json exists but cannot be parsed. Search treats it as a
 // signal to fall back to the independent compressed search packs instead of
 // silently returning an empty result set.
@@ -54,6 +44,11 @@ const quarantineCorruptIndex = async (path, cause) => {
         }
     }
 };
+// Exposed so the binary backend can serve reads from a legacy index.json until
+// the next index run materializes the binary store (seamless migration).
+export function readJsonStoredIndex(vaultPath, strict = false) {
+    return readIndex(vaultPath, strict);
+}
 const readIndex = async (vaultPath, strict = false) => {
     const path = indexStoragePath(vaultPath);
     let stats = null;
@@ -126,403 +121,21 @@ const writeIndex = async (vaultPath, index) => {
         index
     });
 };
-const normalizeToken = (value) => value
-    .normalize('NFKD')
-    .replace(/\p{Diacritic}/gu, '')
-    .toLowerCase();
-const tokenize = (query) => query
-    .match(queryTokenPattern)
-    ?.map(normalizeToken)
-    .filter((token) => token.length > 1) ?? [];
-const countOccurrences = (text, token) => {
-    let hits = 0;
-    let cursor = 0;
-    while (cursor < text.length) {
-        const index = text.indexOf(token, cursor);
-        if (index < 0) {
-            break;
-        }
-        hits += 1;
-        cursor = index + token.length;
+// JSON file backend: the original single `index.json` store. Loading/persisting
+// is its own concern; the graph/search/link reads come from the shared query
+// core so this backend stays behavior-identical to the binary backend.
+export const openFileIndex = (vaultPath, options = {}) => ({
+    reset: async () => {
+        await writeIndex(vaultPath, emptyIndex());
+    },
+    saveDocuments: async (documents) => {
+        await writeIndex(vaultPath, toStoredIndex(documents));
+    },
+    ...createStoredIndexQueries({
+        load: () => readIndex(vaultPath),
+        loadStrict: () => readIndex(vaultPath, true)
+    }, options),
+    close: () => {
+        // File-based index has no persistent connection.
     }
-    return hits;
-};
-const titleFieldWeight = 5;
-const tagFieldWeight = 4;
-const pathFieldWeight = 2;
-const contentHitCap = 6;
-const hybridTextWeight = 0.6;
-const hybridSemanticWeight = 0.4;
-const normalizeFields = (row) => ({
-    title: normalizeToken(row.title),
-    path: normalizeToken(row.path),
-    content: normalizeToken(row.content),
-    tags: normalizeToken(row.tags.join(' '))
 });
-// Field-weighted term frequency: title/tag/path matches count more than body
-// matches, and body hits saturate at a cap so a term repeated many times in one
-// chunk cannot dominate. This weighted frequency feeds BM25 below.
-const weightedTermFrequency = (fields, token) => {
-    const titleHits = countOccurrences(fields.title, token);
-    const tagHits = countOccurrences(fields.tags, token);
-    const pathHits = countOccurrences(fields.path, token);
-    const contentHits = countOccurrences(fields.content, token);
-    return (titleHits * titleFieldWeight +
-        tagHits * tagFieldWeight +
-        pathHits * pathFieldWeight +
-        Math.min(contentHits, contentHitCap));
-};
-// Naive additive field score retained for graph-node title/path lookup, where
-// the candidate set is documents (not chunks) and corpus-wide BM25 statistics
-// add no signal over a direct weighted hit count.
-const lexicalFieldScore = (row, tokens) => {
-    if (tokens.length === 0) {
-        return 0;
-    }
-    const fields = normalizeFields(row);
-    return tokens.reduce((score, token) => score + weightedTermFrequency(fields, token), 0);
-};
-// A single chunk with a non-finite cached tokenCount (legacy index, partial
-// write, manual edit) would otherwise poison the shared averageLength and zero
-// out lexical scoring for the whole corpus, so coerce to a safe positive length.
-const safeDocumentLength = (tokenCount) => Number.isFinite(tokenCount) && tokenCount > 0 ? tokenCount : 1;
-const buildCorpusStatistics = (prepared, tokens) => {
-    const documentCount = prepared.length;
-    const totalLength = prepared.reduce((sum, entry) => sum + safeDocumentLength(entry.row.tokenCount), 0);
-    const documentFrequencyByTerm = new Map();
-    for (const term of tokens) {
-        let documentFrequency = 0;
-        for (const entry of prepared) {
-            if ((entry.weightedTermFrequencies.get(term) ?? 0) > 0) {
-                documentFrequency += 1;
-            }
-        }
-        documentFrequencyByTerm.set(term, documentFrequency);
-    }
-    return {
-        documentCount,
-        averageLength: documentCount > 0 ? totalLength / documentCount : 0,
-        documentFrequencyByTerm
-    };
-};
-const bm25TextScore = (entry, tokens, stats) => tokens.reduce((score, term) => score +
-    bm25TermScore({
-        termFrequency: entry.weightedTermFrequencies.get(term) ?? 0,
-        documentLength: safeDocumentLength(entry.row.tokenCount),
-        documentCount: stats.documentCount,
-        documentFrequency: stats.documentFrequencyByTerm.get(term) ?? 0,
-        averageLength: stats.averageLength
-    }), 0);
-const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? dotProduct(queryEmbedding, row.embedding) : 0;
-const rangeOf = (values) => values.reduce((range, value) => ({ min: Math.min(range.min, value), max: Math.max(range.max, value) }), { min: Infinity, max: -Infinity });
-const toResult = (row, mode, score, text, semantic) => ({
-    documentId: row.documentId,
-    agentId: row.agentId,
-    title: row.title,
-    path: row.path,
-    chunkId: row.chunkId,
-    chunkOrdinal: row.chunkOrdinal,
-    content: row.content,
-    score,
-    textScore: text,
-    semanticScore: semantic,
-    tokenCount: row.tokenCount,
-    searchMode: mode,
-    tags: row.tags
-});
-const toGraphLink = (link, documentsById) => {
-    const source = documentsById.get(link.fromDocumentId);
-    const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
-    return {
-        agentId: source?.agentId ?? 'shared',
-        fromTitle: source?.title ?? 'Unknown',
-        fromPath: source?.path ?? 'Unknown',
-        toTitle: target?.title ?? link.toTitle,
-        toPath: target?.path ?? null,
-        weight: link.weight,
-        priority: link.priority
-    };
-};
-export const openFileIndex = (vaultPath) => {
-    const load = async () => readIndex(vaultPath);
-    const persist = async (index) => writeIndex(vaultPath, index);
-    return {
-        reset: async () => {
-            await persist(emptyIndex());
-        },
-        saveDocuments: async (documents) => {
-            const chunks = documents.flatMap((document) => document.chunks);
-            const links = documents.flatMap((document) => document.links);
-            await persist({
-                version: 1,
-                updatedAt: new Date().toISOString(),
-                documents: documents.map((document) => document.document),
-                chunks,
-                links
-            });
-        },
-        getIndexedDocuments: async (agentId) => {
-            const index = await load();
-            const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
-            const selectedDocumentIds = new Set(documents.map((document) => document.id));
-            const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
-                if (!selectedDocumentIds.has(chunk.documentId)) {
-                    return state;
-                }
-                const current = state.get(chunk.documentId) ?? [];
-                current.push(chunk);
-                state.set(chunk.documentId, current);
-                return state;
-            }, new Map());
-            const linksByDocumentId = index.links.reduce((state, link) => {
-                if (!selectedDocumentIds.has(link.fromDocumentId)) {
-                    return state;
-                }
-                const current = state.get(link.fromDocumentId) ?? [];
-                current.push(link);
-                state.set(link.fromDocumentId, current);
-                return state;
-            }, new Map());
-            return documents
-                .map((document) => ({
-                document,
-                chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
-                links: linksByDocumentId.get(document.id) ?? []
-            }))
-                .sort((left, right) => left.document.path.localeCompare(right.document.path));
-        },
-        search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
-            // Strict load: a corrupt index throws so the caller can fall back to the
-            // independent search packs rather than reporting "no results".
-            const index = await readIndex(vaultPath, true);
-            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
-            const rows = index.chunks.flatMap((chunk) => {
-                const document = documentsById.get(chunk.documentId);
-                if (!document) {
-                    return [];
-                }
-                if (agentId && document.agentId !== agentId) {
-                    return [];
-                }
-                return [
-                    {
-                        documentId: document.id,
-                        agentId: document.agentId,
-                        title: document.title,
-                        path: document.path,
-                        chunkId: chunk.id,
-                        chunkOrdinal: chunk.ordinal,
-                        content: chunk.content,
-                        tokenCount: chunk.tokenCount,
-                        tags: document.tags,
-                        embedding: chunk.embedding,
-                        buckets: chunk.buckets
-                    }
-                ];
-            });
-            const tokens = tokenize(query);
-            // Pure-semantic scoring on a large vault can skip chunks that share no
-            // embedding bucket with the query. Hybrid/fts keep every row so lexical
-            // matches are never dropped; the prefilter also falls back to a full scan
-            // on small or partially-indexed vaults.
-            const candidates = mode === 'semantic' && queryEmbedding.length > 0
-                ? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
-                : rows;
-            // Pass 1: field-weighted term frequencies per candidate, then corpus-wide
-            // statistics (document frequency, average length) needed for BM25 IDF and
-            // length normalization.
-            const prepared = candidates.map((row) => {
-                const fields = normalizeFields(row);
-                const weightedTermFrequencies = new Map();
-                for (const term of tokens) {
-                    const frequency = weightedTermFrequency(fields, term);
-                    if (frequency > 0) {
-                        weightedTermFrequencies.set(term, frequency);
-                    }
-                }
-                return { row, weightedTermFrequencies };
-            });
-            const stats = buildCorpusStatistics(prepared, tokens);
-            // Pass 2: raw lexical (BM25) and semantic (cosine) component scores.
-            const componentScored = prepared.map((entry) => ({
-                row: entry.row,
-                text: tokens.length > 0 ? bm25TextScore(entry, tokens, stats) : 0,
-                semantic: semanticScore(entry.row, queryEmbedding)
-            }));
-            // Filter on raw relevance before normalization so a legitimate match that
-            // happens to be the weakest in its mode is never min-max scaled to zero
-            // and dropped. fts stays lexical-only and semantic stays vector-only.
-            const relevant = componentScored.filter((entry) => {
-                if (tokens.length === 0) {
-                    return true;
-                }
-                if (mode === 'fts') {
-                    return entry.text > 0;
-                }
-                if (mode === 'semantic') {
-                    return entry.semantic > 0;
-                }
-                return entry.text > 0 || entry.semantic > 0;
-            });
-            if (relevant.length === 0) {
-                return [];
-            }
-            const textRange = rangeOf(relevant.map((entry) => entry.text));
-            const semanticRange = rangeOf(relevant.map((entry) => entry.semantic));
-            const finalScore = (entry) => {
-                if (mode === 'fts') {
-                    return entry.text;
-                }
-                if (mode === 'semantic') {
-                    return entry.semantic;
-                }
-                // Hybrid mixes the two components on a shared [0, 1] scale so neither the
-                // unbounded BM25 magnitude nor the bounded cosine value can swamp the
-                // other, replacing the previous fixed `text + semantic * 8` blend.
-                const textNorm = minMaxNormalize(entry.text, textRange.min, textRange.max);
-                const semanticNorm = minMaxNormalize(entry.semantic, semanticRange.min, semanticRange.max);
-                return textNorm * hybridTextWeight + semanticNorm * hybridSemanticWeight;
-            };
-            return relevant
-                .map((entry) => toResult(entry.row, mode, finalScore(entry), entry.text, entry.semantic))
-                .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
-                .slice(0, Math.max(0, limit));
-        },
-        listLinks: async (agentId) => {
-            const index = await load();
-            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
-            return index.links
-                .filter((link) => {
-                const source = documentsById.get(link.fromDocumentId);
-                return agentId ? source?.agentId === agentId : true;
-            })
-                .map((link) => toGraphLink(link, documentsById))
-                .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
-        },
-        listBacklinks: async (title, agentId) => {
-            const index = await load();
-            const titleKey = title.toLowerCase();
-            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
-            return index.links
-                .filter((link) => link.toTitle.toLowerCase() === titleKey)
-                .filter((link) => {
-                const source = documentsById.get(link.fromDocumentId);
-                return agentId ? source?.agentId === agentId : true;
-            })
-                .map((link) => toGraphLink(link, documentsById))
-                .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
-        },
-        getGraph: async (agentId) => {
-            const index = await load();
-            const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
-            const documentIds = new Set(documents.map((document) => document.id));
-            const edges = index.links
-                .filter((link) => documentIds.has(link.fromDocumentId))
-                .map((link) => ({
-                source: link.fromDocumentId,
-                target: link.toDocumentId,
-                targetTitle: link.toTitle,
-                weight: link.weight,
-                priority: link.priority
-            }));
-            return {
-                nodes: documents.map((document) => ({
-                    id: document.id,
-                    agentId: document.agentId,
-                    title: document.title,
-                    path: document.path,
-                    content: document.content,
-                    tags: document.tags,
-                    contextLinks: document.contextLinks ?? []
-                })),
-                edges
-            };
-        },
-        getGraphSummary: async (agentId) => {
-            const graph = await (async () => {
-                const index = await load();
-                const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
-                const documentIds = new Set(documents.map((document) => document.id));
-                const edges = index.links
-                    .filter((link) => documentIds.has(link.fromDocumentId))
-                    .map((link) => ({
-                    source: link.fromDocumentId,
-                    target: link.toDocumentId,
-                    targetTitle: link.toTitle,
-                    weight: link.weight,
-                    priority: link.priority
-                }));
-                return {
-                    nodes: documents.map((document) => ({
-                        id: document.id,
-                        agentId: document.agentId,
-                        title: document.title,
-                        path: document.path,
-                        content: '',
-                        tags: document.tags,
-                        contextLinks: document.contextLinks ?? []
-                    })),
-                    edges
-                };
-            })();
-            return graph;
-        },
-        getGraphNode: async (id, agentId) => {
-            const index = await load();
-            const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
-            return document
-                ? {
-                    id: document.id,
-                    agentId: document.agentId,
-                    title: document.title,
-                    path: document.path,
-                    content: document.content,
-                    tags: document.tags,
-                    contextLinks: document.contextLinks ?? []
-                }
-                : undefined;
-        },
-        searchGraphNodeIds: async (query, limit, agentId) => {
-            const index = await load();
-            const normalized = normalizeToken(query);
-            if (normalized.length === 0 || limit <= 0) {
-                return [];
-            }
-            const tokens = tokenize(query);
-            const scored = index.documents
-                .filter((document) => (!agentId || document.agentId === agentId))
-                .map((document) => {
-                const score = lexicalFieldScore({
-                    documentId: document.id,
-                    agentId: document.agentId,
-                    title: document.title,
-                    path: document.path,
-                    chunkId: document.id,
-                    chunkOrdinal: 0,
-                    content: document.content,
-                    tokenCount: 0,
-                    tags: document.tags,
-                    embedding: []
-                }, tokens);
-                return { id: document.id, score };
-            })
-                .filter((row) => row.score > 0)
-                .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
-                .slice(0, limit);
-            return scored.map((row) => row.id);
-        },
-        listAgents: async () => {
-            const index = await load();
-            const counts = index.documents.reduce((state, document) => {
-                state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
-                return state;
-            }, new Map());
-            return Array.from(counts.entries())
-                .sort((left, right) => left[0].localeCompare(right[0]))
-                .map(([id, documentCount]) => ({ id, documentCount }));
-        },
-        close: () => {
-            // File-based index has no persistent connection.
-        }
-    };
-};

package/dist/infrastructure/index-signature.js

@@ -0,0 +1,27 @@
+import { stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { indexStoragePath } from './file-index.js';
+const fileSignature = async (path) => {
+    try {
+        const info = await stat(path);
+        return `${Math.floor(info.mtimeMs)}:${info.size}`;
+    }
+    catch {
+        return '0:0';
+    }
+};
+// Commit signature of the derived index, independent of which storage backend is
+// active. The JSON backend commits `index.json`; the binary backend commits
+// `store/meta.json` (written last) and removes `index.json`. Combining both file
+// signatures means the value changes on any index write under either backend —
+// and on a backend switch, when one file appears and the other disappears — so
+// the in-memory search/context caches invalidate correctly. On the binary
+// backend `index.json` is absent, so keying a cache on it alone would never
+// change and would serve stale results after an edit.
+export const indexCommitSignature = async (vaultPath) => {
+    const [jsonSignature, binarySignature] = await Promise.all([
+        fileSignature(indexStoragePath(vaultPath)),
+        fileSignature(join(vaultPath, '.brainlink', 'store', 'meta.json'))
+    ]);
+    return `${jsonSignature}|${binarySignature}`;
+};

package/dist/infrastructure/index-state.js

@@ -29,6 +29,11 @@ export const readIndexState = async (vaultPath) => {
             updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
             chunkSize: typeof parsed.chunkSize === 'number' ? parsed.chunkSize : 1200,
             embeddingProvider: typeof parsed.embeddingProvider === 'string' ? parsed.embeddingProvider : 'none',
+            // Absent on states written before the backend was tracked: default to the
+            // historical 'json' backend so a json vault does not spuriously reindex,
+            // while a vault already switched to binary reindexes once to materialize
+            // the new store (which also writes the int8 scan index).
+            storageBackend: typeof parsed.storageBackend === 'string' ? parsed.storageBackend : 'json',
             graphLinkModelVersion: typeof parsed.graphLinkModelVersion === 'number' ? parsed.graphLinkModelVersion : 1,
             searchPackRowChunkSize: typeof parsed.searchPackRowChunkSize === 'number' ? parsed.searchPackRowChunkSize : 5_000,
             searchPackCompressionLevel: typeof parsed.searchPackCompressionLevel === 'number' ? parsed.searchPackCompressionLevel : 5,
@@ -47,6 +52,7 @@ export const writeIndexState = async (vaultPath, state) => {
         updatedAt: new Date().toISOString(),
         chunkSize: state.chunkSize,
         embeddingProvider: state.embeddingProvider,
+        storageBackend: state.storageBackend,
         graphLinkModelVersion: state.graphLinkModelVersion,
         searchPackRowChunkSize: state.searchPackRowChunkSize,
         searchPackCompressionLevel: state.searchPackCompressionLevel,