@andespindola/brainlink 0.1.0-beta.13 → 0.1.0-beta.131

package/dist/infrastructure/file-index.js

@@ -0,0 +1,358 @@
+import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { cosineSimilarity } from '../domain/embeddings.js';
+const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const indexCacheMaxEntries = 16;
+const indexCache = new Map();
+const emptyIndex = () => ({
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    documents: [],
+    chunks: [],
+    links: []
+});
+export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
+const readIndex = async (vaultPath) => {
+    const path = indexStoragePath(vaultPath);
+    let stats = null;
+    try {
+        const fileStats = await stat(path);
+        stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            indexCache.delete(path);
+            return emptyIndex();
+        }
+        return emptyIndex();
+    }
+    const cached = indexCache.get(path);
+    if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
+        return cached.index;
+    }
+    try {
+        const parsed = JSON.parse(await readFile(path, 'utf8'));
+        const loaded = {
+            version: 1,
+            updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
+            documents: Array.isArray(parsed.documents) ? parsed.documents : [],
+            chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
+            links: Array.isArray(parsed.links) ? parsed.links : []
+        };
+        indexCache.set(path, { ...stats, index: loaded });
+        if (indexCache.size > indexCacheMaxEntries) {
+            const oldest = indexCache.keys().next().value;
+            if (typeof oldest === 'string') {
+                indexCache.delete(oldest);
+            }
+        }
+        return loaded;
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            indexCache.delete(path);
+            return emptyIndex();
+        }
+        return emptyIndex();
+    }
+};
+const writeIndex = async (vaultPath, index) => {
+    const target = indexStoragePath(vaultPath);
+    const temp = `${target}.tmp`;
+    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+    await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
+    await rename(temp, target);
+    const fileStats = await stat(target);
+    indexCache.set(target, {
+        mtimeMs: fileStats.mtimeMs,
+        size: fileStats.size,
+        index
+    });
+};
+const normalizeToken = (value) => value
+    .normalize('NFKD')
+    .replace(/\p{Diacritic}/gu, '')
+    .toLowerCase();
+const tokenize = (query) => query
+    .match(queryTokenPattern)
+    ?.map(normalizeToken)
+    .filter((token) => token.length > 1) ?? [];
+const countOccurrences = (text, token) => {
+    let hits = 0;
+    let cursor = 0;
+    while (cursor < text.length) {
+        const index = text.indexOf(token, cursor);
+        if (index < 0) {
+            break;
+        }
+        hits += 1;
+        cursor = index + token.length;
+    }
+    return hits;
+};
+const textScore = (row, tokens) => {
+    if (tokens.length === 0) {
+        return 0;
+    }
+    const title = normalizeToken(row.title);
+    const path = normalizeToken(row.path);
+    const content = normalizeToken(row.content);
+    const tags = normalizeToken(row.tags.join(' '));
+    return tokens.reduce((score, token) => {
+        const titleHits = countOccurrences(title, token);
+        const tagHits = countOccurrences(tags, token);
+        const pathHits = countOccurrences(path, token);
+        const contentHits = countOccurrences(content, token);
+        return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 6);
+    }, 0);
+};
+const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? cosineSimilarity(queryEmbedding, row.embedding) : 0;
+const toResult = (row, mode, text, semantic) => {
+    const score = mode === 'fts' ? text : mode === 'semantic' ? semantic : text + semantic * 8;
+    return {
+        documentId: row.documentId,
+        agentId: row.agentId,
+        title: row.title,
+        path: row.path,
+        chunkId: row.chunkId,
+        chunkOrdinal: row.chunkOrdinal,
+        content: row.content,
+        score,
+        textScore: text,
+        semanticScore: semantic,
+        searchMode: mode,
+        tags: row.tags
+    };
+};
+const toGraphLink = (link, documentsById) => {
+    const source = documentsById.get(link.fromDocumentId);
+    const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
+    return {
+        agentId: source?.agentId ?? 'shared',
+        fromTitle: source?.title ?? 'Unknown',
+        fromPath: source?.path ?? 'Unknown',
+        toTitle: target?.title ?? link.toTitle,
+        toPath: target?.path ?? null,
+        weight: link.weight,
+        priority: link.priority
+    };
+};
+export const openFileIndex = (vaultPath) => {
+    const load = async () => readIndex(vaultPath);
+    const persist = async (index) => writeIndex(vaultPath, index);
+    return {
+        reset: async () => {
+            await persist(emptyIndex());
+        },
+        saveDocuments: async (documents) => {
+            const chunks = documents.flatMap((document) => document.chunks);
+            const links = documents.flatMap((document) => document.links);
+            await persist({
+                version: 1,
+                updatedAt: new Date().toISOString(),
+                documents: documents.map((document) => document.document),
+                chunks,
+                links
+            });
+        },
+        getIndexedDocuments: async (agentId) => {
+            const index = await load();
+            const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+            const selectedDocumentIds = new Set(documents.map((document) => document.id));
+            const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
+                if (!selectedDocumentIds.has(chunk.documentId)) {
+                    return state;
+                }
+                const current = state.get(chunk.documentId) ?? [];
+                current.push(chunk);
+                state.set(chunk.documentId, current);
+                return state;
+            }, new Map());
+            const linksByDocumentId = index.links.reduce((state, link) => {
+                if (!selectedDocumentIds.has(link.fromDocumentId)) {
+                    return state;
+                }
+                const current = state.get(link.fromDocumentId) ?? [];
+                current.push(link);
+                state.set(link.fromDocumentId, current);
+                return state;
+            }, new Map());
+            return documents
+                .map((document) => ({
+                document,
+                chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
+                links: linksByDocumentId.get(document.id) ?? []
+            }))
+                .sort((left, right) => left.document.path.localeCompare(right.document.path));
+        },
+        search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
+            const index = await load();
+            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+            const rows = index.chunks.flatMap((chunk) => {
+                const document = documentsById.get(chunk.documentId);
+                if (!document) {
+                    return [];
+                }
+                if (agentId && document.agentId !== agentId) {
+                    return [];
+                }
+                return [
+                    {
+                        documentId: document.id,
+                        agentId: document.agentId,
+                        title: document.title,
+                        path: document.path,
+                        chunkId: chunk.id,
+                        chunkOrdinal: chunk.ordinal,
+                        content: chunk.content,
+                        tags: document.tags,
+                        embedding: chunk.embedding
+                    }
+                ];
+            });
+            const tokens = tokenize(query);
+            const results = rows
+                .map((row) => {
+                const text = textScore(row, tokens);
+                const semantic = semanticScore(row, queryEmbedding);
+                return toResult(row, mode, text, semantic);
+            })
+                .filter((row) => row.score > 0 || tokens.length === 0)
+                .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
+                .slice(0, Math.max(0, limit));
+            return results;
+        },
+        listLinks: async (agentId) => {
+            const index = await load();
+            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+            return index.links
+                .filter((link) => {
+                const source = documentsById.get(link.fromDocumentId);
+                return agentId ? source?.agentId === agentId : true;
+            })
+                .map((link) => toGraphLink(link, documentsById))
+                .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
+        },
+        listBacklinks: async (title, agentId) => {
+            const index = await load();
+            const titleKey = title.toLowerCase();
+            const documentsById = new Map(index.documents.map((document) => [document.id, document]));
+            return index.links
+                .filter((link) => link.toTitle.toLowerCase() === titleKey)
+                .filter((link) => {
+                const source = documentsById.get(link.fromDocumentId);
+                return agentId ? source?.agentId === agentId : true;
+            })
+                .map((link) => toGraphLink(link, documentsById))
+                .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
+        },
+        getGraph: async (agentId) => {
+            const index = await load();
+            const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+            const documentIds = new Set(documents.map((document) => document.id));
+            const edges = index.links
+                .filter((link) => documentIds.has(link.fromDocumentId))
+                .map((link) => ({
+                source: link.fromDocumentId,
+                target: link.toDocumentId,
+                targetTitle: link.toTitle,
+                weight: link.weight,
+                priority: link.priority
+            }));
+            return {
+                nodes: documents.map((document) => ({
+                    id: document.id,
+                    agentId: document.agentId,
+                    title: document.title,
+                    path: document.path,
+                    content: document.content,
+                    tags: document.tags
+                })),
+                edges
+            };
+        },
+        getGraphSummary: async (agentId) => {
+            const graph = await (async () => {
+                const index = await load();
+                const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+                const documentIds = new Set(documents.map((document) => document.id));
+                const edges = index.links
+                    .filter((link) => documentIds.has(link.fromDocumentId))
+                    .map((link) => ({
+                    source: link.fromDocumentId,
+                    target: link.toDocumentId,
+                    targetTitle: link.toTitle,
+                    weight: link.weight,
+                    priority: link.priority
+                }));
+                return {
+                    nodes: documents.map((document) => ({
+                        id: document.id,
+                        agentId: document.agentId,
+                        title: document.title,
+                        path: document.path,
+                        content: '',
+                        tags: document.tags
+                    })),
+                    edges
+                };
+            })();
+            return graph;
+        },
+        getGraphNode: async (id, agentId) => {
+            const index = await load();
+            const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
+            return document
+                ? {
+                    id: document.id,
+                    agentId: document.agentId,
+                    title: document.title,
+                    path: document.path,
+                    content: document.content,
+                    tags: document.tags
+                }
+                : undefined;
+        },
+        searchGraphNodeIds: async (query, limit, agentId) => {
+            const index = await load();
+            const normalized = normalizeToken(query);
+            if (normalized.length === 0 || limit <= 0) {
+                return [];
+            }
+            const tokens = tokenize(query);
+            const scored = index.documents
+                .filter((document) => (!agentId || document.agentId === agentId))
+                .map((document) => {
+                const score = textScore({
+                    documentId: document.id,
+                    agentId: document.agentId,
+                    title: document.title,
+                    path: document.path,
+                    chunkId: document.id,
+                    chunkOrdinal: 0,
+                    content: document.content,
+                    tags: document.tags,
+                    embedding: []
+                }, tokens);
+                return { id: document.id, score };
+            })
+                .filter((row) => row.score > 0)
+                .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
+                .slice(0, limit);
+            return scored.map((row) => row.id);
+        },
+        listAgents: async () => {
+            const index = await load();
+            const counts = index.documents.reduce((state, document) => {
+                state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
+                return state;
+            }, new Map());
+            return Array.from(counts.entries())
+                .sort((left, right) => left[0].localeCompare(right[0]))
+                .map(([id, documentCount]) => ({ id, documentCount }));
+        },
+        close: () => {
+            // File-based index has no persistent connection.
+        }
+    };
+};

package/dist/infrastructure/file-system-vault.js

@@ -76,6 +76,21 @@ export const readMarkdownFiles = async (vaultPath) => {
         };
     }));
 };
+export const readMarkdownFileSummaries = async (vaultPath) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const paths = await walkMarkdownFiles(absoluteVaultPath);
+    const summaries = await Promise.all(paths.map(async (absolutePath) => {
+        const fileStats = await stat(absolutePath);
+        return {
+            absolutePath,
+            relativePath: relative(absoluteVaultPath, absolutePath),
+            createdAt: fileStats.birthtime,
+            updatedAt: fileStats.mtime,
+            size: fileStats.size
+        };
+    }));
+    return summaries.sort((left, right) => left.relativePath.localeCompare(right.relativePath));
+};
 export const listVaultFiles = async (vaultPath) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
     return walkVaultFiles(absoluteVaultPath);

package/dist/infrastructure/index-state.js

@@ -0,0 +1,58 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+const indexStateFileName = 'index-state.json';
+const toIndexStatePath = (vaultPath) => join(vaultPath, '.brainlink', indexStateFileName);
+export const readIndexState = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toIndexStatePath(vaultPath), 'utf8'));
+        if (parsed.version !== 1 || !Array.isArray(parsed.files)) {
+            return null;
+        }
+        const files = parsed.files.flatMap((entry) => {
+            if (!entry || typeof entry !== 'object') {
+                return [];
+            }
+            const row = entry;
+            if (typeof row.path !== 'string' || typeof row.mtimeMs !== 'number' || typeof row.size !== 'number') {
+                return [];
+            }
+            return [
+                {
+                    path: row.path,
+                    mtimeMs: row.mtimeMs,
+                    size: row.size
+                }
+            ];
+        });
+        return {
+            version: 1,
+            updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
+            chunkSize: typeof parsed.chunkSize === 'number' ? parsed.chunkSize : 1200,
+            embeddingProvider: typeof parsed.embeddingProvider === 'string' ? parsed.embeddingProvider : 'none',
+            graphLinkModelVersion: typeof parsed.graphLinkModelVersion === 'number' ? parsed.graphLinkModelVersion : 1,
+            searchPackRowChunkSize: typeof parsed.searchPackRowChunkSize === 'number' ? parsed.searchPackRowChunkSize : 5_000,
+            searchPackCompressionLevel: typeof parsed.searchPackCompressionLevel === 'number' ? parsed.searchPackCompressionLevel : 5,
+            searchPackUseDictionary: typeof parsed.searchPackUseDictionary === 'boolean' ? parsed.searchPackUseDictionary : true,
+            files,
+            pendingPackChanges: typeof parsed.pendingPackChanges === 'number' && parsed.pendingPackChanges >= 0 ? parsed.pendingPackChanges : 0
+        };
+    }
+    catch {
+        return null;
+    }
+};
+export const writeIndexState = async (vaultPath, state) => {
+    const payload = {
+        version: 1,
+        updatedAt: new Date().toISOString(),
+        chunkSize: state.chunkSize,
+        embeddingProvider: state.embeddingProvider,
+        graphLinkModelVersion: state.graphLinkModelVersion,
+        searchPackRowChunkSize: state.searchPackRowChunkSize,
+        searchPackCompressionLevel: state.searchPackCompressionLevel,
+        searchPackUseDictionary: state.searchPackUseDictionary,
+        files: [...state.files].sort((left, right) => left.path.localeCompare(right.path)),
+        pendingPackChanges: Math.max(0, Math.floor(state.pendingPackChanges))
+    };
+    await writeFile(toIndexStatePath(vaultPath), `${JSON.stringify(payload)}\n`, 'utf8');
+};

package/dist/infrastructure/private-pack-codec.js

@@ -1,13 +1,25 @@
 import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
-import { brotliCompressSync, brotliDecompressSync } from 'node:zlib';
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { getBrainlinkHomePath } from './paths.js';
 const magic = Buffer.from('BLPK2', 'ascii');
-const version = 1;
+const legacyVersion = 1;
+const currentVersion = 2;
 const nonceLength = 12;
 const authTagLength = 16;
 const algorithm = 'aes-256-gcm';
+const compressionLevelMask = 0x0f;
+const compressionDictionaryMask = 0x10;
+const defaultCompressionLevel = 5;
+const builtinDictionary = Buffer.from([
+    '"documentId","agentId","title","path","chunkId","chunkOrdinal","content","tags"',
+    '"searchMode","textScore","semanticScore","weight","priority","shared"',
+    'agents/shared memory-hub architecture context index search graph markdown tags links',
+    '#memory #architecture #context #graph #search #index [[Memory Hub]] [[Architecture]]',
+    'The quick brown fox jumps over the lazy dog. Brainlink context package metadata.',
+    'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-:/.#[]{}(), '
+].join('\n'), 'utf8');
 const keyFilePath = (vaultPath) => {
     const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
     return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
@@ -40,34 +52,83 @@ const parseHeader = (payload) => {
         throw new Error('Invalid private pack payload: too short.');
     }
     const payloadMagic = payload.subarray(0, magic.length);
-    const payloadVersion = payload[magic.length];
-    if (!payloadMagic.equals(magic) || payloadVersion !== version) {
+    const payloadVersion = payload[magic.length] ?? 0;
+    if (!payloadMagic.equals(magic) || (payloadVersion !== legacyVersion && payloadVersion !== currentVersion)) {
         throw new Error('Invalid private pack payload: unsupported format.');
     }
-    const nonceStart = magic.length + 1;
+    const hasCompressionSettings = payloadVersion >= 2;
+    const settingsByte = hasCompressionSettings ? payload[magic.length + 1] ?? 0 : null;
+    const nonceStart = magic.length + 1 + (hasCompressionSettings ? 1 : 0);
     const authTagStart = nonceStart + nonceLength;
     const dataStart = authTagStart + authTagLength;
     return {
+        compression: settingsByte != null
+            ? {
+                compressionLevel: settingsByte & compressionLevelMask,
+                useDictionary: (settingsByte & compressionDictionaryMask) !== 0
+            }
+            : {
+                compressionLevel: defaultCompressionLevel,
+                useDictionary: false
+            },
         nonce: payload.subarray(nonceStart, authTagStart),
         authTag: payload.subarray(authTagStart, dataStart),
         ciphertext: payload.subarray(dataStart)
     };
 };
-export const encodePrivatePack = async (vaultPath, content) => {
+const toCompressionLevel = (value) => {
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return defaultCompressionLevel;
+    }
+    const normalized = Math.round(value);
+    if (normalized < 0) {
+        return 0;
+    }
+    if (normalized > 11) {
+        return 11;
+    }
+    return normalized;
+};
+const encodeCompressionSettings = (settings) => (settings.compressionLevel & compressionLevelMask) | (settings.useDictionary ? compressionDictionaryMask : 0);
+const brotliEncode = (content, settings) => {
+    const options = {
+        params: {
+            [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
+            [zlibConstants.BROTLI_PARAM_QUALITY]: settings.compressionLevel
+        }
+    };
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliCompressSync(content, options);
+};
+const brotliDecode = (content, settings) => {
+    const options = {};
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliDecompressSync(content, options);
+};
+export const encodePrivatePack = async (vaultPath, content, settings) => {
     const key = await readOrCreateKey(vaultPath);
     const nonce = randomBytes(nonceLength);
-    const compressed = brotliCompressSync(content);
+    const normalizedSettings = {
+        compressionLevel: toCompressionLevel(settings?.compressionLevel),
+        useDictionary: settings?.useDictionary ?? true
+    };
+    const compressed = brotliEncode(content, normalizedSettings);
     const cipher = createCipheriv(algorithm, key, nonce);
     const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
     const authTag = cipher.getAuthTag();
-    return Buffer.concat([magic, Buffer.from([version]), nonce, authTag, ciphertext]);
+    const settingsByte = Buffer.from([encodeCompressionSettings(normalizedSettings)]);
+    return Buffer.concat([magic, Buffer.from([currentVersion]), settingsByte, nonce, authTag, ciphertext]);
 };
 export const decodePrivatePack = async (vaultPath, payload) => {
     const key = await readOrCreateKey(vaultPath);
-    const { nonce, authTag, ciphertext } = parseHeader(payload);
+    const { nonce, authTag, ciphertext, compression } = parseHeader(payload);
     const decipher = createDecipheriv(algorithm, key, nonce);
     decipher.setAuthTag(authTag);
     const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
-    return brotliDecompressSync(compressed);
+    return brotliDecode(compressed, compression);
 };
 export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);