@andespindola/brainlink 0.1.0-beta.15 → 0.1.0-beta.151

package/dist/infrastructure/file-index.js

@@ -1,7 +1,9 @@
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { cosineSimilarity } from '../domain/embeddings.js';
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const indexCacheMaxEntries = 16;
+const indexCache = new Map();
 const emptyIndex = () => ({
     version: 1,
     updatedAt: new Date().toISOString(),
@@ -11,18 +13,44 @@ const emptyIndex = () => ({
 });
 export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
 const readIndex = async (vaultPath) => {
+    const path = indexStoragePath(vaultPath);
+    let stats = null;
     try {
-        const parsed = JSON.parse(await readFile(indexStoragePath(vaultPath), 'utf8'));
-        return {
+        const fileStats = await stat(path);
+        stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            indexCache.delete(path);
+            return emptyIndex();
+        }
+        return emptyIndex();
+    }
+    const cached = indexCache.get(path);
+    if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
+        return cached.index;
+    }
+    try {
+        const parsed = JSON.parse(await readFile(path, 'utf8'));
+        const loaded = {
             version: 1,
             updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
             documents: Array.isArray(parsed.documents) ? parsed.documents : [],
             chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
             links: Array.isArray(parsed.links) ? parsed.links : []
         };
+        indexCache.set(path, { ...stats, index: loaded });
+        if (indexCache.size > indexCacheMaxEntries) {
+            const oldest = indexCache.keys().next().value;
+            if (typeof oldest === 'string') {
+                indexCache.delete(oldest);
+            }
+        }
+        return loaded;
     }
     catch (error) {
         if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            indexCache.delete(path);
             return emptyIndex();
         }
         return emptyIndex();
@@ -34,6 +62,12 @@ const writeIndex = async (vaultPath, index) => {
     await mkdir(dirname(target), { recursive: true, mode: 0o700 });
     await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
     await rename(temp, target);
+    const fileStats = await stat(target);
+    indexCache.set(target, {
+        mtimeMs: fileStats.mtimeMs,
+        size: fileStats.size,
+        index
+    });
 };
 const normalizeToken = (value) => value
     .normalize('NFKD')
@@ -81,6 +115,7 @@ const toResult = (row, mode, text, semantic) => {
         title: row.title,
         path: row.path,
         chunkId: row.chunkId,
+        chunkOrdinal: row.chunkOrdinal,
         content: row.content,
         score,
         textScore: text,
@@ -120,6 +155,36 @@ export const openFileIndex = (vaultPath) => {
                 links
             });
         },
+        getIndexedDocuments: async (agentId) => {
+            const index = await load();
+            const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
+            const selectedDocumentIds = new Set(documents.map((document) => document.id));
+            const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
+                if (!selectedDocumentIds.has(chunk.documentId)) {
+                    return state;
+                }
+                const current = state.get(chunk.documentId) ?? [];
+                current.push(chunk);
+                state.set(chunk.documentId, current);
+                return state;
+            }, new Map());
+            const linksByDocumentId = index.links.reduce((state, link) => {
+                if (!selectedDocumentIds.has(link.fromDocumentId)) {
+                    return state;
+                }
+                const current = state.get(link.fromDocumentId) ?? [];
+                current.push(link);
+                state.set(link.fromDocumentId, current);
+                return state;
+            }, new Map());
+            return documents
+                .map((document) => ({
+                document,
+                chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
+                links: linksByDocumentId.get(document.id) ?? []
+            }))
+                .sort((left, right) => left.document.path.localeCompare(right.document.path));
+        },
         search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
             const index = await load();
             const documentsById = new Map(index.documents.map((document) => [document.id, document]));
@@ -138,6 +203,7 @@ export const openFileIndex = (vaultPath) => {
                         title: document.title,
                         path: document.path,
                         chunkId: chunk.id,
+                        chunkOrdinal: chunk.ordinal,
                         content: chunk.content,
                         tags: document.tags,
                         embedding: chunk.embedding
@@ -263,6 +329,7 @@ export const openFileIndex = (vaultPath) => {
                     title: document.title,
                     path: document.path,
                     chunkId: document.id,
+                    chunkOrdinal: 0,
                     content: document.content,
                     tags: document.tags,
                     embedding: []

package/dist/infrastructure/file-system-vault.js

@@ -76,6 +76,21 @@ export const readMarkdownFiles = async (vaultPath) => {
         };
     }));
 };
+export const readMarkdownFileSummaries = async (vaultPath) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const paths = await walkMarkdownFiles(absoluteVaultPath);
+    const summaries = await Promise.all(paths.map(async (absolutePath) => {
+        const fileStats = await stat(absolutePath);
+        return {
+            absolutePath,
+            relativePath: relative(absoluteVaultPath, absolutePath),
+            createdAt: fileStats.birthtime,
+            updatedAt: fileStats.mtime,
+            size: fileStats.size
+        };
+    }));
+    return summaries.sort((left, right) => left.relativePath.localeCompare(right.relativePath));
+};
 export const listVaultFiles = async (vaultPath) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
     return walkVaultFiles(absoluteVaultPath);

package/dist/infrastructure/index-state.js

@@ -0,0 +1,58 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+const indexStateFileName = 'index-state.json';
+const toIndexStatePath = (vaultPath) => join(vaultPath, '.brainlink', indexStateFileName);
+export const readIndexState = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toIndexStatePath(vaultPath), 'utf8'));
+        if (parsed.version !== 1 || !Array.isArray(parsed.files)) {
+            return null;
+        }
+        const files = parsed.files.flatMap((entry) => {
+            if (!entry || typeof entry !== 'object') {
+                return [];
+            }
+            const row = entry;
+            if (typeof row.path !== 'string' || typeof row.mtimeMs !== 'number' || typeof row.size !== 'number') {
+                return [];
+            }
+            return [
+                {
+                    path: row.path,
+                    mtimeMs: row.mtimeMs,
+                    size: row.size
+                }
+            ];
+        });
+        return {
+            version: 1,
+            updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
+            chunkSize: typeof parsed.chunkSize === 'number' ? parsed.chunkSize : 1200,
+            embeddingProvider: typeof parsed.embeddingProvider === 'string' ? parsed.embeddingProvider : 'none',
+            graphLinkModelVersion: typeof parsed.graphLinkModelVersion === 'number' ? parsed.graphLinkModelVersion : 1,
+            searchPackRowChunkSize: typeof parsed.searchPackRowChunkSize === 'number' ? parsed.searchPackRowChunkSize : 5_000,
+            searchPackCompressionLevel: typeof parsed.searchPackCompressionLevel === 'number' ? parsed.searchPackCompressionLevel : 5,
+            searchPackUseDictionary: typeof parsed.searchPackUseDictionary === 'boolean' ? parsed.searchPackUseDictionary : true,
+            files,
+            pendingPackChanges: typeof parsed.pendingPackChanges === 'number' && parsed.pendingPackChanges >= 0 ? parsed.pendingPackChanges : 0
+        };
+    }
+    catch {
+        return null;
+    }
+};
+export const writeIndexState = async (vaultPath, state) => {
+    const payload = {
+        version: 1,
+        updatedAt: new Date().toISOString(),
+        chunkSize: state.chunkSize,
+        embeddingProvider: state.embeddingProvider,
+        graphLinkModelVersion: state.graphLinkModelVersion,
+        searchPackRowChunkSize: state.searchPackRowChunkSize,
+        searchPackCompressionLevel: state.searchPackCompressionLevel,
+        searchPackUseDictionary: state.searchPackUseDictionary,
+        files: [...state.files].sort((left, right) => left.path.localeCompare(right.path)),
+        pendingPackChanges: Math.max(0, Math.floor(state.pendingPackChanges))
+    };
+    await writeFile(toIndexStatePath(vaultPath), `${JSON.stringify(payload)}\n`, 'utf8');
+};

package/dist/infrastructure/private-pack-codec.js

@@ -1,13 +1,25 @@
 import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
-import { brotliCompressSync, brotliDecompressSync } from 'node:zlib';
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { getBrainlinkHomePath } from './paths.js';
 const magic = Buffer.from('BLPK2', 'ascii');
-const version = 1;
+const legacyVersion = 1;
+const currentVersion = 2;
 const nonceLength = 12;
 const authTagLength = 16;
 const algorithm = 'aes-256-gcm';
+const compressionLevelMask = 0x0f;
+const compressionDictionaryMask = 0x10;
+const defaultCompressionLevel = 5;
+const builtinDictionary = Buffer.from([
+    '"documentId","agentId","title","path","chunkId","chunkOrdinal","content","tags"',
+    '"searchMode","textScore","semanticScore","weight","priority","shared"',
+    'agents/shared memory-hub architecture context index search graph markdown tags links',
+    '#memory #architecture #context #graph #search #index [[Memory Hub]] [[Architecture]]',
+    'The quick brown fox jumps over the lazy dog. Brainlink context package metadata.',
+    'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-:/.#[]{}(), '
+].join('\n'), 'utf8');
 const keyFilePath = (vaultPath) => {
     const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
     return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
@@ -40,34 +52,83 @@ const parseHeader = (payload) => {
         throw new Error('Invalid private pack payload: too short.');
     }
     const payloadMagic = payload.subarray(0, magic.length);
-    const payloadVersion = payload[magic.length];
-    if (!payloadMagic.equals(magic) || payloadVersion !== version) {
+    const payloadVersion = payload[magic.length] ?? 0;
+    if (!payloadMagic.equals(magic) || (payloadVersion !== legacyVersion && payloadVersion !== currentVersion)) {
         throw new Error('Invalid private pack payload: unsupported format.');
     }
-    const nonceStart = magic.length + 1;
+    const hasCompressionSettings = payloadVersion >= 2;
+    const settingsByte = hasCompressionSettings ? payload[magic.length + 1] ?? 0 : null;
+    const nonceStart = magic.length + 1 + (hasCompressionSettings ? 1 : 0);
     const authTagStart = nonceStart + nonceLength;
     const dataStart = authTagStart + authTagLength;
     return {
+        compression: settingsByte != null
+            ? {
+                compressionLevel: settingsByte & compressionLevelMask,
+                useDictionary: (settingsByte & compressionDictionaryMask) !== 0
+            }
+            : {
+                compressionLevel: defaultCompressionLevel,
+                useDictionary: false
+            },
         nonce: payload.subarray(nonceStart, authTagStart),
         authTag: payload.subarray(authTagStart, dataStart),
         ciphertext: payload.subarray(dataStart)
     };
 };
-export const encodePrivatePack = async (vaultPath, content) => {
+const toCompressionLevel = (value) => {
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return defaultCompressionLevel;
+    }
+    const normalized = Math.round(value);
+    if (normalized < 0) {
+        return 0;
+    }
+    if (normalized > 11) {
+        return 11;
+    }
+    return normalized;
+};
+const encodeCompressionSettings = (settings) => (settings.compressionLevel & compressionLevelMask) | (settings.useDictionary ? compressionDictionaryMask : 0);
+const brotliEncode = (content, settings) => {
+    const options = {
+        params: {
+            [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
+            [zlibConstants.BROTLI_PARAM_QUALITY]: settings.compressionLevel
+        }
+    };
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliCompressSync(content, options);
+};
+const brotliDecode = (content, settings) => {
+    const options = {};
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliDecompressSync(content, options);
+};
+export const encodePrivatePack = async (vaultPath, content, settings) => {
     const key = await readOrCreateKey(vaultPath);
     const nonce = randomBytes(nonceLength);
-    const compressed = brotliCompressSync(content);
+    const normalizedSettings = {
+        compressionLevel: toCompressionLevel(settings?.compressionLevel),
+        useDictionary: settings?.useDictionary ?? true
+    };
+    const compressed = brotliEncode(content, normalizedSettings);
     const cipher = createCipheriv(algorithm, key, nonce);
     const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
     const authTag = cipher.getAuthTag();
-    return Buffer.concat([magic, Buffer.from([version]), nonce, authTag, ciphertext]);
+    const settingsByte = Buffer.from([encodeCompressionSettings(normalizedSettings)]);
+    return Buffer.concat([magic, Buffer.from([currentVersion]), settingsByte, nonce, authTag, ciphertext]);
 };
 export const decodePrivatePack = async (vaultPath, payload) => {
     const key = await readOrCreateKey(vaultPath);
-    const { nonce, authTag, ciphertext } = parseHeader(payload);
+    const { nonce, authTag, ciphertext, compression } = parseHeader(payload);
     const decipher = createDecipheriv(algorithm, key, nonce);
     decipher.setAuthTag(authTag);
     const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
-    return brotliDecompressSync(compressed);
+    return brotliDecode(compressed, compression);
 };
 export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);