@andespindola/brainlink 0.1.0-beta.13 → 0.1.0-beta.130

package/dist/infrastructure/search-packs.js

@@ -1,11 +1,19 @@
 import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { middleOutIndices } from '../domain/middle-out.js';
 import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
-const rowChunkSize = 5_000;
+const defaultBuildOptions = {
+    rowChunkSize: 5_000,
+    compressionLevel: 5,
+    useDictionary: true
+};
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const bloomBytes = 256;
+const bloomBitSize = bloomBytes * 8;
+const bloomSeeds = [0x9e3779b1, 0x85ebca6b, 0xc2b2ae35];
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
 const parseRowsFromPack = async (vaultPath, content) => {
@@ -15,7 +23,29 @@ const parseRowsFromPack = async (vaultPath, content) => {
         .split('\n')
         .map((line) => line.trim())
         .filter((line) => line.length > 0)
-        .map((line) => JSON.parse(line));
+        .map((line) => JSON.parse(line))
+        .flatMap((row) => {
+        if (typeof row.documentId !== 'string' ||
+            typeof row.agentId !== 'string' ||
+            typeof row.title !== 'string' ||
+            typeof row.path !== 'string' ||
+            typeof row.chunkId !== 'string' ||
+            typeof row.content !== 'string') {
+            return [];
+        }
+        return [
+            {
+                documentId: row.documentId,
+                agentId: row.agentId,
+                title: row.title,
+                path: row.path,
+                chunkId: row.chunkId,
+                chunkOrdinal: typeof row.chunkOrdinal === 'number' ? row.chunkOrdinal : 0,
+                content: row.content,
+                tags: Array.isArray(row.tags) ? row.tags.filter((item) => typeof item === 'string') : []
+            }
+        ];
+    });
 };
 const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
     documentId: document.document.id,
@@ -23,12 +53,121 @@ const toRows = (documents) => documents.flatMap((document) => document.chunks.ma
     title: document.document.title,
     path: document.document.path,
     chunkId: chunk.id,
+    chunkOrdinal: chunk.ordinal,
     content: chunk.content,
     tags: document.document.tags
 })));
 const writeManifest = async (vaultPath, manifest) => {
     await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
 };
+const readManifest = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toManifestPath(vaultPath), 'utf8'));
+        if (parsed.version === 2 && parsed.format === 'private-v2') {
+            return {
+                version: 2,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : 0,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2'
+            };
+        }
+        if (parsed.version === 3 && parsed.format === 'private-v2') {
+            const packIndex = Array.isArray(parsed.packIndex)
+                ? parsed.packIndex.flatMap((entry) => {
+                    if (!entry || typeof entry !== 'object') {
+                        return [];
+                    }
+                    const candidate = entry;
+                    if (typeof candidate.fileName !== 'string' || typeof candidate.tokenBloomB64 !== 'string') {
+                        return [];
+                    }
+                    return [
+                        {
+                            fileName: candidate.fileName,
+                            recordCount: typeof candidate.recordCount === 'number' ? candidate.recordCount : 0,
+                            agents: Array.isArray(candidate.agents) ? candidate.agents.filter((item) => typeof item === 'string') : [],
+                            tokenBloomB64: candidate.tokenBloomB64
+                        }
+                    ];
+                })
+                : [];
+            return {
+                version: 3,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : packIndex.length,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2',
+                packIndex,
+                ...(parsed.packConfig && typeof parsed.packConfig === 'object'
+                    ? {
+                        packConfig: {
+                            rowChunkSize: typeof parsed.packConfig.rowChunkSize === 'number'
+                                ? parsed.packConfig.rowChunkSize
+                                : defaultBuildOptions.rowChunkSize,
+                            compressionLevel: typeof parsed.packConfig.compressionLevel === 'number'
+                                ? parsed.packConfig.compressionLevel
+                                : defaultBuildOptions.compressionLevel,
+                            useDictionary: typeof parsed.packConfig.useDictionary === 'boolean'
+                                ? parsed.packConfig.useDictionary
+                                : defaultBuildOptions.useDictionary
+                        }
+                    }
+                    : {}),
+                ...(parsed.compression &&
+                    typeof parsed.compression === 'object' &&
+                    typeof parsed.compression.inputBytes === 'number' &&
+                    typeof parsed.compression.outputBytes === 'number' &&
+                    typeof parsed.compression.ratio === 'number' &&
+                    typeof parsed.compression.savedBytes === 'number'
+                    ? {
+                        compression: {
+                            inputBytes: parsed.compression.inputBytes,
+                            outputBytes: parsed.compression.outputBytes,
+                            ratio: parsed.compression.ratio,
+                            savedBytes: parsed.compression.savedBytes
+                        }
+                    }
+                    : {})
+            };
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+};
+export const ensureSearchPackManifest = async (vaultPath) => {
+    const manifest = await readManifest(vaultPath);
+    if (manifest) {
+        return {
+            repaired: false,
+            source: 'not-needed',
+            packCount: manifest.packCount
+        };
+    }
+    const files = await sortedPackFiles(vaultPath);
+    const packFiles = files.filter((file) => file.endsWith('.blpk'));
+    if (packFiles.length === 0) {
+        return {
+            repaired: false,
+            source: 'no-packs',
+            packCount: 0
+        };
+    }
+    await writeManifest(vaultPath, {
+        version: 2,
+        createdAt: new Date().toISOString(),
+        packCount: packFiles.length,
+        recordCount: 0,
+        format: 'private-v2'
+    });
+    return {
+        repaired: true,
+        source: 'existing-packs',
+        packCount: packFiles.length
+    };
+};
 const chunkRows = (rows, size) => {
     const chunks = [];
     for (let index = 0; index < rows.length; index += size) {
@@ -57,6 +196,51 @@ const countOccurrences = (text, token) => {
     }
     return hits;
 };
+const hashToken = (token, seed) => {
+    let hash = seed >>> 0;
+    for (let index = 0; index < token.length; index += 1) {
+        hash ^= token.charCodeAt(index);
+        hash = Math.imul(hash, 16777619) >>> 0;
+    }
+    return hash >>> 0;
+};
+const createBloom = () => new Uint8Array(bloomBytes);
+const bloomAdd = (bloom, token) => {
+    bloomSeeds.forEach((seed) => {
+        const bit = hashToken(token, seed) % bloomBitSize;
+        bloom[Math.floor(bit / 8)] |= 1 << (bit % 8);
+    });
+};
+const bloomMayContain = (bloom, token) => bloomSeeds.every((seed) => {
+    const bit = hashToken(token, seed) % bloomBitSize;
+    return (bloom[Math.floor(bit / 8)] & (1 << (bit % 8))) !== 0;
+});
+const bloomFromRows = (rows) => {
+    const bloom = createBloom();
+    rows.forEach((row) => {
+        tokenize([row.title, row.path, row.tags.join(' '), row.content].join(' ')).forEach((token) => bloomAdd(bloom, token));
+    });
+    return bloom;
+};
+const bloomToBase64 = (bloom) => Buffer.from(bloom).toString('base64url');
+const bloomFromBase64 = (value) => {
+    try {
+        const decoded = Buffer.from(value, 'base64url');
+        if (decoded.byteLength === bloomBytes) {
+            return {
+                bloom: new Uint8Array(decoded),
+                valid: true
+            };
+        }
+    }
+    catch {
+        // fallback below
+    }
+    return {
+        bloom: createBloom(),
+        valid: false
+    };
+};
 const computeTextScore = (row, tokens) => {
     if (tokens.length === 0) {
         return 0;
@@ -79,6 +263,7 @@ const toSearchResult = (row, score) => ({
     title: row.title,
     path: row.path,
     chunkId: row.chunkId,
+    chunkOrdinal: row.chunkOrdinal,
     content: row.content,
     score,
     textScore: score,
@@ -100,47 +285,158 @@ const sortedPackFiles = async (vaultPath) => {
         throw error;
     }
 };
-export const buildSearchPacks = async (vaultPath, documents) => {
+const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting, options) => {
+    const startedAt = process.hrtime.bigint();
     const directory = toPackDirectory(vaultPath);
-    const rows = toRows(documents);
     await mkdir(directory, { recursive: true });
-    const current = await readdir(directory);
-    await Promise.all(current
-        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
-        .map((name) => rm(join(directory, name), { force: true })));
-    const chunks = chunkRows(rows, rowChunkSize);
-    await Promise.all(chunks.map(async (chunk, index) => {
+    if (clearExisting) {
+        const current = await readdir(directory);
+        await Promise.all(current
+            .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
+            .map((name) => rm(join(directory, name), { force: true })));
+    }
+    const chunks = chunkRows(rows, options.rowChunkSize);
+    const packIndex = [];
+    let inputBytes = 0;
+    let outputBytes = 0;
+    for (let index = 0; index < chunks.length; index += 1) {
+        const chunk = chunks[index];
         const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
         const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
-        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'));
+        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'), {
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        });
+        const tokenBloomB64 = bloomToBase64(bloomFromRows(chunk));
         await writeFile(join(directory, fileName), compressed);
-    }));
+        inputBytes += Buffer.byteLength(serialized, 'utf8');
+        outputBytes += compressed.byteLength;
+        packIndex.push({
+            fileName,
+            recordCount: chunk.length,
+            agents: Array.from(new Set(chunk.map((row) => row.agentId))).sort((left, right) => left.localeCompare(right)),
+            tokenBloomB64
+        });
+    }
     await writeManifest(vaultPath, {
-        version: 2,
+        version: 3,
         createdAt: new Date().toISOString(),
         packCount: chunks.length,
         recordCount: rows.length,
-        format: 'private-v2'
+        format: 'private-v2',
+        packIndex,
+        packConfig: {
+            rowChunkSize: options.rowChunkSize,
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        },
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / Math.max(inputBytes, 1),
+            savedBytes: Math.max(inputBytes - outputBytes, 0)
+        }
     });
+    const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const safeInput = Math.max(inputBytes, 1);
+    const savedBytes = Math.max(inputBytes - outputBytes, 0);
     return {
         packCount: chunks.length,
-        recordCount: rows.length
+        recordCount: rows.length,
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / safeInput,
+            savedBytes
+        },
+        durationMs
     };
 };
+const selectCandidatePackFiles = async (vaultPath, tokens, agentId) => {
+    const allFiles = await sortedPackFiles(vaultPath);
+    if (allFiles.length === 0) {
+        return [];
+    }
+    const manifest = await readManifest(vaultPath);
+    if (!manifest || manifest.version !== 3 || !Array.isArray(manifest.packIndex)) {
+        return allFiles;
+    }
+    const normalizedAgent = agentId?.trim();
+    const byAgent = manifest.packIndex.filter((entry) => normalizedAgent ? entry.agents.includes(normalizedAgent) : true);
+    if (tokens.length === 0) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    let hasInvalidBloomIndex = false;
+    const byToken = byAgent.filter((entry) => {
+        const decoded = bloomFromBase64(entry.tokenBloomB64);
+        if (!decoded.valid) {
+            hasInvalidBloomIndex = true;
+            return true;
+        }
+        return tokens.some((token) => bloomMayContain(decoded.bloom, token));
+    });
+    // Lossless guarantee: if compressed metadata is partially invalid, do not prune packs.
+    if (hasInvalidBloomIndex) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    if (byToken.length > 0) {
+        return byToken.map((entry) => entry.fileName);
+    }
+    return byAgent.length > 0 ? byAgent.map((entry) => entry.fileName) : allFiles;
+};
+export const buildSearchPacks = async (vaultPath, documents, options) => {
+    const resolvedOptions = {
+        rowChunkSize: options?.rowChunkSize ?? defaultBuildOptions.rowChunkSize,
+        compressionLevel: options?.compressionLevel ?? defaultBuildOptions.compressionLevel,
+        useDictionary: options?.useDictionary ?? defaultBuildOptions.useDictionary
+    };
+    return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true, resolvedOptions);
+};
+export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
+    const files = await sortedPackFiles(vaultPath);
+    if (files.some((file) => file.endsWith('.blpk'))) {
+        return { imported: false };
+    }
+    const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
+    if (legacyPackFiles.length > 0) {
+        const rows = [];
+        for (const file of legacyPackFiles) {
+            const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+            rows.push(...parsed);
+        }
+        const report = await writeRowsAsPrivatePacks(vaultPath, rows, true, defaultBuildOptions);
+        return {
+            imported: true,
+            source: 'legacy-packs',
+            ...report
+        };
+    }
+    return { imported: false };
+};
+export const toSearchPackBuildOptions = (config) => ({
+    rowChunkSize: config.searchPack.rowChunkSize,
+    compressionLevel: config.searchPack.compressionLevel,
+    useDictionary: config.searchPack.useDictionary
+});
 export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     const normalizedAgent = agentId?.trim();
     const tokens = tokenize(query);
     if (limit <= 0 || tokens.length === 0) {
         return [];
     }
-    const files = await sortedPackFiles(vaultPath);
+    const files = await selectCandidatePackFiles(vaultPath, tokens, normalizedAgent);
     if (files.length === 0) {
         return [];
     }
     const scored = [];
     for (const file of files) {
         const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
-        rows.forEach((row) => {
+        const traversal = middleOutIndices(rows.length, Math.floor(rows.length / 2));
+        traversal.forEach((rowIndex) => {
+            const row = rows[rowIndex];
+            if (!row) {
+                return;
+            }
             if (normalizedAgent && row.agentId !== normalizedAgent) {
                 return;
             }

package/dist/infrastructure/volatile-memory.js

@@ -0,0 +1,100 @@
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+export const volatileMemoryStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'volatile.json');
+const emptyStore = () => ({
+    version: 1,
+    entries: []
+});
+const normalizeToken = (value) => value
+    .normalize('NFKD')
+    .replace(/\p{Diacritic}/gu, '')
+    .toLowerCase();
+const tokens = (value) => value
+    .match(/[\p{L}\p{N}_-]+/gu)
+    ?.map(normalizeToken)
+    .filter((token) => token.length > 1) ?? [];
+const readStore = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(volatileMemoryStoragePath(vaultPath), 'utf8'));
+        return {
+            version: 1,
+            entries: Array.isArray(parsed.entries) ? parsed.entries.filter(isEntry) : []
+        };
+    }
+    catch {
+        return emptyStore();
+    }
+};
+const writeStore = async (vaultPath, store) => {
+    const target = volatileMemoryStoragePath(vaultPath);
+    const temp = `${target}.tmp`;
+    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+    await writeFile(temp, `${JSON.stringify(store, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+    await rename(temp, target);
+};
+const isEntry = (value) => {
+    const record = value;
+    return Boolean(record &&
+        typeof record.id === 'string' &&
+        typeof record.agentId === 'string' &&
+        typeof record.content === 'string' &&
+        typeof record.createdAt === 'string' &&
+        typeof record.expiresAt === 'string' &&
+        Array.isArray(record.tags));
+};
+const activeEntries = (entries, now = Date.now()) => entries.filter((entry) => Date.parse(entry.expiresAt) > now);
+const scoreEntry = (entry, query) => {
+    const queryTokens = tokens(query);
+    if (queryTokens.length === 0) {
+        return 1;
+    }
+    const haystack = normalizeToken([entry.content, entry.tags.join(' ')].join(' '));
+    return queryTokens.reduce((score, token) => score + (haystack.includes(token) ? 1 : 0), 0);
+};
+export const addVolatileMemory = async (vaultPath, content, agentId = 'shared', ttlMinutes = 240, tags = []) => {
+    const now = new Date();
+    const entry = {
+        id: `volatile-${now.getTime()}-${Math.random().toString(36).slice(2, 10)}`,
+        agentId,
+        content,
+        createdAt: now.toISOString(),
+        expiresAt: new Date(now.getTime() + Math.max(1, ttlMinutes) * 60_000).toISOString(),
+        source: 'agent',
+        tags
+    };
+    const store = await readStore(vaultPath);
+    await writeStore(vaultPath, {
+        version: 1,
+        entries: [...activeEntries(store.entries), entry]
+    });
+    return entry;
+};
+export const searchVolatileMemory = async (vaultPath, query, limit, agentId, mode = 'hybrid') => {
+    const store = await readStore(vaultPath);
+    const entries = activeEntries(store.entries)
+        .filter((entry) => (agentId ? entry.agentId === agentId : true))
+        .map((entry) => ({ entry, score: scoreEntry(entry, query) }))
+        .filter(({ score }) => score > 0)
+        .sort((left, right) => right.score - left.score || right.entry.createdAt.localeCompare(left.entry.createdAt))
+        .slice(0, Math.max(0, limit));
+    if (entries.length !== store.entries.length) {
+        await writeStore(vaultPath, { version: 1, entries: activeEntries(store.entries) });
+    }
+    return entries.map(({ entry, score }) => ({
+        title: 'Volatile Memory',
+        path: `volatile://${entry.id}`,
+        content: entry.content,
+        score,
+        searchMode: mode,
+        tags: ['volatile', ...entry.tags],
+        volatile: true,
+        expiresAt: entry.expiresAt
+    }));
+};
+export const clearVolatileMemory = async (vaultPath, agentId) => {
+    const store = await readStore(vaultPath);
+    const active = activeEntries(store.entries);
+    const kept = agentId ? active.filter((entry) => entry.agentId !== agentId) : [];
+    await writeStore(vaultPath, { version: 1, entries: kept });
+    return active.length - kept.length;
+};

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, contextInputSchema, contextTool, graphInputSchema, graphTool, indexInputSchema, indexTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, statsInputSchema, statsTool, syncInputSchema, syncTool, validateInputSchema, validateTool } from './tools.js';
+import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, volatileAddInputSchema, volatileAddTool, volatileClearInputSchema, volatileClearTool, dedupeInputSchema, dedupeResolveInputSchema, dedupeResolveTool, dedupeTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, contextInputSchema, contextTool, graphInputSchema, graphTool, indexInputSchema, indexTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, statsInputSchema, statsTool, syncInputSchema, syncTool, validateInputSchema, validateTool } from './tools.js';
 const readPackageVersion = () => {
     const packagePath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
     const metadata = JSON.parse(readFileSync(packagePath, 'utf8'));
@@ -40,11 +40,31 @@ export const createBrainlinkMcpServer = () => {
         description: 'Search indexed Brainlink notes with FTS, semantic or hybrid retrieval.',
         inputSchema: searchInputSchema
     }, searchTool);
+    server.registerTool('brainlink_dedupe', {
+        title: 'Detect Duplicate Notes',
+        description: 'Detect possible duplicate notes using exact content hash and semantic similarity scoring.',
+        inputSchema: dedupeInputSchema
+    }, dedupeTool);
+    server.registerTool('brainlink_resolve_duplicate', {
+        title: 'Resolve Duplicate Notes',
+        description: 'Resolve a duplicate pair with merge, link or ignore. Non-merge actions still create low-priority related edges.',
+        inputSchema: dedupeResolveInputSchema
+    }, dedupeResolveTool);
     server.registerTool('brainlink_add_note', {
         title: 'Add Brainlink Note',
         description: 'Write durable Markdown memory, then reindex the vault. Include explicit [[wiki links]] for connected graph memory. Add priority markers near links, such as priority: high, #important or #critical, when a relationship should be weighted higher.',
         inputSchema: addNoteInputSchema
     }, addNoteTool);
+    server.registerTool('brainlink_volatile_add', {
+        title: 'Add Volatile Brainlink Memory',
+        description: 'Write temporary agent-decided memory with TTL. Use for transient task state without polluting durable Markdown memory.',
+        inputSchema: volatileAddInputSchema
+    }, volatileAddTool);
+    server.registerTool('brainlink_volatile_clear', {
+        title: 'Clear Volatile Brainlink Memory',
+        description: 'Clear active volatile memory for the current vault/agent namespace.',
+        inputSchema: volatileClearInputSchema
+    }, volatileClearTool);
     server.registerTool('brainlink_add_file', {
         title: 'Ingest Markdown File',
         description: 'Read a local markdown/text file and ingest it as a Brainlink note. Reindex defaults to true.',