@andespindola/brainlink 0.1.0-beta.12 → 0.1.0-beta.121

package/dist/infrastructure/search-packs.js

@@ -1,11 +1,19 @@
 import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { middleOutIndices } from '../domain/middle-out.js';
 import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
-const rowChunkSize = 5_000;
+const defaultBuildOptions = {
+    rowChunkSize: 5_000,
+    compressionLevel: 5,
+    useDictionary: true
+};
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const bloomBytes = 256;
+const bloomBitSize = bloomBytes * 8;
+const bloomSeeds = [0x9e3779b1, 0x85ebca6b, 0xc2b2ae35];
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
 const parseRowsFromPack = async (vaultPath, content) => {
@@ -15,7 +23,29 @@ const parseRowsFromPack = async (vaultPath, content) => {
         .split('\n')
         .map((line) => line.trim())
         .filter((line) => line.length > 0)
-        .map((line) => JSON.parse(line));
+        .map((line) => JSON.parse(line))
+        .flatMap((row) => {
+        if (typeof row.documentId !== 'string' ||
+            typeof row.agentId !== 'string' ||
+            typeof row.title !== 'string' ||
+            typeof row.path !== 'string' ||
+            typeof row.chunkId !== 'string' ||
+            typeof row.content !== 'string') {
+            return [];
+        }
+        return [
+            {
+                documentId: row.documentId,
+                agentId: row.agentId,
+                title: row.title,
+                path: row.path,
+                chunkId: row.chunkId,
+                chunkOrdinal: typeof row.chunkOrdinal === 'number' ? row.chunkOrdinal : 0,
+                content: row.content,
+                tags: Array.isArray(row.tags) ? row.tags.filter((item) => typeof item === 'string') : []
+            }
+        ];
+    });
 };
 const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
     documentId: document.document.id,
@@ -23,12 +53,121 @@ const toRows = (documents) => documents.flatMap((document) => document.chunks.ma
     title: document.document.title,
     path: document.document.path,
     chunkId: chunk.id,
+    chunkOrdinal: chunk.ordinal,
     content: chunk.content,
     tags: document.document.tags
 })));
 const writeManifest = async (vaultPath, manifest) => {
     await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
 };
+const readManifest = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toManifestPath(vaultPath), 'utf8'));
+        if (parsed.version === 2 && parsed.format === 'private-v2') {
+            return {
+                version: 2,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : 0,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2'
+            };
+        }
+        if (parsed.version === 3 && parsed.format === 'private-v2') {
+            const packIndex = Array.isArray(parsed.packIndex)
+                ? parsed.packIndex.flatMap((entry) => {
+                    if (!entry || typeof entry !== 'object') {
+                        return [];
+                    }
+                    const candidate = entry;
+                    if (typeof candidate.fileName !== 'string' || typeof candidate.tokenBloomB64 !== 'string') {
+                        return [];
+                    }
+                    return [
+                        {
+                            fileName: candidate.fileName,
+                            recordCount: typeof candidate.recordCount === 'number' ? candidate.recordCount : 0,
+                            agents: Array.isArray(candidate.agents) ? candidate.agents.filter((item) => typeof item === 'string') : [],
+                            tokenBloomB64: candidate.tokenBloomB64
+                        }
+                    ];
+                })
+                : [];
+            return {
+                version: 3,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : packIndex.length,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2',
+                packIndex,
+                ...(parsed.packConfig && typeof parsed.packConfig === 'object'
+                    ? {
+                        packConfig: {
+                            rowChunkSize: typeof parsed.packConfig.rowChunkSize === 'number'
+                                ? parsed.packConfig.rowChunkSize
+                                : defaultBuildOptions.rowChunkSize,
+                            compressionLevel: typeof parsed.packConfig.compressionLevel === 'number'
+                                ? parsed.packConfig.compressionLevel
+                                : defaultBuildOptions.compressionLevel,
+                            useDictionary: typeof parsed.packConfig.useDictionary === 'boolean'
+                                ? parsed.packConfig.useDictionary
+                                : defaultBuildOptions.useDictionary
+                        }
+                    }
+                    : {}),
+                ...(parsed.compression &&
+                    typeof parsed.compression === 'object' &&
+                    typeof parsed.compression.inputBytes === 'number' &&
+                    typeof parsed.compression.outputBytes === 'number' &&
+                    typeof parsed.compression.ratio === 'number' &&
+                    typeof parsed.compression.savedBytes === 'number'
+                    ? {
+                        compression: {
+                            inputBytes: parsed.compression.inputBytes,
+                            outputBytes: parsed.compression.outputBytes,
+                            ratio: parsed.compression.ratio,
+                            savedBytes: parsed.compression.savedBytes
+                        }
+                    }
+                    : {})
+            };
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+};
+export const ensureSearchPackManifest = async (vaultPath) => {
+    const manifest = await readManifest(vaultPath);
+    if (manifest) {
+        return {
+            repaired: false,
+            source: 'not-needed',
+            packCount: manifest.packCount
+        };
+    }
+    const files = await sortedPackFiles(vaultPath);
+    const packFiles = files.filter((file) => file.endsWith('.blpk'));
+    if (packFiles.length === 0) {
+        return {
+            repaired: false,
+            source: 'no-packs',
+            packCount: 0
+        };
+    }
+    await writeManifest(vaultPath, {
+        version: 2,
+        createdAt: new Date().toISOString(),
+        packCount: packFiles.length,
+        recordCount: 0,
+        format: 'private-v2'
+    });
+    return {
+        repaired: true,
+        source: 'existing-packs',
+        packCount: packFiles.length
+    };
+};
 const chunkRows = (rows, size) => {
     const chunks = [];
     for (let index = 0; index < rows.length; index += size) {
@@ -57,6 +196,51 @@ const countOccurrences = (text, token) => {
     }
     return hits;
 };
+const hashToken = (token, seed) => {
+    let hash = seed >>> 0;
+    for (let index = 0; index < token.length; index += 1) {
+        hash ^= token.charCodeAt(index);
+        hash = Math.imul(hash, 16777619) >>> 0;
+    }
+    return hash >>> 0;
+};
+const createBloom = () => new Uint8Array(bloomBytes);
+const bloomAdd = (bloom, token) => {
+    bloomSeeds.forEach((seed) => {
+        const bit = hashToken(token, seed) % bloomBitSize;
+        bloom[Math.floor(bit / 8)] |= 1 << (bit % 8);
+    });
+};
+const bloomMayContain = (bloom, token) => bloomSeeds.every((seed) => {
+    const bit = hashToken(token, seed) % bloomBitSize;
+    return (bloom[Math.floor(bit / 8)] & (1 << (bit % 8))) !== 0;
+});
+const bloomFromRows = (rows) => {
+    const bloom = createBloom();
+    rows.forEach((row) => {
+        tokenize([row.title, row.path, row.tags.join(' '), row.content].join(' ')).forEach((token) => bloomAdd(bloom, token));
+    });
+    return bloom;
+};
+const bloomToBase64 = (bloom) => Buffer.from(bloom).toString('base64url');
+const bloomFromBase64 = (value) => {
+    try {
+        const decoded = Buffer.from(value, 'base64url');
+        if (decoded.byteLength === bloomBytes) {
+            return {
+                bloom: new Uint8Array(decoded),
+                valid: true
+            };
+        }
+    }
+    catch {
+        // fallback below
+    }
+    return {
+        bloom: createBloom(),
+        valid: false
+    };
+};
 const computeTextScore = (row, tokens) => {
     if (tokens.length === 0) {
         return 0;
@@ -79,6 +263,7 @@ const toSearchResult = (row, score) => ({
     title: row.title,
     path: row.path,
     chunkId: row.chunkId,
+    chunkOrdinal: row.chunkOrdinal,
     content: row.content,
     score,
     textScore: score,
@@ -100,47 +285,158 @@ const sortedPackFiles = async (vaultPath) => {
         throw error;
     }
 };
-export const buildSearchPacks = async (vaultPath, documents) => {
+const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting, options) => {
+    const startedAt = process.hrtime.bigint();
     const directory = toPackDirectory(vaultPath);
-    const rows = toRows(documents);
     await mkdir(directory, { recursive: true });
-    const current = await readdir(directory);
-    await Promise.all(current
-        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
-        .map((name) => rm(join(directory, name), { force: true })));
-    const chunks = chunkRows(rows, rowChunkSize);
-    await Promise.all(chunks.map(async (chunk, index) => {
+    if (clearExisting) {
+        const current = await readdir(directory);
+        await Promise.all(current
+            .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
+            .map((name) => rm(join(directory, name), { force: true })));
+    }
+    const chunks = chunkRows(rows, options.rowChunkSize);
+    const packIndex = [];
+    let inputBytes = 0;
+    let outputBytes = 0;
+    for (let index = 0; index < chunks.length; index += 1) {
+        const chunk = chunks[index];
         const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
         const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
-        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'));
+        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'), {
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        });
+        const tokenBloomB64 = bloomToBase64(bloomFromRows(chunk));
         await writeFile(join(directory, fileName), compressed);
-    }));
+        inputBytes += Buffer.byteLength(serialized, 'utf8');
+        outputBytes += compressed.byteLength;
+        packIndex.push({
+            fileName,
+            recordCount: chunk.length,
+            agents: Array.from(new Set(chunk.map((row) => row.agentId))).sort((left, right) => left.localeCompare(right)),
+            tokenBloomB64
+        });
+    }
     await writeManifest(vaultPath, {
-        version: 2,
+        version: 3,
         createdAt: new Date().toISOString(),
         packCount: chunks.length,
         recordCount: rows.length,
-        format: 'private-v2'
+        format: 'private-v2',
+        packIndex,
+        packConfig: {
+            rowChunkSize: options.rowChunkSize,
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        },
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / Math.max(inputBytes, 1),
+            savedBytes: Math.max(inputBytes - outputBytes, 0)
+        }
     });
+    const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const safeInput = Math.max(inputBytes, 1);
+    const savedBytes = Math.max(inputBytes - outputBytes, 0);
     return {
         packCount: chunks.length,
-        recordCount: rows.length
+        recordCount: rows.length,
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / safeInput,
+            savedBytes
+        },
+        durationMs
     };
 };
+const selectCandidatePackFiles = async (vaultPath, tokens, agentId) => {
+    const allFiles = await sortedPackFiles(vaultPath);
+    if (allFiles.length === 0) {
+        return [];
+    }
+    const manifest = await readManifest(vaultPath);
+    if (!manifest || manifest.version !== 3 || !Array.isArray(manifest.packIndex)) {
+        return allFiles;
+    }
+    const normalizedAgent = agentId?.trim();
+    const byAgent = manifest.packIndex.filter((entry) => normalizedAgent ? entry.agents.includes(normalizedAgent) : true);
+    if (tokens.length === 0) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    let hasInvalidBloomIndex = false;
+    const byToken = byAgent.filter((entry) => {
+        const decoded = bloomFromBase64(entry.tokenBloomB64);
+        if (!decoded.valid) {
+            hasInvalidBloomIndex = true;
+            return true;
+        }
+        return tokens.some((token) => bloomMayContain(decoded.bloom, token));
+    });
+    // Lossless guarantee: if compressed metadata is partially invalid, do not prune packs.
+    if (hasInvalidBloomIndex) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    if (byToken.length > 0) {
+        return byToken.map((entry) => entry.fileName);
+    }
+    return byAgent.length > 0 ? byAgent.map((entry) => entry.fileName) : allFiles;
+};
+export const buildSearchPacks = async (vaultPath, documents, options) => {
+    const resolvedOptions = {
+        rowChunkSize: options?.rowChunkSize ?? defaultBuildOptions.rowChunkSize,
+        compressionLevel: options?.compressionLevel ?? defaultBuildOptions.compressionLevel,
+        useDictionary: options?.useDictionary ?? defaultBuildOptions.useDictionary
+    };
+    return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true, resolvedOptions);
+};
+export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
+    const files = await sortedPackFiles(vaultPath);
+    if (files.some((file) => file.endsWith('.blpk'))) {
+        return { imported: false };
+    }
+    const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
+    if (legacyPackFiles.length > 0) {
+        const rows = [];
+        for (const file of legacyPackFiles) {
+            const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+            rows.push(...parsed);
+        }
+        const report = await writeRowsAsPrivatePacks(vaultPath, rows, true, defaultBuildOptions);
+        return {
+            imported: true,
+            source: 'legacy-packs',
+            ...report
+        };
+    }
+    return { imported: false };
+};
+export const toSearchPackBuildOptions = (config) => ({
+    rowChunkSize: config.searchPack.rowChunkSize,
+    compressionLevel: config.searchPack.compressionLevel,
+    useDictionary: config.searchPack.useDictionary
+});
 export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     const normalizedAgent = agentId?.trim();
     const tokens = tokenize(query);
     if (limit <= 0 || tokens.length === 0) {
         return [];
     }
-    const files = await sortedPackFiles(vaultPath);
+    const files = await selectCandidatePackFiles(vaultPath, tokens, normalizedAgent);
     if (files.length === 0) {
         return [];
     }
     const scored = [];
     for (const file of files) {
         const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
-        rows.forEach((row) => {
+        const traversal = middleOutIndices(rows.length, Math.floor(rows.length / 2));
+        traversal.forEach((rowIndex) => {
+            const row = rows[rowIndex];
+            if (!row) {
+                return;
+            }
             if (normalizedAgent && row.agentId !== normalizedAgent) {
                 return;
             }

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, contextInputSchema, contextTool, graphInputSchema, graphTool, indexInputSchema, indexTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, statsInputSchema, statsTool, syncInputSchema, syncTool, validateInputSchema, validateTool } from './tools.js';
+import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, dedupeInputSchema, dedupeResolveInputSchema, dedupeResolveTool, dedupeTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, contextInputSchema, contextTool, graphInputSchema, graphTool, indexInputSchema, indexTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, statsInputSchema, statsTool, syncInputSchema, syncTool, validateInputSchema, validateTool } from './tools.js';
 const readPackageVersion = () => {
     const packagePath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
     const metadata = JSON.parse(readFileSync(packagePath, 'utf8'));
@@ -40,6 +40,16 @@ export const createBrainlinkMcpServer = () => {
         description: 'Search indexed Brainlink notes with FTS, semantic or hybrid retrieval.',
         inputSchema: searchInputSchema
     }, searchTool);
+    server.registerTool('brainlink_dedupe', {
+        title: 'Detect Duplicate Notes',
+        description: 'Detect possible duplicate notes using exact content hash and semantic similarity scoring.',
+        inputSchema: dedupeInputSchema
+    }, dedupeTool);
+    server.registerTool('brainlink_resolve_duplicate', {
+        title: 'Resolve Duplicate Notes',
+        description: 'Resolve a duplicate pair with merge, link or ignore. Non-merge actions still create low-priority related edges.',
+        inputSchema: dedupeResolveInputSchema
+    }, dedupeResolveTool);
     server.registerTool('brainlink_add_note', {
         title: 'Add Brainlink Note',
         description: 'Write durable Markdown memory, then reindex the vault. Include explicit [[wiki links]] for connected graph memory. Add priority markers near links, such as priority: high, #important or #critical, when a relationship should be weighted higher.',

package/dist/mcp/tools.js

@@ -4,6 +4,7 @@ import { z } from 'zod';
 import { getBrokenLinksReport, getOrphansReport, getStats, validateVault } from '../application/analyze-vault.js';
 import { addNoteWithMetadata } from '../application/add-note.js';
 import { buildContextPackage } from '../application/build-context.js';
+import { resolveDuplicateNotes, scanDuplicateNotes } from '../application/dedupe-notes.js';
 import { getGraph } from '../application/get-graph.js';
 import { indexVault } from '../application/index-vault.js';
 import { searchKnowledge } from '../application/search-knowledge.js';
@@ -311,6 +312,20 @@ export const recommendationsInputSchema = {
     limit: optionalPositiveInteger().describe('Optional context limit override for generated recommendations.'),
     tokens: optionalPositiveInteger().describe('Optional context token budget override for generated recommendations.')
 };
+export const dedupeInputSchema = {
+    ...vaultInput,
+    ...agentInput,
+    limit: optionalPositiveInteger().describe('Maximum duplicate candidate pairs to return.'),
+    minScore: z.number().min(0).max(1).optional().describe('Minimum semantic similarity score between 0 and 1.'),
+    semantic: z.boolean().optional().default(true).describe('Enable semantic duplicate detection in addition to exact content hash matches.')
+};
+export const dedupeResolveInputSchema = {
+    ...vaultInput,
+    leftPath: z.string().min(1).describe('Left note path from dedupe results.'),
+    rightPath: z.string().min(1).describe('Right note path from dedupe results.'),
+    action: z.enum(['merge', 'link', 'ignore']).describe('Resolution action.'),
+    autoIndex: z.boolean().optional().default(true).describe('Reindex after duplicate resolution.')
+};
 export const contextTool = async (input) => {
     const context = await resolveExecutionContext(input);
     const readiness = await ensureBootstrapReady(context, input, 'brainlink_context');
@@ -364,6 +379,14 @@ export const addNoteTool = async (input) => {
         allowSensitive: input.allowSensitive
     });
     const index = shouldIndex ? await indexVault(context.vault) : undefined;
+    const focusPath = added.path.includes('agents/') ? added.path.slice(added.path.indexOf('agents/')).replaceAll('\\', '/') : undefined;
+    const possibleDuplicates = await scanDuplicateNotes(context.vault, {
+        agentId: context.agent,
+        focusPath,
+        limit: 5,
+        minSemanticScore: 0.92,
+        includeSemantic: true
+    });
     return jsonResult({
         vault: context.vault,
         title: input.title,
@@ -374,6 +397,7 @@ export const addNoteTool = async (input) => {
             linkTarget: added.linkTarget,
             guaranteedEdge: true
         },
+        possibleDuplicates,
         ...(index ? { index } : {})
     });
 };
@@ -792,6 +816,17 @@ export const recommendationsTool = async (input) => {
                 tokens
             }
         },
+        {
+            tool: 'brainlink_dedupe',
+            reason: 'Detect and resolve duplicate durable notes to keep memory quality high.',
+            args: {
+                vault: context.vault,
+                ...(context.agent ? { agent: context.agent } : {}),
+                limit: 10,
+                minScore: 0.92,
+                semantic: true
+            }
+        },
         {
             tool: 'brainlink_add_note',
             reason: 'Persist durable outcomes after task completion (write responses include connectivity metadata).',
@@ -818,3 +853,30 @@ export const recommendationsTool = async (input) => {
         recommendations
     });
 };
+export const dedupeTool = async (input) => {
+    const context = await resolveExecutionContext(input);
+    const duplicates = await scanDuplicateNotes(context.vault, {
+        agentId: context.agent,
+        limit: input.limit ?? 25,
+        minSemanticScore: input.minScore ?? 0.92,
+        includeSemantic: input.semantic !== false
+    });
+    return jsonResult({
+        vault: context.vault,
+        agent: context.agent,
+        duplicates
+    });
+};
+export const dedupeResolveTool = async (input) => {
+    const context = await resolveExecutionContext(input);
+    const result = await resolveDuplicateNotes(context.vault, {
+        leftPath: input.leftPath,
+        rightPath: input.rightPath,
+        action: input.action,
+        autoIndex: isTruthy(input.autoIndex)
+    });
+    return jsonResult({
+        vault: context.vault,
+        ...result
+    });
+};