@andespindola/brainlink 0.1.0-beta.8 → 0.1.0-beta.80

package/dist/application/add-note.js

@@ -1,30 +1,79 @@
+import { access } from 'node:fs/promises';
+import { join } from 'node:path';
 import { writeMarkdownFile } from '../infrastructure/file-system-vault.js';
 import { sanitizeAgentId, sharedAgentId } from '../domain/agents.js';
+import { extractWikiLinks } from '../domain/markdown.js';
 import { validateNoteInput } from '../domain/note-safety.js';
+import { ensureVault } from '../infrastructure/file-system-vault.js';
 const slugify = (title) => title
     .normalize('NFKD')
     .replace(/[\u0300-\u036f]/g, '')
     .toLowerCase()
     .replace(/[^a-z0-9]+/g, '-')
     .replace(/^-+|-+$/g, '');
-export const addNote = async (vaultPath, title, content, agentId = sharedAgentId, options = {}) => {
+const systemHubTitle = 'Memory Hub';
+const systemRootTitle = 'Knowledge Root';
+const normalizeTitle = (title) => title.trim().replace(/\.md$/i, '').toLowerCase();
+const noteFilename = (agentId, title) => `agents/${agentId}/${slugify(title) || 'untitled'}.md`;
+const buildNote = (title, content, agentId) => [
+    `---`,
+    `title: "${title.replaceAll('"', '\\"')}"`,
+    `agent: "${agentId}"`,
+    `---`,
+    '',
+    `# ${title}`,
+    '',
+    content.trim(),
+    ''
+].join('\n');
+const ensureSystemNote = async (vaultPath, absoluteVaultPath, agentId, title, content) => {
+    const filename = noteFilename(agentId, title);
+    const absolutePath = join(absoluteVaultPath, filename);
+    try {
+        await access(absolutePath);
+        return;
+    }
+    catch { }
+    await writeMarkdownFile(vaultPath, filename, buildNote(title, content, agentId));
+};
+const ensureNonOrphanContent = async (vaultPath, absoluteVaultPath, title, content, agentId) => {
+    const links = extractWikiLinks(content).filter((link) => normalizeTitle(link) !== normalizeTitle(title));
+    if (links.length > 0) {
+        return {
+            content: content.trim(),
+            autoLinked: false,
+            linkTarget: null
+        };
+    }
+    const fallbackTitle = normalizeTitle(title) === normalizeTitle(systemHubTitle) ? systemRootTitle : systemHubTitle;
+    if (fallbackTitle === systemRootTitle) {
+        await ensureSystemNote(vaultPath, absoluteVaultPath, agentId, systemRootTitle, `Entry point for agent memory. [[${systemHubTitle}]] #memory #root`);
+    }
+    else {
+        await ensureSystemNote(vaultPath, absoluteVaultPath, agentId, systemHubTitle, 'Central memory index for this agent namespace. #memory #hub');
+    }
+    return {
+        content: `${content.trim()}\n\nRelated: [[${fallbackTitle}]]`,
+        autoLinked: true,
+        linkTarget: fallbackTitle
+    };
+};
+export const addNoteWithMetadata = async (vaultPath, title, content, agentId = sharedAgentId, options = {}) => {
     validateNoteInput({
         title,
         content,
         allowSensitive: options.allowSensitive
     });
     const sanitizedAgentId = sanitizeAgentId(agentId);
+    const absoluteVaultPath = await ensureVault(vaultPath);
     const filename = `agents/${sanitizedAgentId}/${slugify(title) || 'untitled'}.md`;
-    const note = [
-        `---`,
-        `title: "${title.replaceAll('"', '\\"')}"`,
-        `agent: "${sanitizedAgentId}"`,
-        `---`,
-        '',
-        `# ${title}`,
-        '',
-        content.trim(),
-        ''
-    ].join('\n');
-    return writeMarkdownFile(vaultPath, filename, note);
+    const linkedContent = await ensureNonOrphanContent(vaultPath, absoluteVaultPath, title, content, sanitizedAgentId);
+    const note = buildNote(title, linkedContent.content, sanitizedAgentId);
+    const path = await writeMarkdownFile(vaultPath, filename, note);
+    return {
+        path,
+        autoLinked: linkedContent.autoLinked,
+        linkTarget: linkedContent.linkTarget
+    };
 };
+export const addNote = async (vaultPath, title, content, agentId = sharedAgentId, options = {}) => (await addNoteWithMetadata(vaultPath, title, content, agentId, options)).path;

package/dist/application/analyze-vault.js

@@ -1,10 +1,89 @@
+import { stat } from 'node:fs/promises';
+import { performance } from 'node:perf_hooks';
 import { validateGraph, getBrokenLinks, getOrphanNodes, getVaultStats } from '../domain/graph-analysis.js';
-import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
-import { getGraph } from './get-graph.js';
-export const getStats = async (vaultPath, agentId) => getVaultStats(await getGraph(vaultPath, agentId));
-export const getBrokenLinksReport = async (vaultPath, agentId) => getBrokenLinks(await getGraph(vaultPath, agentId));
-export const getOrphansReport = async (vaultPath, agentId) => getOrphanNodes(await getGraph(vaultPath, agentId));
-export const validateVault = async (vaultPath, agentId) => validateGraph(await getGraph(vaultPath, agentId));
+import { ensureVault, listVaultFiles, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
+import { resolveAgentRuntimeDefaults } from '../infrastructure/config.js';
+import { getGraphSummary } from './get-graph-summary.js';
+import { buildContextPackage } from './build-context.js';
+import { indexVault } from './index-vault.js';
+import { searchKnowledge } from './search-knowledge.js';
+import { loadBrainlinkConfig } from '../infrastructure/config.js';
+export const getStats = async (vaultPath, agentId) => getVaultStats(await getGraphSummary(vaultPath, agentId));
+export const getBrokenLinksReport = async (vaultPath, agentId) => getBrokenLinks(await getGraphSummary(vaultPath, agentId));
+export const getOrphansReport = async (vaultPath, agentId) => getOrphanNodes(await getGraphSummary(vaultPath, agentId));
+export const validateVault = async (vaultPath, agentId) => validateGraph(await getGraphSummary(vaultPath, agentId));
+const toRatio = (part, total) => total === 0 ? 0 : Number((part / total).toFixed(4));
+export const getExtendedStats = async (vaultPath, agentId) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const graph = await getGraphSummary(absoluteVaultPath, agentId);
+    const stats = getVaultStats(graph);
+    const markdownFiles = await readMarkdownFiles(absoluteVaultPath);
+    const allFiles = await listVaultFiles(absoluteVaultPath);
+    const totalBytes = (await Promise.all(allFiles.map(async (filePath) => {
+        try {
+            return (await stat(filePath)).size;
+        }
+        catch {
+            return 0;
+        }
+    }))).reduce((sum, value) => sum + value, 0);
+    const updatedAt = markdownFiles
+        .map((file) => file.updatedAt.getTime())
+        .filter((time) => Number.isFinite(time))
+        .sort((left, right) => left - right);
+    const priorities = graph.edges.reduce((state, edge) => ({
+        ...state,
+        [edge.priority]: state[edge.priority] + 1
+    }), {
+        low: 0,
+        normal: 0,
+        high: 0,
+        critical: 0
+    });
+    const config = await loadBrainlinkConfig();
+    const defaults = resolveAgentRuntimeDefaults(config, agentId);
+    const probeQuery = graph.nodes[0]?.title ?? 'architecture';
+    const indexStart = performance.now();
+    await indexVault(absoluteVaultPath);
+    const indexLatency = performance.now() - indexStart;
+    const searchStart = performance.now();
+    await searchKnowledge(absoluteVaultPath, probeQuery, Math.min(defaults.defaultSearchLimit, 8), agentId, 'hybrid');
+    const searchLatency = performance.now() - searchStart;
+    const contextStart = performance.now();
+    await buildContextPackage(absoluteVaultPath, probeQuery, Math.min(defaults.defaultSearchLimit, 8), defaults.defaultContextTokens, agentId, 'hybrid');
+    const contextLatency = performance.now() - contextStart;
+    return {
+        stats,
+        storage: {
+            markdownFileCount: markdownFiles.length,
+            totalFileCount: allFiles.length,
+            totalBytes,
+            averageMarkdownBytes: markdownFiles.length === 0
+                ? 0
+                : Math.round(markdownFiles.reduce((sum, file) => sum + Buffer.byteLength(file.content, 'utf8'), 0) / markdownFiles.length),
+            ...(updatedAt.length > 0
+                ? {
+                    oldestNoteUpdatedAt: new Date(updatedAt[0]).toISOString(),
+                    newestNoteUpdatedAt: new Date(updatedAt[updatedAt.length - 1]).toISOString()
+                }
+                : {})
+        },
+        quality: {
+            resolvedLinkRatio: toRatio(stats.resolvedLinkCount, stats.linkCount),
+            brokenLinkRatio: toRatio(stats.brokenLinkCount, stats.linkCount),
+            orphanRatio: toRatio(stats.orphanCount, Math.max(stats.documentCount, 1)),
+            priorityDistribution: priorities
+        },
+        observability: {
+            probeQuery,
+            latenciesMs: {
+                index: Number(indexLatency.toFixed(2)),
+                search: Number(searchLatency.toFixed(2)),
+                context: Number(contextLatency.toFixed(2))
+            }
+        }
+    };
+};
 const createCheck = (name, ok, message) => ({
     name,
     ok,
@@ -13,7 +92,7 @@ const createCheck = (name, ok, message) => ({
 export const doctorVault = async (vaultPath) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
     const files = await readMarkdownFiles(absoluteVaultPath);
-    const graph = await getGraph(absoluteVaultPath);
+    const graph = await getGraphSummary(absoluteVaultPath);
     const validation = validateGraph(graph);
     const checks = [
         createCheck('vault', true, `Vault ready at ${absoluteVaultPath}`),
@@ -21,8 +100,16 @@ export const doctorVault = async (vaultPath) => {
         createCheck('index', graph.nodes.length > 0, `${graph.nodes.length} indexed documents found`),
         createCheck('broken-links', validation.brokenLinks.length === 0, `${validation.brokenLinks.length} broken links found`)
     ];
+    const recommendations = files.length === 0 && graph.nodes.length === 0
+        ? [
+            `Vault is empty. Add your first note: blink add "Architecture" --vault "${absoluteVaultPath}" --content "Markdown source of truth. #architecture"`,
+            `If this path is not the expected vault, inspect active config: blink config where`,
+            `If you changed vault recently, migrate existing memory: blink migrate-vault --from ~/.brainlink/vault --to "${absoluteVaultPath}"`
+        ]
+        : [];
     return {
         ok: checks.every((check) => check.ok),
-        checks
+        checks,
+        ...(recommendations.length > 0 ? { recommendations } : {})
     };
 };

package/dist/application/build-context.js

@@ -1,13 +1,68 @@
+import { stat } from 'node:fs/promises';
 import { formatContextPackage, selectContextSections } from '../domain/context.js';
+import { indexStoragePath } from '../infrastructure/file-index.js';
 import { searchKnowledge } from './search-knowledge.js';
+const contextCacheTtlMs = 45_000;
+const contextCacheMaxEntries = 200;
+const contextCache = new Map();
+const readIndexMtimeMs = async (vaultPath) => {
+    try {
+        return (await stat(indexStoragePath(vaultPath))).mtimeMs;
+    }
+    catch {
+        return 0;
+    }
+};
+const toCacheKey = (vaultPath, query, limit, maxTokens, agentId, mode) => JSON.stringify({
+    vaultPath,
+    query: query.trim().toLowerCase(),
+    limit,
+    maxTokens,
+    agentId: agentId?.trim().toLowerCase() ?? '*',
+    mode: mode ?? 'default'
+});
+const contextCacheGet = (key, indexMtimeMs) => {
+    const entry = contextCache.get(key);
+    if (!entry) {
+        return undefined;
+    }
+    const fresh = Date.now() - entry.createdAt <= contextCacheTtlMs && entry.indexMtimeMs === indexMtimeMs;
+    if (!fresh) {
+        contextCache.delete(key);
+        return undefined;
+    }
+    return entry.context;
+};
+const contextCacheSet = (entry) => {
+    contextCache.set(entry.key, entry);
+    if (contextCache.size <= contextCacheMaxEntries) {
+        return;
+    }
+    const overflow = contextCache.size - contextCacheMaxEntries;
+    const keys = Array.from(contextCache.keys()).slice(0, overflow);
+    keys.forEach((key) => contextCache.delete(key));
+};
 export const buildContextPackage = async (vaultPath, query, limit, maxTokens, agentId, mode) => {
+    const cacheKey = toCacheKey(vaultPath, query, limit, maxTokens, agentId, mode);
+    const indexMtimeMs = await readIndexMtimeMs(vaultPath);
+    const cached = contextCacheGet(cacheKey, indexMtimeMs);
+    if (cached) {
+        return cached;
+    }
     const results = await searchKnowledge(vaultPath, query, limit, agentId, mode);
     const sections = selectContextSections(results, maxTokens);
-    return {
+    const context = {
         query,
         sections,
         content: formatContextPackage(query, sections)
     };
+    contextCacheSet({
+        key: cacheKey,
+        createdAt: Date.now(),
+        indexMtimeMs,
+        context
+    });
+    return context;
 };
 export const buildContext = async (vaultPath, query, limit, maxTokens, agentId, mode) => {
     const contextPackage = await buildContextPackage(vaultPath, query, limit, maxTokens, agentId, mode);

package/dist/application/dedupe-notes.js

@@ -0,0 +1,226 @@
+import { createHash } from 'node:crypto';
+import { createEmbeddingBuckets, createLocalEmbedding, cosineSimilarity } from '../domain/embeddings.js';
+import { parseMarkdownDocument } from '../domain/markdown.js';
+import { writeMarkdownFile, ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
+import { indexVault } from './index-vault.js';
+const tokenPattern = /[\p{L}\p{N}_-]+/gu;
+const frontmatterPattern = /^---\n[\s\S]*?\n---\n?/m;
+const rootHeadingPattern = /^#\s+.+\n+/m;
+const maxCandidatesPerBucket = 240;
+const normalizePath = (path) => path.replaceAll('\\', '/').replace(/^\.\//, '');
+const toComparableBody = (content) => content
+    .replace(frontmatterPattern, '')
+    .replace(rootHeadingPattern, '')
+    .replaceAll('\r\n', '\n')
+    .trim();
+const normalizeStrictContent = (content) => toComparableBody(content);
+const normalizeSemanticContent = (content) => toComparableBody(content)
+    .replace(/\s+/g, ' ')
+    .trim();
+const toHash = (value) => createHash('sha256').update(value, 'utf8').digest('hex');
+const toCandidateId = (leftPath, rightPath) => [normalizePath(leftPath), normalizePath(rightPath)].sort((left, right) => left.localeCompare(right)).join('|');
+const hasSharedTokens = (left, right) => {
+    const leftTokens = new Set((left.match(tokenPattern) ?? []).map((token) => token.toLowerCase()).filter((token) => token.length > 2));
+    const rightTokens = new Set((right.match(tokenPattern) ?? []).map((token) => token.toLowerCase()).filter((token) => token.length > 2));
+    if (leftTokens.size === 0 || rightTokens.size === 0) {
+        return false;
+    }
+    for (const token of leftTokens) {
+        if (rightTokens.has(token)) {
+            return true;
+        }
+    }
+    return false;
+};
+const relatedMarker = (targetTitle) => `Related: [[${targetTitle}]] priority: low #related-to`;
+const ensureRelatedEdgeLine = (content, targetTitle) => {
+    const linkPattern = new RegExp(`\\[\\[\\s*${targetTitle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*(?:[\\]|#])?`, 'i');
+    if (linkPattern.test(content)) {
+        return content;
+    }
+    const trimmed = content.trimEnd();
+    return `${trimmed}\n\n${relatedMarker(targetTitle)}\n`;
+};
+const ensureMergedMarker = (content, targetTitle) => {
+    const marker = `Merged into [[${targetTitle}]]`;
+    if (content.includes(marker)) {
+        return content;
+    }
+    return `${content.trimEnd()}\n\n${marker} priority: low #related-to\n`;
+};
+const appendMergedContent = (baseContent, mergedTitle, mergedContent) => {
+    const marker = `## Merged Memory From [[${mergedTitle}]]`;
+    if (baseContent.includes(marker)) {
+        return baseContent;
+    }
+    const mergedBody = normalizeSemanticContent(mergedContent);
+    return `${baseContent.trimEnd()}\n\n${marker}\n\n${mergedBody}\n`;
+};
+const loadNoteRecords = async (vaultPath, agentId) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const files = await readMarkdownFiles(vaultPath);
+    return files
+        .map((file) => {
+        const parsed = parseMarkdownDocument({
+            absolutePath: file.absolutePath,
+            vaultPath: absoluteVaultPath,
+            content: file.content,
+            createdAt: file.createdAt,
+            updatedAt: file.updatedAt
+        });
+        const strict = normalizeStrictContent(parsed.content);
+        const semantic = normalizeSemanticContent(parsed.content);
+        const embedding = createLocalEmbedding(`${parsed.title}\n${semantic}`);
+        return {
+            title: parsed.title,
+            path: normalizePath(parsed.path),
+            agentId: parsed.agentId,
+            content: parsed.content,
+            normalizedStrictContent: strict,
+            semanticContent: semantic,
+            embedding,
+            buckets: createEmbeddingBuckets(embedding, 20)
+        };
+    })
+        .filter((record) => (agentId ? record.agentId === agentId : true));
+};
+const pairToCandidate = (left, right, kind, score, reason) => ({
+    id: toCandidateId(left.path, right.path),
+    possibleDuplicate: true,
+    kind,
+    score: Number(score.toFixed(4)),
+    left: {
+        title: left.title,
+        path: left.path,
+        agentId: left.agentId
+    },
+    right: {
+        title: right.title,
+        path: right.path,
+        agentId: right.agentId
+    },
+    reason
+});
+const indexCandidatePairs = (notes) => {
+    const bucketMap = new Map();
+    notes.forEach((note, index) => {
+        note.buckets.forEach((bucket) => {
+            const current = bucketMap.get(bucket) ?? [];
+            if (current.length < maxCandidatesPerBucket) {
+                current.push(index);
+                bucketMap.set(bucket, current);
+            }
+        });
+    });
+    const pairKeys = new Set();
+    const pairs = [];
+    bucketMap.forEach((indexes) => {
+        for (let leftIndex = 0; leftIndex < indexes.length; leftIndex += 1) {
+            for (let rightIndex = leftIndex + 1; rightIndex < indexes.length; rightIndex += 1) {
+                const left = Math.min(indexes[leftIndex] ?? 0, indexes[rightIndex] ?? 0);
+                const right = Math.max(indexes[leftIndex] ?? 0, indexes[rightIndex] ?? 0);
+                const key = `${left}|${right}`;
+                if (!pairKeys.has(key)) {
+                    pairKeys.add(key);
+                    pairs.push([left, right]);
+                }
+            }
+        }
+    });
+    return pairs;
+};
+export const scanDuplicateNotes = async (vaultPath, options = {}) => {
+    const notes = await loadNoteRecords(vaultPath, options.agentId);
+    if (notes.length < 2) {
+        return [];
+    }
+    const minSemanticScore = options.minSemanticScore ?? 0.92;
+    const includeSemantic = options.includeSemantic !== false;
+    const seen = new Map();
+    const byHash = notes.reduce((state, note) => {
+        const key = toHash(note.normalizedStrictContent);
+        const current = state.get(key) ?? [];
+        current.push(note);
+        state.set(key, current);
+        return state;
+    }, new Map());
+    byHash.forEach((group) => {
+        if (group.length < 2) {
+            return;
+        }
+        const [base, ...rest] = group.sort((left, right) => left.path.localeCompare(right.path));
+        rest.forEach((note) => {
+            const candidate = pairToCandidate(base, note, 'exact', 1, 'Exact content hash match');
+            seen.set(candidate.id, candidate);
+        });
+    });
+    if (includeSemantic) {
+        const pairs = indexCandidatePairs(notes);
+        pairs.forEach(([leftIndex, rightIndex]) => {
+            const left = notes[leftIndex];
+            const right = notes[rightIndex];
+            if (!left || !right || left.path === right.path) {
+                return;
+            }
+            const id = toCandidateId(left.path, right.path);
+            if (seen.has(id)) {
+                return;
+            }
+            const score = cosineSimilarity(left.embedding, right.embedding);
+            const titleShared = hasSharedTokens(left.title, right.title);
+            const contentShared = hasSharedTokens(left.semanticContent, right.semanticContent);
+            if (score >= minSemanticScore && (titleShared || contentShared || score >= 0.975)) {
+                const candidate = pairToCandidate(left, right, 'semantic', score, 'High semantic similarity');
+                seen.set(id, candidate);
+            }
+        });
+    }
+    const focusPath = options.focusPath ? normalizePath(options.focusPath) : undefined;
+    const limited = Array.from(seen.values())
+        .filter((item) => (focusPath ? item.left.path === focusPath || item.right.path === focusPath : true))
+        .sort((left, right) => right.score - left.score || left.left.path.localeCompare(right.left.path))
+        .slice(0, Math.max(1, options.limit ?? 25));
+    return limited;
+};
+export const resolveDuplicateNotes = async (vaultPath, options) => {
+    const leftPath = normalizePath(options.leftPath);
+    const rightPath = normalizePath(options.rightPath);
+    if (leftPath === rightPath) {
+        throw new Error('leftPath and rightPath must be different notes.');
+    }
+    const notes = await loadNoteRecords(vaultPath);
+    const byPath = new Map(notes.map((note) => [note.path, note]));
+    const left = byPath.get(leftPath);
+    const right = byPath.get(rightPath);
+    if (!left || !right) {
+        throw new Error(`Duplicate resolution paths were not found in vault index source: ${leftPath}, ${rightPath}`);
+    }
+    const updates = new Map();
+    const leftRelated = ensureRelatedEdgeLine(left.content, right.title);
+    const rightRelated = ensureRelatedEdgeLine(right.content, left.title);
+    if (options.action === 'link') {
+        updates.set(left.path, leftRelated);
+        updates.set(right.path, rightRelated);
+    }
+    else if (options.action === 'ignore') {
+        updates.set(left.path, leftRelated);
+    }
+    else {
+        const mergedLeft = appendMergedContent(leftRelated, right.title, right.content);
+        const mergedRight = ensureMergedMarker(rightRelated, left.title);
+        updates.set(left.path, mergedLeft);
+        updates.set(right.path, mergedRight);
+    }
+    for (const [path, content] of updates) {
+        await writeMarkdownFile(vaultPath, path, content);
+    }
+    const shouldIndex = options.autoIndex !== false;
+    const index = shouldIndex ? await indexVault(vaultPath) : undefined;
+    return {
+        action: options.action,
+        leftPath,
+        rightPath,
+        updatedPaths: Array.from(updates.keys()).sort((leftValue, rightValue) => leftValue.localeCompare(rightValue)),
+        ...(index ? { index } : {})
+    };
+};