@andespindola/brainlink 0.1.0-beta.16 → 0.1.0-beta.160

package/dist/application/build-context.js

@@ -1,13 +1,74 @@
+import { stat } from 'node:fs/promises';
 import { formatContextPackage, selectContextSections } from '../domain/context.js';
+import { indexStoragePath } from '../infrastructure/file-index.js';
+import { searchVolatileMemory, volatileMemoryStoragePath } from '../infrastructure/volatile-memory.js';
 import { searchKnowledge } from './search-knowledge.js';
+const contextCacheTtlMs = 45_000;
+const contextCacheMaxEntries = 200;
+const contextCache = new Map();
+const readFileSignature = async (path) => {
+    try {
+        const info = await stat(path);
+        return `${Math.floor(info.mtimeMs)}:${info.size}`;
+    }
+    catch {
+        return '0:0';
+    }
+};
+const readContextDataSignature = async (vaultPath) => `${await readFileSignature(indexStoragePath(vaultPath))}|${await readFileSignature(volatileMemoryStoragePath(vaultPath))}`;
+const toCacheKey = (vaultPath, query, limit, maxTokens, agentId, mode) => JSON.stringify({
+    vaultPath,
+    query: query.trim().toLowerCase(),
+    limit,
+    maxTokens,
+    agentId: agentId?.trim().toLowerCase() ?? '*',
+    mode: mode ?? 'default'
+});
+const contextCacheGet = (key, dataSignature) => {
+    const entry = contextCache.get(key);
+    if (!entry) {
+        return undefined;
+    }
+    const fresh = Date.now() - entry.createdAt <= contextCacheTtlMs && entry.dataSignature === dataSignature;
+    if (!fresh) {
+        contextCache.delete(key);
+        return undefined;
+    }
+    return entry.context;
+};
+const contextCacheSet = (entry) => {
+    contextCache.set(entry.key, entry);
+    if (contextCache.size <= contextCacheMaxEntries) {
+        return;
+    }
+    const overflow = contextCache.size - contextCacheMaxEntries;
+    const keys = Array.from(contextCache.keys()).slice(0, overflow);
+    keys.forEach((key) => contextCache.delete(key));
+};
 export const buildContextPackage = async (vaultPath, query, limit, maxTokens, agentId, mode) => {
+    const cacheKey = toCacheKey(vaultPath, query, limit, maxTokens, agentId, mode);
+    const dataSignature = await readContextDataSignature(vaultPath);
+    const cached = contextCacheGet(cacheKey, dataSignature);
+    if (cached) {
+        return cached;
+    }
     const results = await searchKnowledge(vaultPath, query, limit, agentId, mode);
-    const sections = selectContextSections(results, maxTokens);
-    return {
+    const durableSections = selectContextSections(results, maxTokens);
+    const volatileSections = await searchVolatileMemory(vaultPath, query, Math.min(3, limit), agentId, mode ?? 'hybrid');
+    const sections = [...volatileSections, ...durableSections];
+    const context = {
         query,
         sections,
-        content: formatContextPackage(query, sections)
+        content: formatContextPackage(query, sections),
+        ...(volatileSections.length > 0 ? { volatileSections } : {})
     };
+    contextCacheSet({
+        key: cacheKey,
+        createdAt: Date.now(),
+        dataSignature,
+        context
+    });
+    return context;
 };
 export const buildContext = async (vaultPath, query, limit, maxTokens, agentId, mode) => {
     const contextPackage = await buildContextPackage(vaultPath, query, limit, maxTokens, agentId, mode);

package/dist/application/canonical-context-links.js

@@ -0,0 +1,209 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { inferVisualGraphContext } from '../domain/graph-contexts.js';
+import { sanitizeAgentId, sharedAgentId } from '../domain/agents.js';
+import { extractContextLinkWeights, parseMarkdownDocument } from '../domain/markdown.js';
+import { ensureVault, readMarkdownFileSummaries, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
+const canonicalPriority = 'high';
+const slugify = (title) => title
+    .normalize('NFKD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+export const hubTitleForContext = (contextTitle) => `${contextTitle} Hub`;
+const hubPathForContext = (contextTitle, agentId) => {
+    if (contextTitle === 'GitHub Repositories')
+        return 'github-repos/github-repositories-hub.md';
+    if (contextTitle === 'GitHub Organizations')
+        return 'github-org-repos/github-organizations-hub.md';
+    if (contextTitle === 'Machine Configuration')
+        return 'machine-config/machine-configuration-hub.md';
+    return join('agents', sanitizeAgentId(agentId), `${slugify(hubTitleForContext(contextTitle))}.md`).replaceAll('\\', '/');
+};
+const normalizeTitle = (title) => title.trim().replace(/\.md$/i, '').toLowerCase();
+const hasCanonicalLink = (content, hubTitle) => extractContextLinkWeights(content).some((link) => normalizeTitle(link.title) === normalizeTitle(hubTitle));
+const linkLine = (hubTitle) => `- [[${hubTitle}]] priority: ${canonicalPriority}`;
+const contextLinksHeading = (line) => line.match(/^(#{2,6})\s+(Context Links|Links de Contexto)\s*$/i);
+export const upsertCanonicalContextLink = (content, hubTitle) => {
+    if (hasCanonicalLink(content, hubTitle)) {
+        return content;
+    }
+    const lines = content.replace(/\s+$/u, '').split('\n');
+    const headingIndex = lines.findIndex((line) => contextLinksHeading(line.trim()));
+    if (headingIndex === -1) {
+        return `${lines.join('\n')}\n\n## Context Links\n\n${linkLine(hubTitle)}\n`;
+    }
+    const heading = contextLinksHeading(lines[headingIndex].trim());
+    const headingDepth = heading?.[1]?.length ?? 2;
+    const insertIndex = lines.findIndex((line, index) => {
+        if (index <= headingIndex)
+            return false;
+        const candidate = line.match(/^(#{2,6})\s+/);
+        return Boolean(candidate && candidate[1].length <= headingDepth);
+    });
+    const targetIndex = insertIndex === -1 ? lines.length : insertIndex;
+    const before = lines.slice(0, targetIndex);
+    const after = lines.slice(targetIndex);
+    const needsSpacer = before[before.length - 1]?.trim() !== '';
+    const nextLines = [...before, ...(needsSpacer ? [''] : []), linkLine(hubTitle), ...after];
+    return `${nextLines.join('\n').replace(/\s+$/u, '')}\n`;
+};
+const buildHubContent = (hubTitle, contextTitle, agentId) => [
+    '---',
+    `title: "${hubTitle.replaceAll('"', '\\"')}"`,
+    `agent: "${sanitizeAgentId(agentId)}"`,
+    '---',
+    '',
+    `# ${hubTitle}`,
+    '',
+    `Canonical hub for the ${contextTitle} context. #memory #hub`,
+    ''
+].join('\n');
+const readNotes = async (vaultPath) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const summaries = await readMarkdownFileSummaries(absoluteVaultPath);
+    return Promise.all(summaries.map(async (summary) => {
+        const content = await readFile(summary.absolutePath, 'utf8');
+        const document = parseMarkdownDocument({
+            absolutePath: summary.absolutePath,
+            vaultPath: absoluteVaultPath,
+            content,
+            createdAt: summary.createdAt,
+            updatedAt: summary.updatedAt
+        });
+        return {
+            summary,
+            content,
+            document
+        };
+    }));
+};
+export const ensureCanonicalContextHub = async (vaultPath, contextTitle, agentId = sharedAgentId) => {
+    const hubTitle = hubTitleForContext(contextTitle);
+    const notes = await readNotes(vaultPath);
+    const existing = notes.find((note) => normalizeTitle(note.document.title) === normalizeTitle(hubTitle));
+    const hubPath = existing?.summary.relativePath ?? hubPathForContext(contextTitle, agentId);
+    if (existing) {
+        return {
+            created: false,
+            title: hubTitle,
+            path: hubPath
+        };
+    }
+    const path = await writeMarkdownFile(vaultPath, hubPath, buildHubContent(hubTitle, contextTitle, agentId));
+    return {
+        created: true,
+        title: hubTitle,
+        path
+    };
+};
+export const canonicalizeContextLinks = async (vaultPath, options = {}) => {
+    const agentId = options.agentId ? sanitizeAgentId(options.agentId) : undefined;
+    const createMissingHubs = options.createMissingHubs !== false;
+    const notes = await readNotes(vaultPath);
+    const scopedNotes = agentId ? notes.filter((note) => note.document.agentId === agentId) : notes;
+    const knownTitles = new Set(notes.map((note) => normalizeTitle(note.document.title)));
+    const entries = [];
+    const ensureHub = async (contextTitle, hubTitle, targetAgentId) => {
+        if (knownTitles.has(normalizeTitle(hubTitle))) {
+            return true;
+        }
+        const path = hubPathForContext(contextTitle, targetAgentId);
+        if (!createMissingHubs) {
+            entries.push({
+                path,
+                title: hubTitle,
+                context: contextTitle,
+                hubTitle,
+                changed: false,
+                reason: 'missing-hub'
+            });
+            return false;
+        }
+        knownTitles.add(normalizeTitle(hubTitle));
+        if (!options.dryRun) {
+            await writeMarkdownFile(vaultPath, path, buildHubContent(hubTitle, contextTitle, targetAgentId));
+        }
+        entries.push({
+            path,
+            title: hubTitle,
+            context: contextTitle,
+            hubTitle,
+            changed: true,
+            reason: 'created-hub'
+        });
+        return true;
+    };
+    for (const note of scopedNotes) {
+        const context = inferVisualGraphContext(note.document);
+        const hubTitle = hubTitleForContext(context.title);
+        const isHub = normalizeTitle(note.document.title) === normalizeTitle(hubTitle);
+        if (isHub) {
+            entries.push({
+                path: note.summary.relativePath,
+                title: note.document.title,
+                context: context.title,
+                hubTitle,
+                changed: false,
+                reason: 'hub-note'
+            });
+            continue;
+        }
+        const hubAvailable = await ensureHub(context.title, hubTitle, note.document.agentId || sharedAgentId);
+        if (!hubAvailable) {
+            continue;
+        }
+        if (hasCanonicalLink(note.content, hubTitle)) {
+            entries.push({
+                path: note.summary.relativePath,
+                title: note.document.title,
+                context: context.title,
+                hubTitle,
+                changed: false,
+                reason: 'already-linked'
+            });
+            continue;
+        }
+        const nextContent = upsertCanonicalContextLink(note.content, hubTitle);
+        if (!options.dryRun) {
+            await writeMarkdownFile(vaultPath, note.summary.relativePath, nextContent);
+        }
+        entries.push({
+            path: note.summary.relativePath,
+            title: note.document.title,
+            context: context.title,
+            hubTitle,
+            changed: true,
+            reason: 'added-context-link'
+        });
+    }
+    const changed = entries.filter((entry) => entry.changed).length;
+    const createdHubs = entries.filter((entry) => entry.reason === 'created-hub' && entry.changed).length;
+    return {
+        dryRun: options.dryRun === true,
+        scanned: scopedNotes.length,
+        changed,
+        createdHubs,
+        skipped: entries.length - changed,
+        entries
+    };
+};
+export const addCanonicalContextLinkToContent = (title, content) => {
+    const context = inferVisualGraphContext({
+        id: '',
+        agentId: sharedAgentId,
+        title,
+        path: '',
+        content,
+        tags: [],
+    });
+    const hubTitle = hubTitleForContext(context.title);
+    const nextContent = normalizeTitle(title) === normalizeTitle(hubTitle) ? content : upsertCanonicalContextLink(content, hubTitle);
+    return {
+        content: nextContent,
+        context: context.title,
+        hubTitle,
+        changed: nextContent !== content
+    };
+};

package/dist/application/dedupe-notes.js

@@ -0,0 +1,226 @@
+import { createHash } from 'node:crypto';
+import { createEmbeddingBuckets, createLocalEmbedding, cosineSimilarity } from '../domain/embeddings.js';
+import { parseMarkdownDocument } from '../domain/markdown.js';
+import { writeMarkdownFile, ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
+import { indexVault } from './index-vault.js';
+const tokenPattern = /[\p{L}\p{N}_-]+/gu;
+const frontmatterPattern = /^---\n[\s\S]*?\n---\n?/m;
+const rootHeadingPattern = /^#\s+.+\n+/m;
+const maxCandidatesPerBucket = 240;
+const normalizePath = (path) => path.replaceAll('\\', '/').replace(/^\.\//, '');
+const toComparableBody = (content) => content
+    .replace(frontmatterPattern, '')
+    .replace(rootHeadingPattern, '')
+    .replaceAll('\r\n', '\n')
+    .trim();
+const normalizeStrictContent = (content) => toComparableBody(content);
+const normalizeSemanticContent = (content) => toComparableBody(content)
+    .replace(/\s+/g, ' ')
+    .trim();
+const toHash = (value) => createHash('sha256').update(value, 'utf8').digest('hex');
+const toCandidateId = (leftPath, rightPath) => [normalizePath(leftPath), normalizePath(rightPath)].sort((left, right) => left.localeCompare(right)).join('|');
+const hasSharedTokens = (left, right) => {
+    const leftTokens = new Set((left.match(tokenPattern) ?? []).map((token) => token.toLowerCase()).filter((token) => token.length > 2));
+    const rightTokens = new Set((right.match(tokenPattern) ?? []).map((token) => token.toLowerCase()).filter((token) => token.length > 2));
+    if (leftTokens.size === 0 || rightTokens.size === 0) {
+        return false;
+    }
+    for (const token of leftTokens) {
+        if (rightTokens.has(token)) {
+            return true;
+        }
+    }
+    return false;
+};
+const relatedMarker = (targetTitle) => `Related: [[${targetTitle}]] priority: low #related-to`;
+const ensureRelatedEdgeLine = (content, targetTitle) => {
+    const linkPattern = new RegExp(`\\[\\[\\s*${targetTitle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*(?:[\\]|#])?`, 'i');
+    if (linkPattern.test(content)) {
+        return content;
+    }
+    const trimmed = content.trimEnd();
+    return `${trimmed}\n\n${relatedMarker(targetTitle)}\n`;
+};
+const ensureMergedMarker = (content, targetTitle) => {
+    const marker = `Merged into [[${targetTitle}]]`;
+    if (content.includes(marker)) {
+        return content;
+    }
+    return `${content.trimEnd()}\n\n${marker} priority: low #related-to\n`;
+};
+const appendMergedContent = (baseContent, mergedTitle, mergedContent) => {
+    const marker = `## Merged Memory From [[${mergedTitle}]]`;
+    if (baseContent.includes(marker)) {
+        return baseContent;
+    }
+    const mergedBody = normalizeSemanticContent(mergedContent);
+    return `${baseContent.trimEnd()}\n\n${marker}\n\n${mergedBody}\n`;
+};
+const loadNoteRecords = async (vaultPath, agentId) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const files = await readMarkdownFiles(vaultPath);
+    return files
+        .map((file) => {
+        const parsed = parseMarkdownDocument({
+            absolutePath: file.absolutePath,
+            vaultPath: absoluteVaultPath,
+            content: file.content,
+            createdAt: file.createdAt,
+            updatedAt: file.updatedAt
+        });
+        const strict = normalizeStrictContent(parsed.content);
+        const semantic = normalizeSemanticContent(parsed.content);
+        const embedding = createLocalEmbedding(`${parsed.title}\n${semantic}`);
+        return {
+            title: parsed.title,
+            path: normalizePath(parsed.path),
+            agentId: parsed.agentId,
+            content: parsed.content,
+            normalizedStrictContent: strict,
+            semanticContent: semantic,
+            embedding,
+            buckets: createEmbeddingBuckets(embedding, 20)
+        };
+    })
+        .filter((record) => (agentId ? record.agentId === agentId : true));
+};
+const pairToCandidate = (left, right, kind, score, reason) => ({
+    id: toCandidateId(left.path, right.path),
+    possibleDuplicate: true,
+    kind,
+    score: Number(score.toFixed(4)),
+    left: {
+        title: left.title,
+        path: left.path,
+        agentId: left.agentId
+    },
+    right: {
+        title: right.title,
+        path: right.path,
+        agentId: right.agentId
+    },
+    reason
+});
+const indexCandidatePairs = (notes) => {
+    const bucketMap = new Map();
+    notes.forEach((note, index) => {
+        note.buckets.forEach((bucket) => {
+            const current = bucketMap.get(bucket) ?? [];
+            if (current.length < maxCandidatesPerBucket) {
+                current.push(index);
+                bucketMap.set(bucket, current);
+            }
+        });
+    });
+    const pairKeys = new Set();
+    const pairs = [];
+    bucketMap.forEach((indexes) => {
+        for (let leftIndex = 0; leftIndex < indexes.length; leftIndex += 1) {
+            for (let rightIndex = leftIndex + 1; rightIndex < indexes.length; rightIndex += 1) {
+                const left = Math.min(indexes[leftIndex] ?? 0, indexes[rightIndex] ?? 0);
+                const right = Math.max(indexes[leftIndex] ?? 0, indexes[rightIndex] ?? 0);
+                const key = `${left}|${right}`;
+                if (!pairKeys.has(key)) {
+                    pairKeys.add(key);
+                    pairs.push([left, right]);
+                }
+            }
+        }
+    });
+    return pairs;
+};
+export const scanDuplicateNotes = async (vaultPath, options = {}) => {
+    const notes = await loadNoteRecords(vaultPath, options.agentId);
+    if (notes.length < 2) {
+        return [];
+    }
+    const minSemanticScore = options.minSemanticScore ?? 0.92;
+    const includeSemantic = options.includeSemantic !== false;
+    const seen = new Map();
+    const byHash = notes.reduce((state, note) => {
+        const key = toHash(note.normalizedStrictContent);
+        const current = state.get(key) ?? [];
+        current.push(note);
+        state.set(key, current);
+        return state;
+    }, new Map());
+    byHash.forEach((group) => {
+        if (group.length < 2) {
+            return;
+        }
+        const [base, ...rest] = group.sort((left, right) => left.path.localeCompare(right.path));
+        rest.forEach((note) => {
+            const candidate = pairToCandidate(base, note, 'exact', 1, 'Exact content hash match');
+            seen.set(candidate.id, candidate);
+        });
+    });
+    if (includeSemantic) {
+        const pairs = indexCandidatePairs(notes);
+        pairs.forEach(([leftIndex, rightIndex]) => {
+            const left = notes[leftIndex];
+            const right = notes[rightIndex];
+            if (!left || !right || left.path === right.path) {
+                return;
+            }
+            const id = toCandidateId(left.path, right.path);
+            if (seen.has(id)) {
+                return;
+            }
+            const score = cosineSimilarity(left.embedding, right.embedding);
+            const titleShared = hasSharedTokens(left.title, right.title);
+            const contentShared = hasSharedTokens(left.semanticContent, right.semanticContent);
+            if (score >= minSemanticScore && (titleShared || contentShared || score >= 0.975)) {
+                const candidate = pairToCandidate(left, right, 'semantic', score, 'High semantic similarity');
+                seen.set(id, candidate);
+            }
+        });
+    }
+    const focusPath = options.focusPath ? normalizePath(options.focusPath) : undefined;
+    const limited = Array.from(seen.values())
+        .filter((item) => (focusPath ? item.left.path === focusPath || item.right.path === focusPath : true))
+        .sort((left, right) => right.score - left.score || left.left.path.localeCompare(right.left.path))
+        .slice(0, Math.max(1, options.limit ?? 25));
+    return limited;
+};
+export const resolveDuplicateNotes = async (vaultPath, options) => {
+    const leftPath = normalizePath(options.leftPath);
+    const rightPath = normalizePath(options.rightPath);
+    if (leftPath === rightPath) {
+        throw new Error('leftPath and rightPath must be different notes.');
+    }
+    const notes = await loadNoteRecords(vaultPath);
+    const byPath = new Map(notes.map((note) => [note.path, note]));
+    const left = byPath.get(leftPath);
+    const right = byPath.get(rightPath);
+    if (!left || !right) {
+        throw new Error(`Duplicate resolution paths were not found in vault index source: ${leftPath}, ${rightPath}`);
+    }
+    const updates = new Map();
+    const leftRelated = ensureRelatedEdgeLine(left.content, right.title);
+    const rightRelated = ensureRelatedEdgeLine(right.content, left.title);
+    if (options.action === 'link') {
+        updates.set(left.path, leftRelated);
+        updates.set(right.path, rightRelated);
+    }
+    else if (options.action === 'ignore') {
+        updates.set(left.path, leftRelated);
+    }
+    else {
+        const mergedLeft = appendMergedContent(leftRelated, right.title, right.content);
+        const mergedRight = ensureMergedMarker(rightRelated, left.title);
+        updates.set(left.path, mergedLeft);
+        updates.set(right.path, mergedRight);
+    }
+    for (const [path, content] of updates) {
+        await writeMarkdownFile(vaultPath, path, content);
+    }
+    const shouldIndex = options.autoIndex !== false;
+    const index = shouldIndex ? await indexVault(vaultPath) : undefined;
+    return {
+        action: options.action,
+        leftPath,
+        rightPath,
+        updatedPaths: Array.from(updates.keys()).sort((leftValue, rightValue) => leftValue.localeCompare(rightValue)),
+        ...(index ? { index } : {})
+    };
+};