@andespindola/brainlink 0.1.0-beta.16 → 0.1.0-beta.161

package/dist/application/index-vault.js

@@ -1,9 +1,11 @@
-import { createIndexedDocument, parseMarkdownDocument } from '../domain/markdown.js';
+import { readFile } from 'node:fs/promises';
+import { createIndexedDocument, graphLinkModelVersion, parseMarkdownDocument } from '../domain/markdown.js';
 import { sharedAgentId } from '../domain/agents.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig } from '../infrastructure/config.js';
-import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
-import { buildSearchPacks } from '../infrastructure/search-packs.js';
+import { ensureVault, readMarkdownFileSummaries } from '../infrastructure/file-system-vault.js';
+import { readIndexState, writeIndexState } from '../infrastructure/index-state.js';
+import { buildSearchPacks, ensureSearchPackManifest, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
 import { openFileIndex } from '../infrastructure/file-index.js';
 const toTitleKey = (title) => title.toLowerCase();
 const appendTitleEntry = (map, document) => {
@@ -34,6 +36,9 @@ const createScopedTitleResolver = (document, titleMaps) => ({
     get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
 });
 const embedIndexedDocuments = async (documents, providerName) => {
+    if (documents.length === 0) {
+        return documents;
+    }
     const provider = createEmbeddingProvider(providerName);
     const chunks = documents.flatMap((document) => document.chunks);
     const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
@@ -47,34 +52,268 @@ const embedIndexedDocuments = async (documents, providerName) => {
         }))
     }));
 };
-export const indexVault = async (vaultPath) => {
+const relinkIndexedDocument = (indexedDocument, titleMaps) => {
+    const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
+    return {
+        ...indexedDocument,
+        links: indexedDocument.links
+            .map((link) => ({
+            ...link,
+            toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
+        }))
+            .filter((link) => link.toDocumentId !== indexedDocument.document.id)
+    };
+};
+const toIndexResult = (documents) => ({
+    documentCount: documents.length,
+    chunkCount: documents.reduce((total, document) => total + document.chunks.length, 0),
+    linkCount: documents.reduce((total, document) => total + document.links.length, 0)
+});
+const toSnapshot = (summaries) => summaries.map((summary) => ({
+    path: summary.relativePath,
+    mtimeMs: summary.updatedAt.getTime(),
+    size: summary.size
+}));
+const createSnapshotMap = (snapshot) => new Map(snapshot.map((entry) => [entry.path, entry]));
+const readChangedDocuments = async (absoluteVaultPath, changedSummaries) => {
+    const parsed = await Promise.all(changedSummaries.map(async (summary) => parseMarkdownDocument({
+        absolutePath: summary.absolutePath,
+        vaultPath: absoluteVaultPath,
+        content: await readFile(summary.absolutePath, 'utf8'),
+        createdAt: summary.createdAt,
+        updatedAt: summary.updatedAt
+    })));
+    return new Map(parsed.map((document) => [document.path, document]));
+};
+export const indexVault = async (vaultPath, options = {}) => {
+    return indexVaultWithOptions(vaultPath, options);
+};
+export const indexVaultWithOptions = async (vaultPath, options) => {
+    const startedAt = process.hrtime.bigint();
+    const elapsedMs = () => Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const emit = (phase, status, message, details) => {
+        options.onProgress?.({
+            phase,
+            status,
+            message,
+            elapsedMs: elapsedMs(),
+            timestamp: new Date().toISOString(),
+            details
+        });
+    };
+    emit('start', 'start', 'Indexing started');
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
-    const files = await readMarkdownFiles(absoluteVaultPath);
-    const documents = files.map((file) => parseMarkdownDocument({
-        absolutePath: file.absolutePath,
-        vaultPath: absoluteVaultPath,
-        content: file.content,
-        createdAt: file.createdAt,
-        updatedAt: file.updatedAt
-    }));
-    const titleMaps = createTitleMaps(documents);
-    const indexedDocuments = await embedIndexedDocuments(documents.map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider);
+    emit('scan', 'start', 'Scanning markdown files');
+    const [summaries, previousState] = await Promise.all([
+        readMarkdownFileSummaries(absoluteVaultPath),
+        readIndexState(absoluteVaultPath)
+    ]);
+    emit('scan', 'finish', 'Scan complete', {
+        markdownFiles: summaries.length,
+        hasPreviousState: previousState != null
+    });
+    const fullReindex = options.full === true;
     const index = openFileIndex(absoluteVaultPath);
     try {
-        await index.reset();
+        const existingIndexedDocuments = await index.getIndexedDocuments();
+        const existingByPath = new Map(existingIndexedDocuments.map((document) => [document.document.path, document]));
+        const currentSnapshot = toSnapshot(summaries);
+        const currentSnapshotMap = createSnapshotMap(currentSnapshot);
+        const previousSnapshotMap = createSnapshotMap(previousState?.files ?? []);
+        const graphLinkModelChanged = previousState != null &&
+            previousState.graphLinkModelVersion !== graphLinkModelVersion;
+        const fullSourceReindex = fullReindex || graphLinkModelChanged;
+        const settingsChanged = previousState == null ||
+            previousState.chunkSize !== config.chunkSize ||
+            previousState.embeddingProvider !== config.embeddingProvider ||
+            graphLinkModelChanged;
+        const packSettingsChanged = previousState == null ||
+            previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
+            previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
+            previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
+        const changedPaths = new Set();
+        for (let index = 0; index < summaries.length; index += 1) {
+            const summary = summaries[index];
+            const previous = previousSnapshotMap.get(summary.relativePath);
+            const changed = fullSourceReindex ||
+                settingsChanged ||
+                previous == null ||
+                previous.mtimeMs !== summary.updatedAt.getTime() ||
+                previous.size !== summary.size ||
+                !existingByPath.has(summary.relativePath);
+            if (changed) {
+                changedPaths.add(summary.relativePath);
+            }
+        }
+        const hasDeletes = previousState
+            ? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
+            : false;
+        const manifestRecovery = await ensureSearchPackManifest(absoluteVaultPath);
+        if (changedPaths.size === 0 &&
+            !hasDeletes &&
+            existingIndexedDocuments.length === summaries.length &&
+            previousState != null &&
+            !fullReindex) {
+            const result = {
+                ...toIndexResult(existingIndexedDocuments),
+                elapsedMs: elapsedMs(),
+                changedDocumentCount: 0,
+                packs: {
+                    rebuilt: false,
+                    reason: manifestRecovery.repaired ? 'No changes detected; pack manifest repaired' : 'No changes detected'
+                }
+            };
+            emit('complete', 'skip', 'Index skipped: no changes detected', {
+                elapsedMs: result.elapsedMs,
+                manifestRepaired: manifestRecovery.repaired,
+                manifestRecoverySource: manifestRecovery.source
+            });
+            return result;
+        }
+        const changedSummaries = summaries.filter((summary) => changedPaths.has(summary.relativePath));
+        emit('parse', 'start', 'Parsing changed markdown files', {
+            changedFiles: changedSummaries.length
+        });
+        const changedDocumentsByPath = await readChangedDocuments(absoluteVaultPath, changedSummaries);
+        emit('parse', 'finish', 'Parse complete', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const documents = summaries.flatMap((summary) => {
+            const changed = changedDocumentsByPath.get(summary.relativePath);
+            if (changed) {
+                return [changed];
+            }
+            const existing = existingByPath.get(summary.relativePath);
+            return existing ? [existing.document] : [];
+        });
+        const titleMaps = createTitleMaps(documents);
+        emit('embed', 'start', 'Embedding changed chunks', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const changedIndexedDocuments = changedDocumentsByPath.size > 0
+            ? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
+            : [];
+        emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
+            changedIndexedDocuments: changedIndexedDocuments.length
+        });
+        const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
+        const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
+        const indexedDocuments = documents.map((document) => {
+            const changed = changedIndexedByPath.get(document.path);
+            if (changed) {
+                return changed;
+            }
+            const existing = existingByPath.get(document.path);
+            if (!existing) {
+                return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
+            }
+            return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
+        });
+        emit('persist', 'start', 'Persisting index');
         await index.saveDocuments(indexedDocuments);
-        try {
-            await buildSearchPacks(absoluteVaultPath, indexedDocuments);
+        emit('persist', 'finish', 'Index persisted', {
+            indexedDocuments: indexedDocuments.length
+        });
+        const existingPackManifest = manifestRecovery.repaired || manifestRecovery.source === 'not-needed';
+        const changedCount = changedPaths.size;
+        const documentCount = Math.max(indexedDocuments.length, 1);
+        const changeRatio = changedCount / documentCount;
+        const previousPendingPackChanges = previousState?.pendingPackChanges ?? 0;
+        const pendingPackChanges = previousPendingPackChanges + changedCount;
+        const shouldRebuildPacks = !existingPackManifest ||
+            fullReindex ||
+            graphLinkModelChanged ||
+            settingsChanged ||
+            packSettingsChanged ||
+            hasDeletes ||
+            changedCount >= 400 ||
+            changeRatio >= 0.04 ||
+            pendingPackChanges >= 1200;
+        let packResult;
+        const packReason = !existingPackManifest
+            ? 'Missing pack manifest'
+            : fullReindex
+                ? 'Full reindex requested'
+                : graphLinkModelChanged
+                    ? 'Graph link model changed'
+                    : manifestRecovery.repaired
+                        ? 'Pack manifest repaired from existing packs'
+                        : settingsChanged
+                            ? 'Index settings changed'
+                            : packSettingsChanged
+                                ? 'Search pack settings changed'
+                                : hasDeletes
+                                    ? 'Document deletions detected'
+                                    : changedCount >= 400
+                                        ? 'Changed file count threshold reached'
+                                        : changeRatio >= 0.04
+                                            ? 'Change ratio threshold reached'
+                                            : pendingPackChanges >= 1200
+                                                ? 'Pending pack changes threshold reached'
+                                                : 'Pack rebuild skipped';
+        if (shouldRebuildPacks) {
+            emit('packs', 'start', 'Rebuilding compressed search packs', {
+                reason: packReason
+            });
+            try {
+                packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
+                emit('packs', 'finish', 'Compressed packs rebuilt', {
+                    reason: packReason,
+                    packCount: packResult.packCount,
+                    recordCount: packResult.recordCount,
+                    durationMs: packResult.durationMs,
+                    compressionRatio: packResult.compression.ratio
+                });
+            }
+            catch {
+                // Pack generation is best-effort. The JSON index remains the primary path.
+                emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
+                    reason: packReason
+                });
+            }
         }
-        catch {
-            // Pack generation is best-effort. The JSON index remains the primary path.
+        else {
+            emit('packs', 'skip', 'Pack rebuild not required', {
+                reason: packReason
+            });
         }
-        return {
-            documentCount: indexedDocuments.length,
-            chunkCount: indexedDocuments.reduce((total, document) => total + document.chunks.length, 0),
-            linkCount: indexedDocuments.reduce((total, document) => total + document.links.length, 0)
+        const packsRebuilt = packResult != null;
+        const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
+        await writeIndexState(absoluteVaultPath, {
+            chunkSize: config.chunkSize,
+            embeddingProvider: config.embeddingProvider,
+            graphLinkModelVersion,
+            searchPackRowChunkSize: config.searchPack.rowChunkSize,
+            searchPackCompressionLevel: config.searchPack.compressionLevel,
+            searchPackUseDictionary: config.searchPack.useDictionary,
+            files: currentSnapshot,
+            pendingPackChanges: packsRebuilt ? 0 : pendingPackChanges
+        });
+        const result = {
+            ...toIndexResult(indexedDocuments),
+            elapsedMs: elapsedMs(),
+            changedDocumentCount: changedDocumentsByPath.size,
+            packs: {
+                rebuilt: packsRebuilt,
+                reason: packResultReason,
+                ...(packResult
+                    ? {
+                        packCount: packResult.packCount,
+                        recordCount: packResult.recordCount,
+                        durationMs: packResult.durationMs,
+                        compression: packResult.compression
+                    }
+                    : {})
+            }
         };
+        emit('complete', 'finish', 'Indexing complete', {
+            documentCount: result.documentCount,
+            chunkCount: result.chunkCount,
+            linkCount: result.linkCount,
+            elapsedMs: result.elapsedMs
+        });
+        return result;
     }
     finally {
         index.close();

package/dist/application/migrate-context-links.js

@@ -0,0 +1,79 @@
+import { readFile } from 'node:fs/promises';
+import { ensureVault, readMarkdownFileSummaries, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
+import { extractContextLinkWeights, extractWikiLinkWeights, hasContextLinksSection, parseMarkdownDocument } from '../domain/markdown.js';
+const defaultContextLinkLimit = 5;
+const normalizeTitle = (title) => title.trim().replace(/\.md$/i, '').toLowerCase();
+const formatPriority = (priority) => priority === 'normal' ? '' : ` priority: ${priority}`;
+const formatContextLinksSection = (links) => [
+    '## Context Links',
+    '',
+    ...links.map((link) => `- [[${link.title}]]${formatPriority(link.priority)}`)
+].join('\n');
+const appendContextLinksSection = (content, links) => `${content.trimEnd()}\n\n${formatContextLinksSection(links)}\n`;
+const selectContextLinkCandidates = (content, title, limit) => extractWikiLinkWeights(content)
+    .filter((link) => normalizeTitle(link.title) !== normalizeTitle(title))
+    .slice(0, limit)
+    .map((link) => ({
+    title: link.title,
+    priority: link.priority,
+    weight: link.weight
+}));
+export const migrateContextLinks = async (vaultPath, options = {}) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const limit = Math.max(1, Math.floor(options.limit ?? defaultContextLinkLimit));
+    const summaries = await readMarkdownFileSummaries(absoluteVaultPath);
+    const entries = [];
+    for (const summary of summaries) {
+        const content = await readFile(summary.absolutePath, 'utf8');
+        const document = parseMarkdownDocument({
+            absolutePath: summary.absolutePath,
+            vaultPath: absoluteVaultPath,
+            content,
+            createdAt: summary.createdAt,
+            updatedAt: summary.updatedAt
+        });
+        if (options.agentId && document.agentId !== options.agentId) {
+            continue;
+        }
+        if (hasContextLinksSection(content)) {
+            entries.push({
+                path: summary.relativePath,
+                title: document.title,
+                changed: false,
+                reason: 'already-has-context-links',
+                links: extractContextLinkWeights(content)
+            });
+            continue;
+        }
+        const links = selectContextLinkCandidates(content, document.title, limit);
+        if (links.length === 0) {
+            entries.push({
+                path: summary.relativePath,
+                title: document.title,
+                changed: false,
+                reason: 'no-link-candidates',
+                links
+            });
+            continue;
+        }
+        if (!options.dryRun) {
+            await writeMarkdownFile(vaultPath, summary.relativePath, appendContextLinksSection(content, links));
+        }
+        entries.push({
+            path: summary.relativePath,
+            title: document.title,
+            changed: true,
+            reason: 'added-context-links',
+            links
+        });
+    }
+    const changed = entries.filter((entry) => entry.changed).length;
+    return {
+        dryRun: options.dryRun === true,
+        scanned: entries.length,
+        changed,
+        skipped: entries.length - changed,
+        limit,
+        entries
+    };
+};

package/dist/application/offline-pack-backup.js

@@ -0,0 +1,44 @@
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { gzipSync } from 'node:zlib';
+import { ensureVault } from '../infrastructure/file-system-vault.js';
+const packsDirectory = (vaultPath) => join(vaultPath, '.brainlink', 'search-packs');
+const toSortedBackupFiles = async (vaultPath) => {
+    const directory = packsDirectory(vaultPath);
+    const names = await readdir(directory);
+    return names
+        .filter((name) => name.endsWith('.blpk') || name === 'manifest.json')
+        .sort((left, right) => left.localeCompare(right));
+};
+export const createOfflinePackBackup = async (input) => {
+    const vaultPath = await ensureVault(input.vaultPath);
+    const fileNames = await toSortedBackupFiles(vaultPath);
+    const files = [];
+    let inputBytes = 0;
+    for (const name of fileNames) {
+        const content = await readFile(join(packsDirectory(vaultPath), name));
+        inputBytes += content.byteLength;
+        files.push({
+            name,
+            contentB64: content.toString('base64')
+        });
+    }
+    const envelope = {
+        version: 1,
+        createdAt: new Date().toISOString(),
+        files
+    };
+    const serialized = Buffer.from(JSON.stringify(envelope), 'utf8');
+    const compressed = gzipSync(serialized, { level: 9 });
+    await mkdir(dirname(input.outputPath), { recursive: true });
+    await writeFile(input.outputPath, compressed);
+    const safeInput = Math.max(inputBytes, 1);
+    return {
+        outputPath: input.outputPath,
+        fileCount: files.length,
+        inputBytes,
+        outputBytes: compressed.byteLength,
+        ratio: compressed.byteLength / safeInput,
+        savedBytes: Math.max(inputBytes - compressed.byteLength, 0)
+    };
+};

package/dist/application/search-graph-node-ids.js

@@ -1,10 +1,70 @@
+import { stat } from 'node:fs/promises';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openFileIndex } from '../infrastructure/file-index.js';
-export const searchGraphNodeIds = async (vaultPath, query, limit, agentId) => {
+import { indexStoragePath, openFileIndex } from '../infrastructure/file-index.js';
+import { getGraphLayout } from './get-graph-layout.js';
+const graphSearchCacheTtlMs = 20_000;
+const graphSearchCacheMaxEntries = 120;
+const graphSearchCache = new Map();
+const readIndexSignature = async (vaultPath) => {
+    try {
+        const info = await stat(indexStoragePath(vaultPath));
+        return `${Math.floor(info.mtimeMs)}:${info.size}`;
+    }
+    catch {
+        return '0:0';
+    }
+};
+const cacheKey = (vaultPath, query, limit, agentId, context) => JSON.stringify({
+    vaultPath,
+    query: query.trim().toLowerCase(),
+    limit,
+    agentId: agentId?.trim().toLowerCase() ?? '*',
+    context: context?.trim().toLowerCase() ?? '*'
+});
+const readCached = (key, indexSignature) => {
+    const entry = graphSearchCache.get(key);
+    if (!entry) {
+        return undefined;
+    }
+    const fresh = Date.now() - entry.createdAt <= graphSearchCacheTtlMs && entry.indexSignature === indexSignature;
+    if (!fresh) {
+        graphSearchCache.delete(key);
+        return undefined;
+    }
+    return entry.nodeIds;
+};
+const writeCached = (key, entry) => {
+    graphSearchCache.set(key, entry);
+    if (graphSearchCache.size <= graphSearchCacheMaxEntries) {
+        return;
+    }
+    const overflow = graphSearchCache.size - graphSearchCacheMaxEntries;
+    Array.from(graphSearchCache.keys()).slice(0, overflow).forEach((cacheKey) => graphSearchCache.delete(cacheKey));
+};
+export const searchGraphNodeIds = async (vaultPath, query, limit, agentId, context) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
+    const indexSignature = await readIndexSignature(absoluteVaultPath);
+    const key = cacheKey(absoluteVaultPath, query, limit, agentId, context);
+    const cached = readCached(key, indexSignature);
+    if (cached) {
+        return cached;
+    }
+    const contextNodeIds = context
+        ? new Set((await getGraphLayout(absoluteVaultPath, { agentId, context })).layout.nodes.map((node) => node.id))
+        : new Set();
     const index = openFileIndex(absoluteVaultPath);
     try {
-        return await index.searchGraphNodeIds(query, limit, agentId);
+        const searchLimit = context ? Math.max(limit, 5000) : limit;
+        const foundNodeIds = await index.searchGraphNodeIds(query, searchLimit, agentId);
+        const nodeIds = context
+            ? foundNodeIds.filter((nodeId) => contextNodeIds.has(nodeId)).slice(0, limit)
+            : foundNodeIds;
+        writeCached(key, {
+            createdAt: Date.now(),
+            indexSignature,
+            nodeIds
+        });
+        return nodeIds;
     }
     finally {
         index.close();