@andespindola/brainlink 1.0.5 → 1.0.6

package/dist/application/frontend/client-render-worker-js.js

@@ -58,7 +58,7 @@ const defaultTheme = {
   ],
   edge: [0.23, 0.31, 0.42, 0.18],
   edgeHeavy: [0.23, 0.31, 0.42, 0.34],
-  clear: [0.96, 0.97, 0.98, 1]
+  clear: [0.031, 0.075, 0.114, 1]
 }
 
 const theme = { ...defaultTheme }

package/dist/application/index-vault-phases.js

@@ -0,0 +1,189 @@
+import { createIndexedDocument, graphLinkModelVersion } from '../domain/markdown.js';
+import { sharedAgentId } from '../domain/agents.js';
+import { createEmbeddingBuckets, createEmbeddingProvider } from '../domain/embeddings.js';
+import { buildSearchPacks, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
+const toTitleKey = (title) => title.toLowerCase();
+const appendTitleEntry = (map, document) => {
+    const key = toTitleKey(document.title);
+    if (!map.has(key)) {
+        map.set(key, {
+            id: document.id,
+            path: document.path
+        });
+    }
+    return map;
+};
+export const createTitleMaps = (documents) => [...documents]
+    .sort((left, right) => left.path.localeCompare(right.path))
+    .reduce((state, document) => {
+    const agentMap = state.byAgent.get(document.agentId) ?? new Map();
+    appendTitleEntry(agentMap, document);
+    state.byAgent.set(document.agentId, agentMap);
+    if (document.agentId === sharedAgentId) {
+        appendTitleEntry(state.shared, document);
+    }
+    return state;
+}, {
+    shared: new Map(),
+    byAgent: new Map()
+});
+const createScopedTitleResolver = (document, titleMaps) => ({
+    get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
+});
+const embedIndexedDocuments = async (documents, providerName) => {
+    if (documents.length === 0) {
+        return documents;
+    }
+    const provider = createEmbeddingProvider(providerName);
+    const chunks = documents.flatMap((document) => document.chunks);
+    const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
+    const embeddingByChunkId = new Map(chunks.map((chunk, index) => [chunk.id, embeddings[index] ?? []]));
+    return documents.map((indexedDocument) => ({
+        ...indexedDocument,
+        chunks: indexedDocument.chunks.map((chunk) => {
+            const embedding = embeddingByChunkId.get(chunk.id) ?? [];
+            return {
+                ...chunk,
+                embeddingProvider: provider.name,
+                embedding,
+                buckets: embedding.length > 0 ? createEmbeddingBuckets(embedding) : []
+            };
+        })
+    }));
+};
+const relinkIndexedDocument = (indexedDocument, titleMaps) => {
+    const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
+    return {
+        ...indexedDocument,
+        links: indexedDocument.links
+            .map((link) => ({
+            ...link,
+            toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
+        }))
+            .filter((link) => link.toDocumentId !== indexedDocument.document.id)
+    };
+};
+export const detectChanges = (params) => {
+    const { summaries, previousState, config, fullReindex, existingByPath, previousSnapshotMap, currentSnapshotMap } = params;
+    const graphLinkModelChanged = previousState != null &&
+        previousState.graphLinkModelVersion !== graphLinkModelVersion;
+    const fullSourceReindex = fullReindex || graphLinkModelChanged;
+    const settingsChanged = previousState == null ||
+        previousState.chunkSize !== config.chunkSize ||
+        previousState.embeddingProvider !== config.embeddingProvider ||
+        graphLinkModelChanged;
+    const packSettingsChanged = previousState == null ||
+        previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
+        previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
+        previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
+    const changedPaths = new Set();
+    for (let index = 0; index < summaries.length; index += 1) {
+        const summary = summaries[index];
+        const previous = previousSnapshotMap.get(summary.relativePath);
+        const changed = fullSourceReindex ||
+            settingsChanged ||
+            previous == null ||
+            previous.mtimeMs !== summary.updatedAt.getTime() ||
+            previous.size !== summary.size ||
+            !existingByPath.has(summary.relativePath);
+        if (changed) {
+            changedPaths.add(summary.relativePath);
+        }
+    }
+    const hasDeletes = previousState
+        ? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
+        : false;
+    return { graphLinkModelChanged, settingsChanged, packSettingsChanged, changedPaths, hasDeletes };
+};
+export const decidePackRebuild = (params) => {
+    const { existingPackManifest, fullReindex, graphLinkModelChanged, manifestRepaired, settingsChanged, packSettingsChanged, hasDeletes, changedCount, changeRatio, pendingPackChanges } = params;
+    const shouldRebuild = !existingPackManifest ||
+        fullReindex ||
+        graphLinkModelChanged ||
+        settingsChanged ||
+        packSettingsChanged ||
+        hasDeletes ||
+        changedCount >= 400 ||
+        changeRatio >= 0.04 ||
+        pendingPackChanges >= 1200;
+    const reason = !existingPackManifest
+        ? 'Missing pack manifest'
+        : fullReindex
+            ? 'Full reindex requested'
+            : graphLinkModelChanged
+                ? 'Graph link model changed'
+                : manifestRepaired
+                    ? 'Pack manifest repaired from existing packs'
+                    : settingsChanged
+                        ? 'Index settings changed'
+                        : packSettingsChanged
+                            ? 'Search pack settings changed'
+                            : hasDeletes
+                                ? 'Document deletions detected'
+                                : changedCount >= 400
+                                    ? 'Changed file count threshold reached'
+                                    : changeRatio >= 0.04
+                                        ? 'Change ratio threshold reached'
+                                        : pendingPackChanges >= 1200
+                                            ? 'Pending pack changes threshold reached'
+                                            : 'Pack rebuild skipped';
+    return { shouldRebuild, reason };
+};
+export const embedChangedDocuments = async (params) => {
+    const { changedDocumentsByPath, titleMaps, config, emit } = params;
+    emit('embed', 'start', 'Embedding changed chunks', {
+        changedDocuments: changedDocumentsByPath.size
+    });
+    const changedIndexedDocuments = changedDocumentsByPath.size > 0
+        ? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
+        : [];
+    emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
+        changedIndexedDocuments: changedIndexedDocuments.length
+    });
+    return changedIndexedDocuments;
+};
+export const assembleIndexedDocuments = (params) => {
+    const { documents, changedIndexedDocuments, existingByPath, titleMaps, config, needsRelink } = params;
+    const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
+    return documents.map((document) => {
+        const changed = changedIndexedByPath.get(document.path);
+        if (changed) {
+            return changed;
+        }
+        const existing = existingByPath.get(document.path);
+        if (!existing) {
+            return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
+        }
+        return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
+    });
+};
+export const rebuildSearchPacksIfNeeded = async (params) => {
+    const { shouldRebuild, reason, absoluteVaultPath, indexedDocuments, config, emit } = params;
+    if (!shouldRebuild) {
+        emit('packs', 'skip', 'Pack rebuild not required', {
+            reason
+        });
+        return undefined;
+    }
+    emit('packs', 'start', 'Rebuilding compressed search packs', {
+        reason
+    });
+    try {
+        const packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
+        emit('packs', 'finish', 'Compressed packs rebuilt', {
+            reason,
+            packCount: packResult.packCount,
+            recordCount: packResult.recordCount,
+            durationMs: packResult.durationMs,
+            compressionRatio: packResult.compression.ratio
+        });
+        return packResult;
+    }
+    catch {
+        // Pack generation is best-effort. The JSON index remains the primary path.
+        emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
+            reason
+        });
+        return undefined;
+    }
+};

package/dist/application/index-vault.js

@@ -1,69 +1,11 @@
 import { readFile } from 'node:fs/promises';
-import { createIndexedDocument, graphLinkModelVersion, parseMarkdownDocument } from '../domain/markdown.js';
-import { sharedAgentId } from '../domain/agents.js';
-import { createEmbeddingProvider } from '../domain/embeddings.js';
+import { graphLinkModelVersion, parseMarkdownDocument } from '../domain/markdown.js';
 import { loadBrainlinkConfig } from '../infrastructure/config.js';
 import { ensureVault, readMarkdownFileSummaries } from '../infrastructure/file-system-vault.js';
 import { readIndexState, writeIndexState } from '../infrastructure/index-state.js';
-import { buildSearchPacks, ensureSearchPackManifest, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
+import { ensureSearchPackManifest } from '../infrastructure/search-packs.js';
 import { openFileIndex } from '../infrastructure/file-index.js';
-const toTitleKey = (title) => title.toLowerCase();
-const appendTitleEntry = (map, document) => {
-    const key = toTitleKey(document.title);
-    if (!map.has(key)) {
-        map.set(key, {
-            id: document.id,
-            path: document.path
-        });
-    }
-    return map;
-};
-const createTitleMaps = (documents) => [...documents]
-    .sort((left, right) => left.path.localeCompare(right.path))
-    .reduce((state, document) => {
-    const agentMap = state.byAgent.get(document.agentId) ?? new Map();
-    appendTitleEntry(agentMap, document);
-    state.byAgent.set(document.agentId, agentMap);
-    if (document.agentId === sharedAgentId) {
-        appendTitleEntry(state.shared, document);
-    }
-    return state;
-}, {
-    shared: new Map(),
-    byAgent: new Map()
-});
-const createScopedTitleResolver = (document, titleMaps) => ({
-    get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
-});
-const embedIndexedDocuments = async (documents, providerName) => {
-    if (documents.length === 0) {
-        return documents;
-    }
-    const provider = createEmbeddingProvider(providerName);
-    const chunks = documents.flatMap((document) => document.chunks);
-    const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
-    const embeddingByChunkId = new Map(chunks.map((chunk, index) => [chunk.id, embeddings[index] ?? []]));
-    return documents.map((indexedDocument) => ({
-        ...indexedDocument,
-        chunks: indexedDocument.chunks.map((chunk) => ({
-            ...chunk,
-            embeddingProvider: provider.name,
-            embedding: embeddingByChunkId.get(chunk.id) ?? []
-        }))
-    }));
-};
-const relinkIndexedDocument = (indexedDocument, titleMaps) => {
-    const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
-    return {
-        ...indexedDocument,
-        links: indexedDocument.links
-            .map((link) => ({
-            ...link,
-            toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
-        }))
-            .filter((link) => link.toDocumentId !== indexedDocument.document.id)
-    };
-};
+import { assembleIndexedDocuments, createTitleMaps, decidePackRebuild, detectChanges, embedChangedDocuments, rebuildSearchPacksIfNeeded } from './index-vault-phases.js';
 const toIndexResult = (documents) => ({
     documentCount: documents.length,
     chunkCount: documents.reduce((total, document) => total + document.chunks.length, 0),
@@ -121,34 +63,15 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
         const currentSnapshot = toSnapshot(summaries);
         const currentSnapshotMap = createSnapshotMap(currentSnapshot);
         const previousSnapshotMap = createSnapshotMap(previousState?.files ?? []);
-        const graphLinkModelChanged = previousState != null &&
-            previousState.graphLinkModelVersion !== graphLinkModelVersion;
-        const fullSourceReindex = fullReindex || graphLinkModelChanged;
-        const settingsChanged = previousState == null ||
-            previousState.chunkSize !== config.chunkSize ||
-            previousState.embeddingProvider !== config.embeddingProvider ||
-            graphLinkModelChanged;
-        const packSettingsChanged = previousState == null ||
-            previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
-            previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
-            previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
-        const changedPaths = new Set();
-        for (let index = 0; index < summaries.length; index += 1) {
-            const summary = summaries[index];
-            const previous = previousSnapshotMap.get(summary.relativePath);
-            const changed = fullSourceReindex ||
-                settingsChanged ||
-                previous == null ||
-                previous.mtimeMs !== summary.updatedAt.getTime() ||
-                previous.size !== summary.size ||
-                !existingByPath.has(summary.relativePath);
-            if (changed) {
-                changedPaths.add(summary.relativePath);
-            }
-        }
-        const hasDeletes = previousState
-            ? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
-            : false;
+        const { graphLinkModelChanged, settingsChanged, packSettingsChanged, changedPaths, hasDeletes } = detectChanges({
+            summaries,
+            previousState,
+            config,
+            fullReindex,
+            existingByPath,
+            previousSnapshotMap,
+            currentSnapshotMap
+        });
         const manifestRecovery = await ensureSearchPackManifest(absoluteVaultPath);
         if (changedPaths.size === 0 &&
             !hasDeletes &&
@@ -188,27 +111,20 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
             return existing ? [existing.document] : [];
         });
         const titleMaps = createTitleMaps(documents);
-        emit('embed', 'start', 'Embedding changed chunks', {
-            changedDocuments: changedDocumentsByPath.size
-        });
-        const changedIndexedDocuments = changedDocumentsByPath.size > 0
-            ? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
-            : [];
-        emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
-            changedIndexedDocuments: changedIndexedDocuments.length
+        const changedIndexedDocuments = await embedChangedDocuments({
+            changedDocumentsByPath,
+            titleMaps,
+            config,
+            emit
         });
-        const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
         const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
-        const indexedDocuments = documents.map((document) => {
-            const changed = changedIndexedByPath.get(document.path);
-            if (changed) {
-                return changed;
-            }
-            const existing = existingByPath.get(document.path);
-            if (!existing) {
-                return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
-            }
-            return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
+        const indexedDocuments = assembleIndexedDocuments({
+            documents,
+            changedIndexedDocuments,
+            existingByPath,
+            titleMaps,
+            config,
+            needsRelink
         });
         emit('persist', 'start', 'Persisting index');
         await index.saveDocuments(indexedDocuments);
@@ -221,63 +137,26 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
         const changeRatio = changedCount / documentCount;
         const previousPendingPackChanges = previousState?.pendingPackChanges ?? 0;
         const pendingPackChanges = previousPendingPackChanges + changedCount;
-        const shouldRebuildPacks = !existingPackManifest ||
-            fullReindex ||
-            graphLinkModelChanged ||
-            settingsChanged ||
-            packSettingsChanged ||
-            hasDeletes ||
-            changedCount >= 400 ||
-            changeRatio >= 0.04 ||
-            pendingPackChanges >= 1200;
-        let packResult;
-        const packReason = !existingPackManifest
-            ? 'Missing pack manifest'
-            : fullReindex
-                ? 'Full reindex requested'
-                : graphLinkModelChanged
-                    ? 'Graph link model changed'
-                    : manifestRecovery.repaired
-                        ? 'Pack manifest repaired from existing packs'
-                        : settingsChanged
-                            ? 'Index settings changed'
-                            : packSettingsChanged
-                                ? 'Search pack settings changed'
-                                : hasDeletes
-                                    ? 'Document deletions detected'
-                                    : changedCount >= 400
-                                        ? 'Changed file count threshold reached'
-                                        : changeRatio >= 0.04
-                                            ? 'Change ratio threshold reached'
-                                            : pendingPackChanges >= 1200
-                                                ? 'Pending pack changes threshold reached'
-                                                : 'Pack rebuild skipped';
-        if (shouldRebuildPacks) {
-            emit('packs', 'start', 'Rebuilding compressed search packs', {
-                reason: packReason
-            });
-            try {
-                packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
-                emit('packs', 'finish', 'Compressed packs rebuilt', {
-                    reason: packReason,
-                    packCount: packResult.packCount,
-                    recordCount: packResult.recordCount,
-                    durationMs: packResult.durationMs,
-                    compressionRatio: packResult.compression.ratio
-                });
-            }
-            catch {
-                // Pack generation is best-effort. The JSON index remains the primary path.
-                emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
-                    reason: packReason
-                });
-            }
-        }
-        else {
-            emit('packs', 'skip', 'Pack rebuild not required', {
-                reason: packReason
-            });
-        }
+        const { shouldRebuild: shouldRebuildPacks, reason: packReason } = decidePackRebuild({
+            existingPackManifest,
+            fullReindex,
+            graphLinkModelChanged,
+            manifestRepaired: manifestRecovery.repaired,
+            settingsChanged,
+            packSettingsChanged,
+            hasDeletes,
+            changedCount,
+            changeRatio,
+            pendingPackChanges
+        });
+        const packResult = await rebuildSearchPacksIfNeeded({
+            shouldRebuild: shouldRebuildPacks,
+            reason: packReason,
+            absoluteVaultPath,
+            indexedDocuments,
+            config,
+            emit
+        });
         const packsRebuilt = packResult != null;
         const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
         await writeIndexState(absoluteVaultPath, {

package/dist/cli/commands/write/dedupe-commands.js

@@ -0,0 +1,59 @@
+import { resolveDuplicateNotes, scanDuplicateNotes } from '../../../application/dedupe-notes.js';
+import { parsePositiveInteger, print, resolveOptions } from '../../runtime.js';
+import { parseScore } from './shared.js';
+export const registerDedupeCommands = (program) => {
+    program
+        .command('dedupe')
+        .option('-v, --vault <vault>', 'vault directory')
+        .option('-a, --agent <agent>', 'agent memory namespace')
+        .option('-l, --limit <limit>', 'maximum duplicate candidate pairs')
+        .option('--min-score <score>', 'minimum semantic similarity score between 0 and 1', '0.92')
+        .option('--no-semantic', 'disable semantic duplicate detection and keep exact-content matching only')
+        .option('--json', 'print machine-readable JSON')
+        .description('detect possible duplicate notes with exact hash and semantic similarity scores')
+        .action(async (options) => {
+        const resolved = await resolveOptions(options);
+        const duplicates = await scanDuplicateNotes(resolved.vault, {
+            agentId: resolved.agent,
+            limit: parsePositiveInteger(options.limit ?? '25', 25),
+            minSemanticScore: parseScore(options.minScore, 0.92),
+            includeSemantic: options.semantic !== false
+        });
+        print(options.json, { vault: resolved.vault, agent: resolved.agent, duplicates }, () => {
+            if (duplicates.length === 0) {
+                return 'No possible duplicates found.';
+            }
+            return duplicates
+                .map((item, index) => `${index + 1}. [${item.kind}] score=${item.score.toFixed(4)} ${item.left.path} <-> ${item.right.path} (${item.reason})`)
+                .join('\n');
+        });
+    });
+    program
+        .command('dedupe-resolve')
+        .option('-v, --vault <vault>', 'vault directory')
+        .option('--left <path>', 'left note relative path from dedupe result')
+        .option('--right <path>', 'right note relative path from dedupe result')
+        .option('--action <action>', 'resolution action: merge, link or ignore')
+        .option('--no-auto-index', 'skip reindex after duplicate resolution')
+        .option('--json', 'print machine-readable JSON')
+        .description('resolve a duplicate candidate with merge, link or ignore')
+        .action(async (options) => {
+        const resolved = await resolveOptions(options);
+        if (!options.left || !options.right) {
+            throw new Error('Use --left <path> and --right <path> to resolve a duplicate pair.');
+        }
+        if (options.action !== 'merge' && options.action !== 'link' && options.action !== 'ignore') {
+            throw new Error('Use --action merge|link|ignore.');
+        }
+        const result = await resolveDuplicateNotes(resolved.vault, {
+            leftPath: options.left,
+            rightPath: options.right,
+            action: options.action,
+            autoIndex: options.autoIndex !== false
+        });
+        print(options.json, {
+            vault: resolved.vault,
+            ...result
+        }, () => `Resolved duplicate (${result.action}) for ${result.leftPath} <-> ${result.rightPath}`);
+    });
+};