@andespindola/brainlink 0.1.0-beta.10 → 0.1.0-beta.11

package/README.md

@@ -58,6 +58,7 @@ LLMs do not have infinite context. Brainlink gives agents an external memory lay
 
 Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable index.
 Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup`. If the main SQLite file is corrupted, Brainlink automatically restores from snapshot (or recreates a clean index when no snapshot exists).
+After each index run, Brainlink also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`. If SQLite is unavailable, search falls back to these packs automatically.
 
 ## Features
 
@@ -563,6 +564,7 @@ The graph UI shows:
 - graph controls for zoom in, zoom out, fit visible nodes and reset-to-fit-all
 - wheel zoom anchored to cursor position for faster navigation in large graphs
 - floating graph totals (notes, links, tags) below the Brainlink title
+- large-graph rendering safeguards (edge draw caps, lower redraw rate, zoom-aware interaction)
 
 The server indexes before starting by default. Use `--no-index` to skip that step:
 

package/dist/application/frontend/client-js.js

@@ -1,9 +1,14 @@
 export const createClientJs = () => `const canvas = document.getElementById('graph')
 const ctx = canvas.getContext('2d')
+const largeGraphNodeThreshold = 4000
+const largeGraphEdgeRenderLimit = 16000
 const state = {
   graph: { nodes: [], edges: [] },
   nodes: [],
   edges: [],
+  visibleNodes: [],
+  visibleEdges: [],
+  nodeDegrees: new Map(),
   selected: null,
   hovered: null,
   query: '',
@@ -102,11 +107,18 @@ const filteredNodes = () => {
   return localFilteredNodes(query)
 }
 
-const visibleIds = () => new Set(filteredNodes().map(node => node.id))
-
-const visibleEdges = () => {
-  const ids = visibleIds()
-  return state.edges.filter(edge => ids.has(edge.source) && edge.target && ids.has(edge.target))
+const recomputeVisibility = () => {
+  const nodes = filteredNodes()
+  const ids = new Set(nodes.map(node => node.id))
+  const edges = state.edges.filter(edge => ids.has(edge.source) && edge.target && ids.has(edge.target))
+  const limitedEdges = state.nodes.length > largeGraphNodeThreshold
+    ? [...edges]
+      .sort((left, right) => edgeWeight(right) - edgeWeight(left))
+      .slice(0, largeGraphEdgeRenderLimit)
+    : edges
+
+  state.visibleNodes = nodes
+  state.visibleEdges = limitedEdges
 }
 
 const edgeWeight = edge => Number.isFinite(edge.weight) ? Math.max(1, edge.weight) : 1
@@ -205,6 +217,7 @@ const resetContentFilter = () => {
     token: state.contentFilter.token + 1,
     timer: null
   }
+  recomputeVisibility()
 }
 
 const syncContentFilter = async (query, token) => {
@@ -228,6 +241,7 @@ const syncContentFilter = async (query, token) => {
 
   state.contentFilter.query = query
   state.contentFilter.ids = new Set(nodeIds)
+  recomputeVisibility()
 }
 
 const scheduleContentFilterSync = () => {
@@ -253,9 +267,11 @@ const scheduleContentFilterSync = () => {
 }
 
 const tick = delta => {
-  const nodes = filteredNodes()
-  const ids = new Set(nodes.map(node => node.id))
-  const edges = state.edges.filter(edge => ids.has(edge.source) && edge.target && ids.has(edge.target))
+  const nodes = state.visibleNodes
+  const edges = state.visibleEdges
+  if (nodes.length > 1200) {
+    return
+  }
   const strength = Math.min(delta / 16, 2)
 
   edges.forEach(edge => {
@@ -314,7 +330,11 @@ const worldPoint = event => {
 }
 
 const hitNode = point => {
-  const nodes = filteredNodes()
+  if (state.nodes.length > largeGraphNodeThreshold && state.transform.scale < 0.55) {
+    return null
+  }
+
+  const nodes = state.visibleNodes
   for (let index = nodes.length - 1; index >= 0; index -= 1) {
     const node = nodes[index]
     const radius = nodeRadius(node)
@@ -324,13 +344,18 @@ const hitNode = point => {
 }
 
 const nodeRadius = node => {
-  const degree = state.edges.filter(edge => edge.source === node.id || edge.target === node.id).length
+  const degree = state.nodeDegrees.get(node.id) ?? 0
   return 9 + Math.min(degree, 8) * 1.6
 }
 
 const render = now => {
   const delta = now - state.last
   state.last = now
+  const minFrameIntervalMs = state.nodes.length > largeGraphNodeThreshold ? 180 : 16
+  if (delta < minFrameIntervalMs) {
+    requestAnimationFrame(render)
+    return
+  }
   const rect = canvas.getBoundingClientRect()
   const width = Math.max(rect.width, 320)
   const height = Math.max(rect.height, 320)
@@ -347,7 +372,10 @@ const render = now => {
   ctx.translate(state.transform.x, state.transform.y)
   ctx.scale(state.transform.scale, state.transform.scale)
 
-  visibleEdges().forEach(edge => {
+  tick(delta)
+  const drawEdges = !(state.nodes.length > largeGraphNodeThreshold && state.transform.scale < 0.22)
+  if (drawEdges) {
+    state.visibleEdges.forEach(edge => {
     const selectedEdge = state.selected && (edge.source === state.selected.id || edge.target === state.selected.id)
     ctx.beginPath()
     ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y)
@@ -355,9 +383,10 @@ const render = now => {
     ctx.strokeStyle = selectedEdge ? graphTheme.edgeActive : graphTheme.edge
     ctx.lineWidth = (selectedEdge ? 1.8 : 1) + Math.min(edgeWeight(edge) - 1, 8) * 0.22
     ctx.stroke()
-  })
+    })
+  }
 
-  filteredNodes().forEach(node => {
+  state.visibleNodes.forEach(node => {
     const radius = nodeRadius(node)
     const isSelected = state.selected?.id === node.id
     const isHovered = state.hovered?.id === node.id
@@ -373,7 +402,11 @@ const render = now => {
     ctx.strokeStyle = isSelected ? graphTheme.nodeStrokeActive : graphTheme.nodeStroke
     ctx.stroke()
 
-    if (isSelected || isHovered || state.transform.scale > 1.18 || state.nodes.length <= 25) {
+    const shouldDrawLabels =
+      isSelected ||
+      isHovered ||
+      (state.nodes.length <= largeGraphNodeThreshold && (state.transform.scale > 1.18 || state.nodes.length <= 25))
+    if (shouldDrawLabels) {
       ctx.fillStyle = graphTheme.label
       ctx.font = '12px Inter, system-ui, sans-serif'
       ctx.textAlign = 'center'
@@ -483,6 +516,7 @@ const bindEvents = () => {
   window.addEventListener('resize', resize)
   elements.search.addEventListener('input', event => {
     state.query = event.target.value
+    recomputeVisibility()
     scheduleContentFilterSync()
   })
   elements.agent.addEventListener('change', event => {
@@ -490,6 +524,7 @@ const bindEvents = () => {
     state.selected = null
     state.nodeDetails = new Map()
     resetContentFilter()
+    recomputeVisibility()
     scheduleContentFilterSync()
     loadGraph({ reset: true }).catch(error => {
       console.error(error)
@@ -504,9 +539,13 @@ const bindEvents = () => {
     zoomAtPoint(Math.max(rect.width, 320) / 2, Math.max(rect.height, 320) / 2, 0.84)
   })
   if (elements.fit) {
-    elements.fit.addEventListener('click', () => fitView({ useFiltered: true }))
+    elements.fit.addEventListener('click', () => {
+      fitView({ useFiltered: true })
+    })
   }
-  elements.reset.addEventListener('click', resetView)
+  elements.reset.addEventListener('click', () => {
+    resetView()
+  })
   elements.contentClose.addEventListener('click', () => elements.contentDialog.close())
   elements.contentDialog.addEventListener('click', event => {
     const target = event.target
@@ -602,8 +641,16 @@ const loadGraph = async (options = { reset: false }) => {
   state.graph = graph
   state.nodes = layout.nodes
   state.edges = layout.edges
+  state.nodeDegrees = state.edges.reduce((degrees, edge) => {
+    degrees.set(edge.source, (degrees.get(edge.source) ?? 0) + edgeWeight(edge))
+    if (edge.target) {
+      degrees.set(edge.target, (degrees.get(edge.target) ?? 0) + edgeWeight(edge))
+    }
+    return degrees
+  }, new Map())
   state.nodeDetails = new Map()
   resetContentFilter()
+  recomputeVisibility()
   scheduleContentFilterSync()
   const tags = new Set(graph.nodes.flatMap(node => node.tags))
   setGraphStatus(state.agentId + ' · ' + graph.nodes.length + ' notes · ' + graph.edges.length + ' links · live')

package/dist/application/get-graph-layout.js

@@ -1,7 +1,18 @@
 import { createHash } from 'node:crypto';
+import { stat } from 'node:fs/promises';
+import { join } from 'node:path';
 import { createCauliflowerGraphLayout } from '../domain/graph-layout.js';
 import { getGraphSummary } from './get-graph-summary.js';
 const graphLayoutCache = new Map();
+const readDatabaseSignature = async (vaultPath) => {
+    try {
+        const info = await stat(join(vaultPath, '.brainlink', 'brainlink.db'));
+        return `${Math.floor(info.mtimeMs)}:${info.size}`;
+    }
+    catch {
+        return '0:0';
+    }
+};
 const createGraphSignature = (graph) => {
     const nodesSignature = graph.nodes.map((node) => `${node.id}|${node.agentId}|${node.title}|${node.path}`).join('\n');
     const edgesSignature = graph.edges
@@ -12,18 +23,19 @@ const createGraphSignature = (graph) => {
         .digest('hex');
 };
 export const getGraphLayout = async (vaultPath, agentId) => {
-    const graph = await getGraphSummary(vaultPath, agentId);
-    const signature = createGraphSignature(graph);
+    const databaseSignature = await readDatabaseSignature(vaultPath);
     const cacheKey = `${vaultPath}:${agentId ?? ''}`;
     const cached = graphLayoutCache.get(cacheKey);
-    if (cached?.signature === signature) {
+    if (cached?.databaseSignature === databaseSignature) {
         return {
-            signature,
+            signature: cached.signature,
             layout: cached.layout
         };
     }
+    const graph = await getGraphSummary(vaultPath, agentId);
+    const signature = createGraphSignature(graph);
     const layout = createCauliflowerGraphLayout(graph);
-    graphLayoutCache.set(cacheKey, { signature, layout });
+    graphLayoutCache.set(cacheKey, { databaseSignature, signature, layout });
     return {
         signature,
         layout

package/dist/application/index-vault.js

@@ -3,6 +3,7 @@ import { sharedAgentId } from '../domain/agents.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig } from '../infrastructure/config.js';
 import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
+import { buildSearchPacks } from '../infrastructure/search-packs.js';
 import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
 const toTitleKey = (title) => title.toLowerCase();
 const appendTitleEntry = (map, document) => {
@@ -63,6 +64,12 @@ export const indexVault = async (vaultPath) => {
     try {
         index.reset();
         index.saveDocuments(indexedDocuments);
+        try {
+            await buildSearchPacks(absoluteVaultPath, indexedDocuments);
+        }
+        catch {
+            // Pack generation is best-effort. SQLite index remains the primary path.
+        }
         return {
             documentCount: indexedDocuments.length,
             chunkCount: indexedDocuments.reduce((total, document) => total + document.chunks.length, 0),

package/dist/application/search-knowledge.js

@@ -1,6 +1,7 @@
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
+import { searchInPacks } from '../infrastructure/search-packs.js';
 import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
@@ -55,20 +56,34 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const provider = createEmbeddingProvider(config.embeddingProvider);
     const shouldEmbedQuery = searchMode !== 'fts' && provider.name !== 'none';
     const queryEmbedding = shouldEmbedQuery ? (await provider.embed([query]))[0] ?? [] : [];
-    const index = openSqliteIndex(absoluteVaultPath);
     try {
-        const results = index.search(query, limit, agentId, searchMode, queryEmbedding);
+        const index = openSqliteIndex(absoluteVaultPath);
+        try {
+            const results = index.search(query, limit, agentId, searchMode, queryEmbedding);
+            if (cacheKey) {
+                cacheSet({
+                    key: cacheKey,
+                    createdAt: Date.now(),
+                    indexMtimeMs,
+                    results
+                });
+            }
+            return results;
+        }
+        finally {
+            index.close();
+        }
+    }
+    catch {
+        const fallbackResults = await searchInPacks(absoluteVaultPath, query, limit, agentId);
         if (cacheKey) {
             cacheSet({
                 key: cacheKey,
                 createdAt: Date.now(),
                 indexMtimeMs,
-                results
+                results: fallbackResults
             });
         }
-        return results;
-    }
-    finally {
-        index.close();
+        return fallbackResults;
     }
 };

package/dist/infrastructure/search-packs.js

@@ -0,0 +1,151 @@
+import { gunzipSync, gzipSync } from 'node:zlib';
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+const packsDirectoryName = 'search-packs';
+const manifestFileName = 'manifest.json';
+const rowChunkSize = 5_000;
+const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
+const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
+const parseRowsFromPack = (content) => gunzipSync(content)
+    .toString('utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .map((line) => JSON.parse(line));
+const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
+    documentId: document.document.id,
+    agentId: document.document.agentId,
+    title: document.document.title,
+    path: document.document.path,
+    chunkId: chunk.id,
+    content: chunk.content,
+    tags: document.document.tags
+})));
+const writeManifest = async (vaultPath, manifest) => {
+    await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
+};
+const chunkRows = (rows, size) => {
+    const chunks = [];
+    for (let index = 0; index < rows.length; index += size) {
+        chunks.push(rows.slice(index, index + size));
+    }
+    return chunks;
+};
+const normalizeToken = (value) => value
+    .normalize('NFKD')
+    .replace(/\p{Diacritic}/gu, '')
+    .toLowerCase();
+const tokenize = (query) => query
+    .match(queryTokenPattern)
+    ?.map(normalizeToken)
+    .filter((token) => token.length > 1) ?? [];
+const countOccurrences = (text, token) => {
+    let hits = 0;
+    let start = 0;
+    while (start < text.length) {
+        const index = text.indexOf(token, start);
+        if (index < 0) {
+            break;
+        }
+        hits += 1;
+        start = index + token.length;
+    }
+    return hits;
+};
+const computeTextScore = (row, tokens) => {
+    if (tokens.length === 0) {
+        return 0;
+    }
+    const title = normalizeToken(row.title);
+    const path = normalizeToken(row.path);
+    const content = normalizeToken(row.content);
+    const tags = normalizeToken(row.tags.join(' '));
+    return tokens.reduce((score, token) => {
+        const titleHits = countOccurrences(title, token);
+        const tagHits = countOccurrences(tags, token);
+        const pathHits = countOccurrences(path, token);
+        const contentHits = countOccurrences(content, token);
+        return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 5);
+    }, 0);
+};
+const toSearchResult = (row, score) => ({
+    documentId: row.documentId,
+    agentId: row.agentId,
+    title: row.title,
+    path: row.path,
+    chunkId: row.chunkId,
+    content: row.content,
+    score,
+    textScore: score,
+    semanticScore: 0,
+    searchMode: 'fts',
+    tags: row.tags
+});
+const sortedPackFiles = async (vaultPath) => {
+    try {
+        const files = await readdir(toPackDirectory(vaultPath));
+        return files
+            .filter((file) => file.endsWith('.jsonl.gz'))
+            .sort((left, right) => left.localeCompare(right));
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+};
+export const buildSearchPacks = async (vaultPath, documents) => {
+    const directory = toPackDirectory(vaultPath);
+    const rows = toRows(documents);
+    await mkdir(directory, { recursive: true });
+    const current = await readdir(directory);
+    await Promise.all(current
+        .filter((name) => name.endsWith('.jsonl.gz') || name === manifestFileName)
+        .map((name) => rm(join(directory, name), { force: true })));
+    const chunks = chunkRows(rows, rowChunkSize);
+    await Promise.all(chunks.map(async (chunk, index) => {
+        const fileName = `pack-${String(index + 1).padStart(4, '0')}.jsonl.gz`;
+        const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
+        const compressed = gzipSync(Buffer.from(serialized, 'utf8'), { level: 6 });
+        await writeFile(join(directory, fileName), compressed);
+    }));
+    await writeManifest(vaultPath, {
+        version: 1,
+        createdAt: new Date().toISOString(),
+        packCount: chunks.length,
+        recordCount: rows.length
+    });
+    return {
+        packCount: chunks.length,
+        recordCount: rows.length
+    };
+};
+export const searchInPacks = async (vaultPath, query, limit, agentId) => {
+    const normalizedAgent = agentId?.trim();
+    const tokens = tokenize(query);
+    if (limit <= 0 || tokens.length === 0) {
+        return [];
+    }
+    const files = await sortedPackFiles(vaultPath);
+    if (files.length === 0) {
+        return [];
+    }
+    const scored = [];
+    for (const file of files) {
+        const rows = parseRowsFromPack(await readFile(join(toPackDirectory(vaultPath), file)));
+        rows.forEach((row) => {
+            if (normalizedAgent && row.agentId !== normalizedAgent) {
+                return;
+            }
+            const score = computeTextScore(row, tokens);
+            if (score > 0) {
+                scored.push(toSearchResult(row, score));
+            }
+        });
+    }
+    return scored
+        .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
+        .slice(0, limit);
+};

package/docs/AGENT_USAGE.md

@@ -635,6 +635,7 @@ GET  /api/validate
 The HTTP API is read-only. Use the CLI for writes and indexing.
 
 Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores from snapshot automatically or recreates a clean index if no snapshot exists yet.
+Indexing also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
 
 ## Agent Integration Contract
 

package/docs/ARCHITECTURE.md

@@ -301,6 +301,7 @@ Markdown keeps the system portable, inspectable, Git-friendly, and compatible wi
 SQLite gives fast local search, local vector storage and rebuildable retrieval without forcing users to run external infrastructure.
 Hybrid retrieval also uses a short-lived in-memory cache keyed by vault/query/agent and invalidated by index file mtime to reduce repeated query latency.
 Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores from snapshot automatically before reopening the index.
+Indexing additionally exports compressed pack files (`.brainlink/search-packs/*.jsonl.gz`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
 
 ### CLI First
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.10",
+  "version": "0.1.0-beta.11",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",