@andespindola/brainlink 0.1.0-beta.14 → 0.1.0-beta.140

package/dist/application/offline-pack-backup.js

@@ -0,0 +1,44 @@
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { gzipSync } from 'node:zlib';
+import { ensureVault } from '../infrastructure/file-system-vault.js';
+const packsDirectory = (vaultPath) => join(vaultPath, '.brainlink', 'search-packs');
+const toSortedBackupFiles = async (vaultPath) => {
+    const directory = packsDirectory(vaultPath);
+    const names = await readdir(directory);
+    return names
+        .filter((name) => name.endsWith('.blpk') || name === 'manifest.json')
+        .sort((left, right) => left.localeCompare(right));
+};
+export const createOfflinePackBackup = async (input) => {
+    const vaultPath = await ensureVault(input.vaultPath);
+    const fileNames = await toSortedBackupFiles(vaultPath);
+    const files = [];
+    let inputBytes = 0;
+    for (const name of fileNames) {
+        const content = await readFile(join(packsDirectory(vaultPath), name));
+        inputBytes += content.byteLength;
+        files.push({
+            name,
+            contentB64: content.toString('base64')
+        });
+    }
+    const envelope = {
+        version: 1,
+        createdAt: new Date().toISOString(),
+        files
+    };
+    const serialized = Buffer.from(JSON.stringify(envelope), 'utf8');
+    const compressed = gzipSync(serialized, { level: 9 });
+    await mkdir(dirname(input.outputPath), { recursive: true });
+    await writeFile(input.outputPath, compressed);
+    const safeInput = Math.max(inputBytes, 1);
+    return {
+        outputPath: input.outputPath,
+        fileCount: files.length,
+        inputBytes,
+        outputBytes: compressed.byteLength,
+        ratio: compressed.byteLength / safeInput,
+        savedBytes: Math.max(inputBytes - compressed.byteLength, 0)
+    };
+};

package/dist/application/search-graph-node-ids.js

@@ -1,10 +1,10 @@
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 export const searchGraphNodeIds = async (vaultPath, query, limit, agentId) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.searchGraphNodeIds(query, limit, agentId);
+        return await index.searchGraphNodeIds(query, limit, agentId);
     }
     finally {
         index.close();

package/dist/application/search-knowledge.js

@@ -1,8 +1,7 @@
 import { stat } from 'node:fs/promises';
-import { join } from 'node:path';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
 import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { indexStoragePath, openFileIndex } from '../infrastructure/file-index.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
 const hybridCacheTtlMs = 30_000;
@@ -10,7 +9,7 @@ const hybridCacheMaxEntries = 200;
 const hybridSearchCache = new Map();
 const readIndexMtimeMs = async (vaultPath) => {
     try {
-        return (await stat(join(vaultPath, '.brainlink', 'brainlink.db'))).mtimeMs;
+        return (await stat(indexStoragePath(vaultPath))).mtimeMs;
     }
     catch {
         return 0;
@@ -58,9 +57,9 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const shouldEmbedQuery = searchMode !== 'fts' && provider.name !== 'none';
     const queryEmbedding = shouldEmbedQuery ? (await provider.embed([query]))[0] ?? [] : [];
     try {
-        const index = openSqliteIndex(absoluteVaultPath);
+        const index = openFileIndex(absoluteVaultPath);
         try {
-            const results = index.search(query, limit, agentId, searchMode, queryEmbedding);
+            const results = await index.search(query, limit, agentId, searchMode, queryEmbedding);
             if (cacheKey) {
                 cacheSet({
                     key: cacheKey,

package/dist/application/server/routes.js

@@ -3,6 +3,7 @@ import { buildContextPackage } from '../build-context.js';
 import { getGraph } from '../get-graph.js';
 import { getGraphNode } from '../get-graph-node.js';
 import { getGraphLayout } from '../get-graph-layout.js';
+import { getGraphView } from '../get-graph-view.js';
 import { listAgents } from '../list-agents.js';
 import { listBacklinks, listLinks } from '../list-links.js';
 import { searchGraphNodeIds } from '../search-graph-node-ids.js';
@@ -11,6 +12,7 @@ import { loadBrainlinkConfig, sanitizeSearchMode } from '../../infrastructure/co
 import { createClientCss } from '../frontend/client-css.js';
 import { createClientHtml } from '../frontend/client-html.js';
 import { createClientJs } from '../frontend/client-js.js';
+import { createClientWorkerJs } from '../frontend/client-worker-js.js';
 import { contentTypes, createJsonResponse, isReadMethod, parsePositiveInteger } from './http.js';
 const readSearchMode = async (url) => {
     const config = await loadBrainlinkConfig();
@@ -51,19 +53,154 @@ const sameEntityTag = (candidate, signature) => {
     return decodeEntityTag(candidate) === signature;
 };
 const readAgentQuery = (url) => url.searchParams.get('agent') ?? undefined;
+const parseNumber = (value, fallback) => {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : fallback;
+};
+const compactGraphLayoutThreshold = 12_000;
+const compactGraphLayoutEdgeLimit = 60_000;
+const graphLayoutBodyCacheLimit = 8;
+const graphLayoutBodyCache = new Map();
+let cachedClientHtml = null;
+let cachedClientCss = null;
+let cachedClientJs = null;
+let cachedClientWorkerJs = null;
+const readClientHtml = () => {
+    if (cachedClientHtml === null) {
+        cachedClientHtml = createClientHtml();
+    }
+    return cachedClientHtml;
+};
+const readClientCss = () => {
+    if (cachedClientCss === null) {
+        cachedClientCss = createClientCss();
+    }
+    return cachedClientCss;
+};
+const readClientJs = () => {
+    if (cachedClientJs === null) {
+        cachedClientJs = createClientJs();
+    }
+    return cachedClientJs;
+};
+const readClientWorkerJs = () => {
+    if (cachedClientWorkerJs === null) {
+        cachedClientWorkerJs = createClientWorkerJs();
+    }
+    return cachedClientWorkerJs;
+};
+const compactGraphLayoutEdgeLimitFor = (nodeCount) => {
+    if (nodeCount > 100_000)
+        return 15_000;
+    if (nodeCount > 50_000)
+        return 22_000;
+    if (nodeCount > 25_000)
+        return 30_000;
+    return compactGraphLayoutEdgeLimit;
+};
+const edgeWeight = (weight) => Number.isFinite(weight) ? Number(weight) : 1;
+const edgeKey = (source, target, priority) => `${source}|${target}|${priority}`;
+const selectCompactEdges = (layout, limit) => {
+    const resolvedEdges = layout.edges.filter((edge) => typeof edge.target === 'string' && edge.target.length > 0);
+    if (resolvedEdges.length <= limit) {
+        return resolvedEdges;
+    }
+    const bestEdgeByEndpoint = new Map();
+    for (let index = 0; index < resolvedEdges.length; index += 1) {
+        const edge = resolvedEdges[index];
+        const endpoints = [edge.source, edge.target];
+        for (let endpointIndex = 0; endpointIndex < endpoints.length; endpointIndex += 1) {
+            const endpoint = endpoints[endpointIndex];
+            const previous = bestEdgeByEndpoint.get(endpoint);
+            if (!previous || edgeWeight(edge.weight) > edgeWeight(previous.weight)) {
+                bestEdgeByEndpoint.set(endpoint, edge);
+            }
+        }
+    }
+    const selected = new Map();
+    for (const edge of bestEdgeByEndpoint.values()) {
+        selected.set(edgeKey(edge.source, edge.target, edge.priority), edge);
+    }
+    if (selected.size > limit) {
+        return Array.from(selected.values())
+            .sort((left, right) => edgeWeight(right.weight) - edgeWeight(left.weight))
+            .slice(0, limit);
+    }
+    const byWeight = [...resolvedEdges].sort((left, right) => edgeWeight(right.weight) - edgeWeight(left.weight));
+    for (let index = 0; index < byWeight.length; index += 1) {
+        if (selected.size >= limit) {
+            break;
+        }
+        const edge = byWeight[index];
+        const key = edgeKey(edge.source, edge.target, edge.priority);
+        if (!selected.has(key)) {
+            selected.set(key, edge);
+        }
+    }
+    return Array.from(selected.values());
+};
 const stripLayoutContent = (layout) => ({
     ...layout,
     nodes: layout.nodes.map(({ content, ...node }) => node)
 });
+const compactLayoutPayload = (layout) => {
+    const edgeLimit = compactGraphLayoutEdgeLimitFor(layout.nodes.length);
+    const compactEdges = selectCompactEdges(layout, edgeLimit);
+    const compactNodes = layout.nodes.map((node) => [node.id, node.title, node.x, node.y, node.group, node.segment]);
+    const compactEdgeRows = compactEdges
+        .map((edge) => [edge.source, edge.target, edge.weight, edge.priority]);
+    const compactGroups = layout.groups?.map((group) => [
+        group.id,
+        group.level,
+        group.parentId,
+        group.title,
+        group.x,
+        group.y,
+        group.radius,
+        group.segment,
+        group.group,
+        group.nodeIds,
+        group.childGroupIds
+    ]);
+    return {
+        compact: true,
+        layout: {
+            nodes: compactNodes,
+            edges: compactEdgeRows,
+            ...(compactGroups && compactGroups.length > 0 ? { groups: compactGroups } : {})
+        },
+        totals: {
+            nodes: layout.nodes.length,
+            edges: layout.edges.length
+        }
+    };
+};
+const encodeLayoutPayload = (layout) => layout.nodes.length > compactGraphLayoutThreshold ? compactLayoutPayload(layout) : { compact: false, layout: stripLayoutContent(layout) };
+const readGraphLayoutBody = (signature) => graphLayoutBodyCache.get(signature) ?? null;
+const storeGraphLayoutBody = (signature, body) => {
+    if (graphLayoutBodyCache.has(signature)) {
+        graphLayoutBodyCache.delete(signature);
+    }
+    graphLayoutBodyCache.set(signature, body);
+    while (graphLayoutBodyCache.size > graphLayoutBodyCacheLimit) {
+        const oldest = graphLayoutBodyCache.keys().next().value;
+        if (!oldest)
+            break;
+        graphLayoutBodyCache.delete(oldest);
+    }
+};
 export const route = async (request, url, vaultPath) => {
     if (isReadMethod(request) && (url.pathname === '/' || url.pathname === '/index.html')) {
-        return createResponse(createClientHtml(), 200, contentTypes['.html']);
+        return createResponse(readClientHtml(), 200, contentTypes['.html']);
     }
     if (isReadMethod(request) && url.pathname === '/styles.css') {
-        return createResponse(createClientCss(), 200, contentTypes['.css']);
+        return createResponse(readClientCss(), 200, contentTypes['.css']);
     }
     if (isReadMethod(request) && url.pathname === '/app.js') {
-        return createResponse(createClientJs(), 200, contentTypes['.js']);
+        return createResponse(readClientJs(), 200, contentTypes['.js']);
+    }
+    if (isReadMethod(request) && url.pathname === '/app-worker.js') {
+        return createResponse(readClientWorkerJs(), 200, contentTypes['.js']);
     }
     if (isReadMethod(request) && url.pathname === '/api/graph') {
         return createResponse(createJsonResponse(await getGraph(vaultPath, readAgentQuery(url))), 200, contentTypes['.json']);
@@ -73,8 +210,6 @@ export const route = async (request, url, vaultPath) => {
         const requestEtags = request.headers['if-none-match'];
         const notModified = sameEntityTag(requestEtags, signature);
         const etag = encodeEntityTag(signature);
-        const body = createJsonResponse({ signature, layout: stripLayoutContent(layout) });
-        const jsonResponse = createResponse(body, 200, contentTypes['.json']);
         const notModifiedResponse = createResponse('', 304, contentTypes['.json']);
         if (notModified) {
             return {
@@ -85,6 +220,12 @@ export const route = async (request, url, vaultPath) => {
                 }
             };
         }
+        const cachedBody = readGraphLayoutBody(signature);
+        const body = cachedBody ?? createJsonResponse({ signature, ...encodeLayoutPayload(layout) });
+        if (!cachedBody) {
+            storeGraphLayoutBody(signature, body);
+        }
+        const jsonResponse = createResponse(body, 200, contentTypes['.json']);
         return {
             ...jsonResponse,
             headers: {
@@ -93,6 +234,16 @@ export const route = async (request, url, vaultPath) => {
             }
         };
     }
+    if (isReadMethod(request) && url.pathname === '/api/graph-view') {
+        return createResponse(createJsonResponse(await getGraphView(vaultPath, {
+            x: parseNumber(url.searchParams.get('x'), -1000),
+            y: parseNumber(url.searchParams.get('y'), -1000),
+            width: parseNumber(url.searchParams.get('w'), 2000),
+            height: parseNumber(url.searchParams.get('h'), 2000),
+            scale: parseNumber(url.searchParams.get('scale'), 1),
+            agentId: readAgentQuery(url)
+        })), 200, contentTypes['.json']);
+    }
     if (isReadMethod(request) && url.pathname === '/api/graph-node') {
         const id = url.searchParams.get('id')?.trim() ?? '';
         if (!id) {

package/dist/application/start-server.js

@@ -1,9 +1,78 @@
 import { createServer } from 'node:http';
+import { brotliCompressSync, constants, gzipSync } from 'node:zlib';
 import { indexVault } from './index-vault.js';
 import { startVaultWatcher } from './watch-vault.js';
 import { assertLoopbackHost } from './server/host-security.js';
 import { contentTypes, createJsonResponse, isHttpError } from './server/http.js';
 import { route } from './server/routes.js';
+const compressionThresholdBytes = 1024;
+const normalizeEncodingToken = (value) => value.trim().toLowerCase();
+const supportsEncoding = (acceptEncoding, target) => {
+    if (!acceptEncoding) {
+        return false;
+    }
+    return acceptEncoding
+        .split(',')
+        .map((entry) => entry.split(';')[0] ?? '')
+        .map(normalizeEncodingToken)
+        .includes(target);
+};
+const isCompressibleContentType = (contentType) => {
+    const normalized = contentType?.toLowerCase() ?? '';
+    return (normalized.includes('application/json') ||
+        normalized.includes('text/javascript') ||
+        normalized.includes('text/css') ||
+        normalized.includes('text/html') ||
+        normalized.startsWith('text/'));
+};
+const maybeCompressResponse = (requestHeaders, statusCode, headers, body) => {
+    if (statusCode === 204 || statusCode === 304) {
+        return { headers, body: '' };
+    }
+    if (!isCompressibleContentType(headers['content-type'])) {
+        return { headers, body };
+    }
+    const bodyBuffer = Buffer.from(body, 'utf8');
+    if (bodyBuffer.byteLength < compressionThresholdBytes) {
+        return { headers, body };
+    }
+    if (headers['content-encoding']) {
+        return { headers, body };
+    }
+    const acceptEncodingHeader = Array.isArray(requestHeaders['accept-encoding'])
+        ? requestHeaders['accept-encoding'].join(',')
+        : requestHeaders['accept-encoding'];
+    const vary = headers.vary ? `${headers.vary}, Accept-Encoding` : 'Accept-Encoding';
+    const withVary = {
+        ...headers,
+        vary
+    };
+    if (supportsEncoding(acceptEncodingHeader, 'br')) {
+        return {
+            headers: {
+                ...withVary,
+                'content-encoding': 'br'
+            },
+            body: brotliCompressSync(bodyBuffer, {
+                params: {
+                    [constants.BROTLI_PARAM_QUALITY]: 5
+                }
+            })
+        };
+    }
+    if (supportsEncoding(acceptEncodingHeader, 'gzip')) {
+        return {
+            headers: {
+                ...withVary,
+                'content-encoding': 'gzip'
+            },
+            body: gzipSync(bodyBuffer, {
+                level: 6
+            })
+        };
+    }
+    return { headers: withVary, body };
+};
 export const startServer = async (input) => {
     assertLoopbackHost(input.host);
     if (input.shouldIndex) {
@@ -19,14 +88,16 @@ export const startServer = async (input) => {
         const url = new URL(request.url ?? '/', `http://${request.headers.host ?? input.host}`);
         route(request, url, input.vaultPath)
             .then((result) => {
-            response.writeHead(result.statusCode, result.headers);
-            response.end(result.body);
+            const encoded = maybeCompressResponse(request.headers, result.statusCode, result.headers, result.body);
+            response.writeHead(result.statusCode, encoded.headers);
+            response.end(encoded.body);
         })
             .catch((error) => {
             const message = error instanceof Error ? error.message : String(error);
             const statusCode = isHttpError(error) ? error.statusCode : 500;
-            response.writeHead(statusCode, { 'content-type': contentTypes['.json'] });
-            response.end(createJsonResponse({ error: message }));
+            const fallback = maybeCompressResponse(request.headers, statusCode, { 'content-type': contentTypes['.json'] }, createJsonResponse({ error: message }));
+            response.writeHead(statusCode, fallback.headers);
+            response.end(fallback.body);
         });
     });
     await new Promise((resolve, reject) => {

package/dist/application/watch-vault.js

@@ -1,5 +1,5 @@
 import { watch } from 'node:fs';
-import { indexVault } from './index-vault.js';
+import { indexVaultWithOptions } from './index-vault.js';
 import { isBucketVaultPath, resolveVaultPath } from '../infrastructure/file-system-vault.js';
 const shouldIgnore = (filename) => {
     if (!filename) {
@@ -14,6 +14,27 @@ export const startVaultWatcher = (input) => {
     const absoluteVaultPath = resolveVaultPath(input.vaultPath);
     const debounceMs = input.debounceMs ?? 350;
     let timeout = null;
+    let running = false;
+    let pending = false;
+    const runIndex = () => {
+        if (running) {
+            pending = true;
+            return;
+        }
+        running = true;
+        indexVaultWithOptions(absoluteVaultPath, {
+            onProgress: input.onProgress
+        })
+            .then(input.onIndex)
+            .catch(input.onError)
+            .finally(() => {
+            running = false;
+            if (pending) {
+                pending = false;
+                runIndex();
+            }
+        });
+    };
     const schedule = (filename) => {
         if (shouldIgnore(filename)) {
             return;
@@ -22,7 +43,7 @@ export const startVaultWatcher = (input) => {
             clearTimeout(timeout);
         }
         timeout = setTimeout(() => {
-            indexVault(absoluteVaultPath).then(input.onIndex).catch(input.onError);
+            runIndex();
         }, debounceMs);
     };
     const watcher = watch(absoluteVaultPath, { recursive: true }, (_eventType, filename) => {

package/dist/benchmarks/large-vault.js

@@ -21,7 +21,7 @@ const readOptions = (args) => ({
 });
 const topics = [
     'authentication jwt token refresh policy',
-    'sqlite graph backlinks markdown vault indexing',
+    'graph backlinks markdown vault indexing',
     'frontend canvas layout graph interaction',
     'agent memory context retrieval summarization',
     'security local server vault path allowlist',

package/dist/cli/commands/agent-commands.js

@@ -141,6 +141,12 @@ const parseAllowedVaults = (value) => {
 export const installAgentIntegration = async (input) => {
     const codexConfigPath = getCodexConfigPath();
     const allowedVaults = parseAllowedVaults(input.allowedVaults);
+    const bootstrapPolicy = await setBootstrapPolicy({
+        enforceBootstrap: true,
+        enforceContextFirst: true,
+        autoBootstrapOnRead: true,
+        autoBootstrapOnStartup: true
+    });
     await upsertCodexMcpConfig(codexConfigPath, {
         allowedVaults,
         brainlinkHome: input.brainlinkHome
@@ -195,6 +201,7 @@ export const installAgentIntegration = async (input) => {
         codexConfigPath,
         mcpServer: 'brainlink',
         command: 'brainlink-mcp',
+        bootstrapPolicy,
         ...(input.mcpOnly !== true ? { pluginSourcePath, pluginSymlinkPath, marketplacePath } : {}),
         ...(selfTestResult ? { selfTest: selfTestResult } : {}),
         ...(warnings.length > 0 ? { warnings } : {})