@andespindola/brainlink 0.1.0-beta.9 → 0.1.0-beta.90

package/dist/application/index-vault.js

@@ -1,9 +1,12 @@
+import { readFile } from 'node:fs/promises';
 import { createIndexedDocument, parseMarkdownDocument } from '../domain/markdown.js';
 import { sharedAgentId } from '../domain/agents.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig } from '../infrastructure/config.js';
-import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { ensureVault, readMarkdownFileSummaries } from '../infrastructure/file-system-vault.js';
+import { readIndexState, writeIndexState } from '../infrastructure/index-state.js';
+import { buildSearchPacks, ensureSearchPackManifest, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 const toTitleKey = (title) => title.toLowerCase();
 const appendTitleEntry = (map, document) => {
     const key = toTitleKey(document.title);
@@ -33,6 +36,9 @@ const createScopedTitleResolver = (document, titleMaps) => ({
     get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
 });
 const embedIndexedDocuments = async (documents, providerName) => {
+    if (documents.length === 0) {
+        return documents;
+    }
     const provider = createEmbeddingProvider(providerName);
     const chunks = documents.flatMap((document) => document.chunks);
     const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
@@ -46,28 +52,255 @@ const embedIndexedDocuments = async (documents, providerName) => {
         }))
     }));
 };
+const relinkIndexedDocument = (indexedDocument, titleMaps) => {
+    const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
+    return {
+        ...indexedDocument,
+        links: indexedDocument.links
+            .map((link) => ({
+            ...link,
+            toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
+        }))
+            .filter((link) => link.toDocumentId !== indexedDocument.document.id)
+    };
+};
+const toIndexResult = (documents) => ({
+    documentCount: documents.length,
+    chunkCount: documents.reduce((total, document) => total + document.chunks.length, 0),
+    linkCount: documents.reduce((total, document) => total + document.links.length, 0)
+});
+const toSnapshot = (summaries) => summaries.map((summary) => ({
+    path: summary.relativePath,
+    mtimeMs: summary.updatedAt.getTime(),
+    size: summary.size
+}));
+const createSnapshotMap = (snapshot) => new Map(snapshot.map((entry) => [entry.path, entry]));
+const readChangedDocuments = async (absoluteVaultPath, changedSummaries) => {
+    const parsed = await Promise.all(changedSummaries.map(async (summary) => parseMarkdownDocument({
+        absolutePath: summary.absolutePath,
+        vaultPath: absoluteVaultPath,
+        content: await readFile(summary.absolutePath, 'utf8'),
+        createdAt: summary.createdAt,
+        updatedAt: summary.updatedAt
+    })));
+    return new Map(parsed.map((document) => [document.path, document]));
+};
 export const indexVault = async (vaultPath) => {
+    return indexVaultWithOptions(vaultPath, {});
+};
+export const indexVaultWithOptions = async (vaultPath, options) => {
+    const startedAt = process.hrtime.bigint();
+    const elapsedMs = () => Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const emit = (phase, status, message, details) => {
+        options.onProgress?.({
+            phase,
+            status,
+            message,
+            elapsedMs: elapsedMs(),
+            timestamp: new Date().toISOString(),
+            details
+        });
+    };
+    emit('start', 'start', 'Indexing started');
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
-    const files = await readMarkdownFiles(absoluteVaultPath);
-    const documents = files.map((file) => parseMarkdownDocument({
-        absolutePath: file.absolutePath,
-        vaultPath: absoluteVaultPath,
-        content: file.content,
-        createdAt: file.createdAt,
-        updatedAt: file.updatedAt
-    }));
-    const titleMaps = createTitleMaps(documents);
-    const indexedDocuments = await embedIndexedDocuments(documents.map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider);
-    const index = openSqliteIndex(absoluteVaultPath);
+    emit('scan', 'start', 'Scanning markdown files');
+    const [summaries, previousState] = await Promise.all([
+        readMarkdownFileSummaries(absoluteVaultPath),
+        readIndexState(absoluteVaultPath)
+    ]);
+    emit('scan', 'finish', 'Scan complete', {
+        markdownFiles: summaries.length,
+        hasPreviousState: previousState != null
+    });
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        index.reset();
-        index.saveDocuments(indexedDocuments);
-        return {
-            documentCount: indexedDocuments.length,
-            chunkCount: indexedDocuments.reduce((total, document) => total + document.chunks.length, 0),
-            linkCount: indexedDocuments.reduce((total, document) => total + document.links.length, 0)
+        const existingIndexedDocuments = await index.getIndexedDocuments();
+        const existingByPath = new Map(existingIndexedDocuments.map((document) => [document.document.path, document]));
+        const currentSnapshot = toSnapshot(summaries);
+        const currentSnapshotMap = createSnapshotMap(currentSnapshot);
+        const previousSnapshotMap = createSnapshotMap(previousState?.files ?? []);
+        const settingsChanged = previousState == null ||
+            previousState.chunkSize !== config.chunkSize ||
+            previousState.embeddingProvider !== config.embeddingProvider;
+        const packSettingsChanged = previousState == null ||
+            previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
+            previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
+            previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
+        const changedPaths = new Set();
+        for (let index = 0; index < summaries.length; index += 1) {
+            const summary = summaries[index];
+            const previous = previousSnapshotMap.get(summary.relativePath);
+            const changed = settingsChanged ||
+                previous == null ||
+                previous.mtimeMs !== summary.updatedAt.getTime() ||
+                previous.size !== summary.size ||
+                !existingByPath.has(summary.relativePath);
+            if (changed) {
+                changedPaths.add(summary.relativePath);
+            }
+        }
+        const hasDeletes = previousState
+            ? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
+            : false;
+        const manifestRecovery = await ensureSearchPackManifest(absoluteVaultPath);
+        if (changedPaths.size === 0 &&
+            !hasDeletes &&
+            existingIndexedDocuments.length === summaries.length &&
+            previousState != null) {
+            const result = {
+                ...toIndexResult(existingIndexedDocuments),
+                elapsedMs: elapsedMs(),
+                changedDocumentCount: 0,
+                packs: {
+                    rebuilt: false,
+                    reason: manifestRecovery.repaired ? 'No changes detected; pack manifest repaired' : 'No changes detected'
+                }
+            };
+            emit('complete', 'skip', 'Index skipped: no changes detected', {
+                elapsedMs: result.elapsedMs,
+                manifestRepaired: manifestRecovery.repaired,
+                manifestRecoverySource: manifestRecovery.source
+            });
+            return result;
+        }
+        const changedSummaries = summaries.filter((summary) => changedPaths.has(summary.relativePath));
+        emit('parse', 'start', 'Parsing changed markdown files', {
+            changedFiles: changedSummaries.length
+        });
+        const changedDocumentsByPath = await readChangedDocuments(absoluteVaultPath, changedSummaries);
+        emit('parse', 'finish', 'Parse complete', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const documents = summaries.flatMap((summary) => {
+            const changed = changedDocumentsByPath.get(summary.relativePath);
+            if (changed) {
+                return [changed];
+            }
+            const existing = existingByPath.get(summary.relativePath);
+            return existing ? [existing.document] : [];
+        });
+        const titleMaps = createTitleMaps(documents);
+        emit('embed', 'start', 'Embedding changed chunks', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const changedIndexedDocuments = changedDocumentsByPath.size > 0
+            ? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
+            : [];
+        emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
+            changedIndexedDocuments: changedIndexedDocuments.length
+        });
+        const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
+        const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
+        const indexedDocuments = documents.map((document) => {
+            const changed = changedIndexedByPath.get(document.path);
+            if (changed) {
+                return changed;
+            }
+            const existing = existingByPath.get(document.path);
+            if (!existing) {
+                return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
+            }
+            return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
+        });
+        emit('persist', 'start', 'Persisting index');
+        await index.reset();
+        await index.saveDocuments(indexedDocuments);
+        emit('persist', 'finish', 'Index persisted', {
+            indexedDocuments: indexedDocuments.length
+        });
+        const existingPackManifest = manifestRecovery.repaired || manifestRecovery.source === 'not-needed';
+        const changedCount = changedPaths.size;
+        const documentCount = Math.max(indexedDocuments.length, 1);
+        const changeRatio = changedCount / documentCount;
+        const previousPendingPackChanges = previousState?.pendingPackChanges ?? 0;
+        const pendingPackChanges = previousPendingPackChanges + changedCount;
+        const shouldRebuildPacks = !existingPackManifest ||
+            settingsChanged ||
+            packSettingsChanged ||
+            hasDeletes ||
+            changedCount >= 400 ||
+            changeRatio >= 0.04 ||
+            pendingPackChanges >= 1200;
+        let packResult;
+        const packReason = !existingPackManifest
+            ? 'Missing pack manifest'
+            : manifestRecovery.repaired
+                ? 'Pack manifest repaired from existing packs'
+                : settingsChanged
+                    ? 'Index settings changed'
+                    : packSettingsChanged
+                        ? 'Search pack settings changed'
+                        : hasDeletes
+                            ? 'Document deletions detected'
+                            : changedCount >= 400
+                                ? 'Changed file count threshold reached'
+                                : changeRatio >= 0.04
+                                    ? 'Change ratio threshold reached'
+                                    : pendingPackChanges >= 1200
+                                        ? 'Pending pack changes threshold reached'
+                                        : 'Pack rebuild skipped';
+        if (shouldRebuildPacks) {
+            emit('packs', 'start', 'Rebuilding compressed search packs', {
+                reason: packReason
+            });
+            try {
+                packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
+                emit('packs', 'finish', 'Compressed packs rebuilt', {
+                    reason: packReason,
+                    packCount: packResult.packCount,
+                    recordCount: packResult.recordCount,
+                    durationMs: packResult.durationMs,
+                    compressionRatio: packResult.compression.ratio
+                });
+            }
+            catch {
+                // Pack generation is best-effort. The JSON index remains the primary path.
+                emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
+                    reason: packReason
+                });
+            }
+        }
+        else {
+            emit('packs', 'skip', 'Pack rebuild not required', {
+                reason: packReason
+            });
+        }
+        const packsRebuilt = packResult != null;
+        const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
+        await writeIndexState(absoluteVaultPath, {
+            chunkSize: config.chunkSize,
+            embeddingProvider: config.embeddingProvider,
+            searchPackRowChunkSize: config.searchPack.rowChunkSize,
+            searchPackCompressionLevel: config.searchPack.compressionLevel,
+            searchPackUseDictionary: config.searchPack.useDictionary,
+            files: currentSnapshot,
+            pendingPackChanges: packsRebuilt ? 0 : pendingPackChanges
+        });
+        const result = {
+            ...toIndexResult(indexedDocuments),
+            elapsedMs: elapsedMs(),
+            changedDocumentCount: changedDocumentsByPath.size,
+            packs: {
+                rebuilt: packsRebuilt,
+                reason: packResultReason,
+                ...(packResult
+                    ? {
+                        packCount: packResult.packCount,
+                        recordCount: packResult.recordCount,
+                        durationMs: packResult.durationMs,
+                        compression: packResult.compression
+                    }
+                    : {})
+            }
         };
+        emit('complete', 'finish', 'Indexing complete', {
+            documentCount: result.documentCount,
+            chunkCount: result.chunkCount,
+            linkCount: result.linkCount,
+            elapsedMs: result.elapsedMs
+        });
+        return result;
     }
     finally {
         index.close();

package/dist/application/list-agents.js

@@ -1,10 +1,10 @@
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 export const listAgents = async (vaultPath) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listAgents();
+        return await index.listAgents();
     }
     finally {
         index.close();

package/dist/application/list-links.js

@@ -1,10 +1,10 @@
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 export const listLinks = async (vaultPath, agentId) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listLinks(agentId);
+        return await index.listLinks(agentId);
     }
     finally {
         index.close();
@@ -12,9 +12,9 @@ export const listLinks = async (vaultPath, agentId) => {
 };
 export const listBacklinks = async (vaultPath, title, agentId) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listBacklinks(title, agentId);
+        return await index.listBacklinks(title, agentId);
     }
     finally {
         index.close();

package/dist/application/offline-pack-backup.js

@@ -0,0 +1,44 @@
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { gzipSync } from 'node:zlib';
+import { ensureVault } from '../infrastructure/file-system-vault.js';
+const packsDirectory = (vaultPath) => join(vaultPath, '.brainlink', 'search-packs');
+const toSortedBackupFiles = async (vaultPath) => {
+    const directory = packsDirectory(vaultPath);
+    const names = await readdir(directory);
+    return names
+        .filter((name) => name.endsWith('.blpk') || name === 'manifest.json')
+        .sort((left, right) => left.localeCompare(right));
+};
+export const createOfflinePackBackup = async (input) => {
+    const vaultPath = await ensureVault(input.vaultPath);
+    const fileNames = await toSortedBackupFiles(vaultPath);
+    const files = [];
+    let inputBytes = 0;
+    for (const name of fileNames) {
+        const content = await readFile(join(packsDirectory(vaultPath), name));
+        inputBytes += content.byteLength;
+        files.push({
+            name,
+            contentB64: content.toString('base64')
+        });
+    }
+    const envelope = {
+        version: 1,
+        createdAt: new Date().toISOString(),
+        files
+    };
+    const serialized = Buffer.from(JSON.stringify(envelope), 'utf8');
+    const compressed = gzipSync(serialized, { level: 9 });
+    await mkdir(dirname(input.outputPath), { recursive: true });
+    await writeFile(input.outputPath, compressed);
+    const safeInput = Math.max(inputBytes, 1);
+    return {
+        outputPath: input.outputPath,
+        fileCount: files.length,
+        inputBytes,
+        outputBytes: compressed.byteLength,
+        ratio: compressed.byteLength / safeInput,
+        savedBytes: Math.max(inputBytes - compressed.byteLength, 0)
+    };
+};

package/dist/application/search-graph-node-ids.js

@@ -0,0 +1,12 @@
+import { ensureVault } from '../infrastructure/file-system-vault.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
+export const searchGraphNodeIds = async (vaultPath, query, limit, agentId) => {
+    const absoluteVaultPath = await ensureVault(vaultPath);
+    const index = openFileIndex(absoluteVaultPath);
+    try {
+        return await index.searchGraphNodeIds(query, limit, agentId);
+    }
+    finally {
+        index.close();
+    }
+};

package/dist/application/search-knowledge.js

@@ -1,7 +1,7 @@
 import { stat } from 'node:fs/promises';
-import { join } from 'node:path';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
+import { indexStoragePath, openFileIndex } from '../infrastructure/file-index.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
 const hybridCacheTtlMs = 30_000;
@@ -9,7 +9,7 @@ const hybridCacheMaxEntries = 200;
 const hybridSearchCache = new Map();
 const readIndexMtimeMs = async (vaultPath) => {
     try {
-        return (await stat(join(vaultPath, '.brainlink', 'brainlink.db'))).mtimeMs;
+        return (await stat(indexStoragePath(vaultPath))).mtimeMs;
     }
     catch {
         return 0;
@@ -46,6 +46,7 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
     const searchMode = sanitizeSearchMode(mode, config.defaultSearchMode);
+    await ensurePrivatePacksFromLegacyIndex(absoluteVaultPath);
     const cacheKey = searchMode === 'hybrid' ? toCacheKey(absoluteVaultPath, query, limit, agentId) : undefined;
     const indexMtimeMs = cacheKey ? await readIndexMtimeMs(absoluteVaultPath) : 0;
     const cached = cacheKey ? cacheGet(cacheKey, indexMtimeMs) : undefined;
@@ -55,20 +56,34 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const provider = createEmbeddingProvider(config.embeddingProvider);
     const shouldEmbedQuery = searchMode !== 'fts' && provider.name !== 'none';
     const queryEmbedding = shouldEmbedQuery ? (await provider.embed([query]))[0] ?? [] : [];
-    const index = openSqliteIndex(absoluteVaultPath);
     try {
-        const results = index.search(query, limit, agentId, searchMode, queryEmbedding);
+        const index = openFileIndex(absoluteVaultPath);
+        try {
+            const results = await index.search(query, limit, agentId, searchMode, queryEmbedding);
+            if (cacheKey) {
+                cacheSet({
+                    key: cacheKey,
+                    createdAt: Date.now(),
+                    indexMtimeMs,
+                    results
+                });
+            }
+            return results;
+        }
+        finally {
+            index.close();
+        }
+    }
+    catch {
+        const fallbackResults = await searchInPacks(absoluteVaultPath, query, limit, agentId);
         if (cacheKey) {
             cacheSet({
                 key: cacheKey,
                 createdAt: Date.now(),
                 indexMtimeMs,
-                results
+                results: fallbackResults
             });
         }
-        return results;
-    }
-    finally {
-        index.close();
+        return fallbackResults;
     }
 };

package/dist/application/server/routes.js

@@ -1,14 +1,17 @@
 import { getBrokenLinksReport, getOrphansReport, getStats, validateVault } from '../analyze-vault.js';
 import { buildContextPackage } from '../build-context.js';
 import { getGraph } from '../get-graph.js';
+import { getGraphNode } from '../get-graph-node.js';
 import { getGraphLayout } from '../get-graph-layout.js';
 import { listAgents } from '../list-agents.js';
 import { listBacklinks, listLinks } from '../list-links.js';
+import { searchGraphNodeIds } from '../search-graph-node-ids.js';
 import { searchKnowledge } from '../search-knowledge.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../../infrastructure/config.js';
 import { createClientCss } from '../frontend/client-css.js';
 import { createClientHtml } from '../frontend/client-html.js';
 import { createClientJs } from '../frontend/client-js.js';
+import { createClientWorkerJs } from '../frontend/client-worker-js.js';
 import { contentTypes, createJsonResponse, isReadMethod, parsePositiveInteger } from './http.js';
 const readSearchMode = async (url) => {
     const config = await loadBrainlinkConfig();
@@ -49,6 +52,81 @@ const sameEntityTag = (candidate, signature) => {
     return decodeEntityTag(candidate) === signature;
 };
 const readAgentQuery = (url) => url.searchParams.get('agent') ?? undefined;
+const compactGraphLayoutThreshold = 12_000;
+const compactGraphLayoutEdgeLimit = 60_000;
+const compactGraphLayoutEdgeLimitFor = (nodeCount) => {
+    if (nodeCount > 100_000)
+        return 15_000;
+    if (nodeCount > 50_000)
+        return 22_000;
+    if (nodeCount > 25_000)
+        return 30_000;
+    return compactGraphLayoutEdgeLimit;
+};
+const edgeWeight = (weight) => Number.isFinite(weight) ? Number(weight) : 1;
+const edgeKey = (source, target, priority) => `${source}|${target}|${priority}`;
+const selectCompactEdges = (layout, limit) => {
+    const resolvedEdges = layout.edges.filter((edge) => typeof edge.target === 'string' && edge.target.length > 0);
+    if (resolvedEdges.length <= limit) {
+        return resolvedEdges;
+    }
+    const bestEdgeByEndpoint = new Map();
+    for (let index = 0; index < resolvedEdges.length; index += 1) {
+        const edge = resolvedEdges[index];
+        const endpoints = [edge.source, edge.target];
+        for (let endpointIndex = 0; endpointIndex < endpoints.length; endpointIndex += 1) {
+            const endpoint = endpoints[endpointIndex];
+            const previous = bestEdgeByEndpoint.get(endpoint);
+            if (!previous || edgeWeight(edge.weight) > edgeWeight(previous.weight)) {
+                bestEdgeByEndpoint.set(endpoint, edge);
+            }
+        }
+    }
+    const selected = new Map();
+    for (const edge of bestEdgeByEndpoint.values()) {
+        selected.set(edgeKey(edge.source, edge.target, edge.priority), edge);
+    }
+    if (selected.size > limit) {
+        return Array.from(selected.values())
+            .sort((left, right) => edgeWeight(right.weight) - edgeWeight(left.weight))
+            .slice(0, limit);
+    }
+    const byWeight = [...resolvedEdges].sort((left, right) => edgeWeight(right.weight) - edgeWeight(left.weight));
+    for (let index = 0; index < byWeight.length; index += 1) {
+        if (selected.size >= limit) {
+            break;
+        }
+        const edge = byWeight[index];
+        const key = edgeKey(edge.source, edge.target, edge.priority);
+        if (!selected.has(key)) {
+            selected.set(key, edge);
+        }
+    }
+    return Array.from(selected.values());
+};
+const stripLayoutContent = (layout) => ({
+    ...layout,
+    nodes: layout.nodes.map(({ content, ...node }) => node)
+});
+const compactLayoutPayload = (layout) => {
+    const edgeLimit = compactGraphLayoutEdgeLimitFor(layout.nodes.length);
+    const compactEdges = selectCompactEdges(layout, edgeLimit);
+    const compactNodes = layout.nodes.map((node) => [node.id, node.title, node.x, node.y, node.group, node.segment]);
+    const compactEdgeRows = compactEdges
+        .map((edge) => [edge.source, edge.target, edge.weight, edge.priority]);
+    return {
+        compact: true,
+        layout: {
+            nodes: compactNodes,
+            edges: compactEdgeRows
+        },
+        totals: {
+            nodes: layout.nodes.length,
+            edges: layout.edges.length
+        }
+    };
+};
+const encodeLayoutPayload = (layout) => layout.nodes.length > compactGraphLayoutThreshold ? compactLayoutPayload(layout) : { compact: false, layout: stripLayoutContent(layout) };
 export const route = async (request, url, vaultPath) => {
     if (isReadMethod(request) && (url.pathname === '/' || url.pathname === '/index.html')) {
         return createResponse(createClientHtml(), 200, contentTypes['.html']);
@@ -59,6 +137,9 @@ export const route = async (request, url, vaultPath) => {
     if (isReadMethod(request) && url.pathname === '/app.js') {
         return createResponse(createClientJs(), 200, contentTypes['.js']);
     }
+    if (isReadMethod(request) && url.pathname === '/app-worker.js') {
+        return createResponse(createClientWorkerJs(), 200, contentTypes['.js']);
+    }
     if (isReadMethod(request) && url.pathname === '/api/graph') {
         return createResponse(createJsonResponse(await getGraph(vaultPath, readAgentQuery(url))), 200, contentTypes['.json']);
     }
@@ -67,7 +148,7 @@ export const route = async (request, url, vaultPath) => {
         const requestEtags = request.headers['if-none-match'];
         const notModified = sameEntityTag(requestEtags, signature);
         const etag = encodeEntityTag(signature);
-        const body = createJsonResponse({ signature, layout });
+        const body = createJsonResponse({ signature, ...encodeLayoutPayload(layout) });
         const jsonResponse = createResponse(body, 200, contentTypes['.json']);
         const notModifiedResponse = createResponse('', 304, contentTypes['.json']);
         if (notModified) {
@@ -87,6 +168,26 @@ export const route = async (request, url, vaultPath) => {
             }
         };
     }
+    if (isReadMethod(request) && url.pathname === '/api/graph-node') {
+        const id = url.searchParams.get('id')?.trim() ?? '';
+        if (!id) {
+            return createResponse(createJsonResponse({ error: 'Missing id query parameter' }), 400, contentTypes['.json']);
+        }
+        const node = await getGraphNode(vaultPath, id, readAgentQuery(url));
+        if (!node) {
+            return createResponse(createJsonResponse({ error: 'Node not found' }), 404, contentTypes['.json']);
+        }
+        return createResponse(createJsonResponse({ node }), 200, contentTypes['.json']);
+    }
+    if (isReadMethod(request) && url.pathname === '/api/graph-filter') {
+        const query = url.searchParams.get('q')?.trim() ?? '';
+        const limit = parsePositiveInteger(url.searchParams.get('limit'), 1200);
+        if (!query) {
+            return createResponse(createJsonResponse({ query, nodeIds: [] }), 200, contentTypes['.json']);
+        }
+        const nodeIds = await searchGraphNodeIds(vaultPath, query, limit, readAgentQuery(url));
+        return createResponse(createJsonResponse({ query, nodeIds }), 200, contentTypes['.json']);
+    }
     if (isReadMethod(request) && url.pathname === '/api/agents') {
         return createResponse(createJsonResponse({ agents: await listAgents(vaultPath) }), 200, contentTypes['.json']);
     }