@andespindola/brainlink 0.1.0-beta.14 → 0.1.0-beta.140

package/dist/application/import-legacy-sqlite.js

@@ -0,0 +1,296 @@
+import { execFile } from 'node:child_process';
+import { access } from 'node:fs/promises';
+import { basename, extname, join, relative, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { promisify } from 'node:util';
+import { extractTags, extractWikiLinks } from '../domain/markdown.js';
+import { sanitizeAgentId, sharedAgentId } from '../domain/agents.js';
+import { ensureVault, listVaultFiles, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
+import { getBrainlinkHomePath } from '../infrastructure/paths.js';
+const execFileAsync = promisify(execFile);
+const fieldSeparator = '\u001f';
+const rowSeparator = '\u001e';
+const contentColumnCandidates = ['content', 'markdown', 'body', 'text', 'note'];
+const titleColumnCandidates = ['title', 'note_title', 'name', 'headline'];
+const pathColumnCandidates = ['path', 'file_path', 'filepath', 'source_path', 'source'];
+const agentColumnCandidates = ['agent', 'agent_id', 'namespace', 'scope'];
+const tagColumnCandidates = ['tags', 'tag_list', 'keywords'];
+const createdColumnCandidates = ['created_at', 'createdat', 'created', 'ctime'];
+const updatedColumnCandidates = ['updated_at', 'updatedat', 'updated', 'mtime'];
+const systemHubTitle = 'Memory Hub';
+const systemRootTitle = 'Knowledge Root';
+const normalizeTitle = (title) => title.trim().replace(/\.md$/i, '').toLowerCase();
+const slugify = (title) => title
+    .normalize('NFKD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+const quoteIdentifier = (value) => `"${value.replaceAll('"', '""')}"`;
+const pickColumn = (columns, candidates) => {
+    const byLower = new Map(columns.map((column) => [column.toLowerCase(), column]));
+    return candidates.map((candidate) => byLower.get(candidate)).find((column) => Boolean(column)) ?? null;
+};
+const parseDelimitedRows = (rawOutput) => {
+    const normalized = rawOutput.trim();
+    if (normalized.length === 0) {
+        return [];
+    }
+    return normalized
+        .split(rowSeparator)
+        .map((row) => row.trim())
+        .filter(Boolean)
+        .map((row) => row.split(fieldSeparator));
+};
+const runSqliteQuery = async (databasePath, sql) => {
+    const baseArgs = ['-noheader', '-separator', fieldSeparator, '-newline', rowSeparator, '-cmd', '.timeout 5000'];
+    const runQuery = async (args) => {
+        const { stdout } = await execFileAsync('sqlite3', [...args, sql], { maxBuffer: 1024 * 1024 * 64 });
+        return parseDelimitedRows(stdout);
+    };
+    try {
+        return await runQuery(['--readonly', ...baseArgs, databasePath]);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        const lower = message.toLowerCase();
+        if (lower.includes('enoent') || lower.includes('not found')) {
+            throw new Error('sqlite3 CLI was not found. Install sqlite3 to use db-import.');
+        }
+        if (lower.includes('database is locked') || lower.includes('(5)')) {
+            try {
+                const uri = pathToFileURL(databasePath);
+                uri.search = 'mode=ro&immutable=1';
+                return await runQuery(['-uri', ...baseArgs, uri.toString()]);
+            }
+            catch (fallbackError) {
+                const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
+                throw new Error(`Unable to read SQLite database (locked). Close writers (server/watch/mcp) or rerun with DB idle. Details: ${fallbackMessage}`);
+            }
+        }
+        throw new Error(`Unable to read SQLite database: ${message}`);
+    }
+};
+const detectLegacyDbPath = async (vaultPath, explicitPath) => {
+    if (explicitPath) {
+        return resolve(explicitPath);
+    }
+    const vaultRoot = await ensureVault(vaultPath);
+    const candidates = [
+        join(vaultRoot, '.brainlink', 'brainlink.db'),
+        join(vaultRoot, '.brainlink', 'index.db'),
+        join(getBrainlinkHomePath(), 'brainlink.db'),
+        join(getBrainlinkHomePath(), 'vault', '.brainlink', 'brainlink.db')
+    ];
+    for (const candidate of candidates) {
+        try {
+            await access(candidate);
+            return candidate;
+        }
+        catch { }
+    }
+    throw new Error(`No legacy SQLite database found. Checked: ${candidates.join(', ')}. Use --db <path-to-db> to import explicitly.`);
+};
+const listTables = async (dbPath) => {
+    const rows = await runSqliteQuery(dbPath, `SELECT name
+     FROM sqlite_master
+     WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
+     ORDER BY name`);
+    return rows.map((columns) => columns[0]).filter(Boolean);
+};
+const listColumns = async (dbPath, table) => {
+    const rows = await runSqliteQuery(dbPath, `PRAGMA table_info(${quoteIdentifier(table)})`);
+    return rows.map((columns) => columns[1]).filter(Boolean);
+};
+const tableScore = (columns) => {
+    const contentColumn = pickColumn(columns, contentColumnCandidates);
+    const titleColumn = pickColumn(columns, titleColumnCandidates);
+    const pathColumn = pickColumn(columns, pathColumnCandidates);
+    const agentColumn = pickColumn(columns, agentColumnCandidates);
+    return (contentColumn ? 6 : 0) + (titleColumn ? 4 : 0) + (pathColumn ? 2 : 0) + (agentColumn ? 1 : 0);
+};
+const detectTableMapping = async (dbPath, tableOverride) => {
+    const tables = await listTables(dbPath);
+    if (tables.length === 0) {
+        throw new Error('Legacy SQLite database has no readable tables.');
+    }
+    const mappings = await Promise.all(tables.map(async (table) => {
+        const columns = await listColumns(dbPath, table);
+        return {
+            table,
+            columns,
+            titleColumn: pickColumn(columns, titleColumnCandidates),
+            contentColumn: pickColumn(columns, contentColumnCandidates),
+            pathColumn: pickColumn(columns, pathColumnCandidates),
+            agentColumn: pickColumn(columns, agentColumnCandidates),
+            tagsColumn: pickColumn(columns, tagColumnCandidates),
+            createdColumn: pickColumn(columns, createdColumnCandidates),
+            updatedColumn: pickColumn(columns, updatedColumnCandidates),
+            score: tableScore(columns)
+        };
+    }));
+    if (tableOverride) {
+        const overridden = mappings.find((mapping) => mapping.table === tableOverride);
+        if (!overridden) {
+            throw new Error(`Table not found in SQLite database: ${tableOverride}`);
+        }
+        if (!overridden.contentColumn) {
+            throw new Error(`Table ${tableOverride} does not expose a readable content column.`);
+        }
+        return { mapping: overridden, detectedTables: tables };
+    }
+    const selected = [...mappings]
+        .filter((mapping) => mapping.contentColumn)
+        .sort((left, right) => right.score - left.score)[0];
+    if (!selected) {
+        throw new Error('Could not detect a legacy table with content column in SQLite database.');
+    }
+    return { mapping: selected, detectedTables: tables };
+};
+const hexExpression = (column) => column ? `hex(COALESCE(CAST(${quoteIdentifier(column)} AS BLOB), X''))` : `hex(X'')`;
+const decodeHexUtf8 = (value) => value ? Buffer.from(value, 'hex').toString('utf8') : '';
+const parseLegacyTags = (value) => Array.from(new Set(value
+    .split(/[\s,;|]+/)
+    .map((item) => item.trim().replace(/^#/, '').toLowerCase())
+    .filter((item) => /^[a-z0-9][a-z0-9_-]*$/i.test(item))));
+const titleFromPath = (pathValue) => basename(pathValue).replace(extname(pathValue), '').replace(/[-_]+/g, ' ').trim();
+const appendMissingTags = (content, tags) => {
+    if (tags.length === 0) {
+        return content;
+    }
+    const existingTags = new Set(extractTags(content).map((tag) => tag.toLowerCase()));
+    const missing = tags.filter((tag) => !existingTags.has(tag.toLowerCase()));
+    if (missing.length === 0) {
+        return content;
+    }
+    return `${content.trim()}\n\nTags: ${missing.map((tag) => `#${tag}`).join(' ')}`;
+};
+const buildNote = (title, content, agentId) => [
+    '---',
+    `title: "${title.replaceAll('"', '\\"')}"`,
+    `agent: "${agentId}"`,
+    '---',
+    '',
+    `# ${title}`,
+    '',
+    content.trim(),
+    ''
+].join('\n');
+const parseLegacyRow = (columns, rowIndex) => {
+    const [titleHex, contentHex, pathHex, agentHex, tagsHex] = columns;
+    const content = decodeHexUtf8(contentHex).trim();
+    const path = decodeHexUtf8(pathHex).trim();
+    const titleCandidate = decodeHexUtf8(titleHex).trim();
+    const fallbackTitleFromPath = path ? titleFromPath(path) : '';
+    const title = titleCandidate || fallbackTitleFromPath || `Imported Memory ${rowIndex + 1}`;
+    return {
+        title,
+        content,
+        path,
+        agent: decodeHexUtf8(agentHex).trim(),
+        tags: parseLegacyTags(decodeHexUtf8(tagsHex))
+    };
+};
+const noteRelativePath = (agentId, slug, suffix = 0) => `agents/${agentId}/${suffix > 0 ? `${slug}-${suffix + 1}` : slug || 'untitled'}.md`;
+const reserveUniquePath = (agentId, title, reserved) => {
+    const slug = slugify(title);
+    for (let suffix = 0; suffix < 10_000; suffix += 1) {
+        const relativePath = noteRelativePath(agentId, slug, suffix);
+        if (!reserved.has(relativePath)) {
+            reserved.add(relativePath);
+            return relativePath;
+        }
+    }
+    throw new Error(`Could not allocate unique path for imported note: ${title}`);
+};
+const ensureSystemNote = async (vaultPath, reserved, created, agentId, title, content, dryRun) => {
+    const filename = noteRelativePath(agentId, slugify(title));
+    if (reserved.has(filename)) {
+        return;
+    }
+    reserved.add(filename);
+    created.add(filename);
+    if (dryRun) {
+        return;
+    }
+    await writeMarkdownFile(vaultPath, filename, buildNote(title, content, agentId));
+};
+const applyConnectivityRule = async (vaultPath, reserved, created, title, content, agentId, dryRun) => {
+    const links = extractWikiLinks(content).filter((link) => normalizeTitle(link) !== normalizeTitle(title));
+    if (links.length > 0) {
+        return content.trim();
+    }
+    const normalized = normalizeTitle(title);
+    if (normalized === normalizeTitle(systemHubTitle)) {
+        await ensureSystemNote(vaultPath, reserved, created, agentId, systemRootTitle, `Entry point for agent memory. [[${systemHubTitle}]] #memory #root`, dryRun);
+        return `${content.trim()}\n\nRelated: [[${systemRootTitle}]]`;
+    }
+    await ensureSystemNote(vaultPath, reserved, created, agentId, systemHubTitle, 'Central memory index for this agent namespace. #memory #hub', dryRun);
+    return `${content.trim()}\n\nRelated: [[${systemHubTitle}]]`;
+};
+const importRowsFromMapping = async (vaultPath, dbPath, mapping, options, reserved) => {
+    const limit = Number.isFinite(options.limit) && (options.limit ?? 0) > 0 ? Math.floor(options.limit ?? 0) : undefined;
+    const sql = [
+        'SELECT',
+        `${hexExpression(mapping.titleColumn)} AS title_hex,`,
+        `${hexExpression(mapping.contentColumn)} AS content_hex,`,
+        `${hexExpression(mapping.pathColumn)} AS path_hex,`,
+        `${hexExpression(mapping.agentColumn)} AS agent_hex,`,
+        `${hexExpression(mapping.tagsColumn)} AS tags_hex,`,
+        `${hexExpression(mapping.createdColumn)} AS created_hex,`,
+        `${hexExpression(mapping.updatedColumn)} AS updated_hex`,
+        `FROM ${quoteIdentifier(mapping.table)}`,
+        ...(limit ? [`LIMIT ${limit}`] : [])
+    ].join(' ');
+    const rows = await runSqliteQuery(dbPath, sql);
+    const createdSystemNotes = new Set();
+    const importedFiles = [];
+    let imported = 0;
+    let skipped = 0;
+    for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
+        const row = parseLegacyRow(rows[rowIndex], rowIndex);
+        if (!row.content) {
+            skipped += 1;
+            continue;
+        }
+        const agentId = sanitizeAgentId(options.agentOverride || row.agent || sharedAgentId);
+        const filename = reserveUniquePath(agentId, row.title, reserved);
+        const mergedContent = appendMissingTags(row.content, row.tags);
+        const connectedContent = await applyConnectivityRule(vaultPath, reserved, createdSystemNotes, row.title, mergedContent, agentId, options.dryRun === true);
+        const note = buildNote(row.title, connectedContent, agentId);
+        if (options.dryRun !== true) {
+            await writeMarkdownFile(vaultPath, filename, note);
+        }
+        importedFiles.push(filename);
+        imported += 1;
+    }
+    return {
+        rowsRead: rows.length,
+        imported,
+        skipped,
+        createdSystemNotes: createdSystemNotes.size,
+        importedFiles
+    };
+};
+export const importLegacySqliteDatabase = async (vaultPath, options = {}) => {
+    const vault = await ensureVault(vaultPath);
+    const dbPath = await detectLegacyDbPath(vaultPath, options.dbPath);
+    const { mapping, detectedTables } = await detectTableMapping(dbPath, options.table);
+    const existingFiles = (await listVaultFiles(vaultPath))
+        .filter((path) => extname(path).toLowerCase() === '.md')
+        .map((path) => relative(vault, path));
+    const reserved = new Set(existingFiles);
+    const imported = await importRowsFromMapping(vaultPath, dbPath, mapping, options, reserved);
+    return {
+        vault,
+        dbPath,
+        table: mapping.table,
+        detectedTables,
+        rowsRead: imported.rowsRead,
+        imported: imported.imported,
+        skipped: imported.skipped,
+        createdSystemNotes: imported.createdSystemNotes,
+        dryRun: options.dryRun === true,
+        importedFiles: imported.importedFiles
+    };
+};

package/dist/application/index-vault.js

@@ -1,10 +1,12 @@
-import { createIndexedDocument, parseMarkdownDocument } from '../domain/markdown.js';
+import { readFile } from 'node:fs/promises';
+import { createIndexedDocument, graphLinkModelVersion, parseMarkdownDocument } from '../domain/markdown.js';
 import { sharedAgentId } from '../domain/agents.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig } from '../infrastructure/config.js';
-import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
-import { buildSearchPacks } from '../infrastructure/search-packs.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { ensureVault, readMarkdownFileSummaries } from '../infrastructure/file-system-vault.js';
+import { readIndexState, writeIndexState } from '../infrastructure/index-state.js';
+import { buildSearchPacks, ensureSearchPackManifest, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 const toTitleKey = (title) => title.toLowerCase();
 const appendTitleEntry = (map, document) => {
     const key = toTitleKey(document.title);
@@ -34,6 +36,9 @@ const createScopedTitleResolver = (document, titleMaps) => ({
     get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
 });
 const embedIndexedDocuments = async (documents, providerName) => {
+    if (documents.length === 0) {
+        return documents;
+    }
     const provider = createEmbeddingProvider(providerName);
     const chunks = documents.flatMap((document) => document.chunks);
     const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
@@ -47,34 +52,257 @@ const embedIndexedDocuments = async (documents, providerName) => {
         }))
     }));
 };
+const relinkIndexedDocument = (indexedDocument, titleMaps) => {
+    const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
+    return {
+        ...indexedDocument,
+        links: indexedDocument.links
+            .map((link) => ({
+            ...link,
+            toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
+        }))
+            .filter((link) => link.toDocumentId !== indexedDocument.document.id)
+    };
+};
+const toIndexResult = (documents) => ({
+    documentCount: documents.length,
+    chunkCount: documents.reduce((total, document) => total + document.chunks.length, 0),
+    linkCount: documents.reduce((total, document) => total + document.links.length, 0)
+});
+const toSnapshot = (summaries) => summaries.map((summary) => ({
+    path: summary.relativePath,
+    mtimeMs: summary.updatedAt.getTime(),
+    size: summary.size
+}));
+const createSnapshotMap = (snapshot) => new Map(snapshot.map((entry) => [entry.path, entry]));
+const readChangedDocuments = async (absoluteVaultPath, changedSummaries) => {
+    const parsed = await Promise.all(changedSummaries.map(async (summary) => parseMarkdownDocument({
+        absolutePath: summary.absolutePath,
+        vaultPath: absoluteVaultPath,
+        content: await readFile(summary.absolutePath, 'utf8'),
+        createdAt: summary.createdAt,
+        updatedAt: summary.updatedAt
+    })));
+    return new Map(parsed.map((document) => [document.path, document]));
+};
 export const indexVault = async (vaultPath) => {
+    return indexVaultWithOptions(vaultPath, {});
+};
+export const indexVaultWithOptions = async (vaultPath, options) => {
+    const startedAt = process.hrtime.bigint();
+    const elapsedMs = () => Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const emit = (phase, status, message, details) => {
+        options.onProgress?.({
+            phase,
+            status,
+            message,
+            elapsedMs: elapsedMs(),
+            timestamp: new Date().toISOString(),
+            details
+        });
+    };
+    emit('start', 'start', 'Indexing started');
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
-    const files = await readMarkdownFiles(absoluteVaultPath);
-    const documents = files.map((file) => parseMarkdownDocument({
-        absolutePath: file.absolutePath,
-        vaultPath: absoluteVaultPath,
-        content: file.content,
-        createdAt: file.createdAt,
-        updatedAt: file.updatedAt
-    }));
-    const titleMaps = createTitleMaps(documents);
-    const indexedDocuments = await embedIndexedDocuments(documents.map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider);
-    const index = openSqliteIndex(absoluteVaultPath);
+    emit('scan', 'start', 'Scanning markdown files');
+    const [summaries, previousState] = await Promise.all([
+        readMarkdownFileSummaries(absoluteVaultPath),
+        readIndexState(absoluteVaultPath)
+    ]);
+    emit('scan', 'finish', 'Scan complete', {
+        markdownFiles: summaries.length,
+        hasPreviousState: previousState != null
+    });
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        index.reset();
-        index.saveDocuments(indexedDocuments);
-        try {
-            await buildSearchPacks(absoluteVaultPath, indexedDocuments);
+        const existingIndexedDocuments = await index.getIndexedDocuments();
+        const existingByPath = new Map(existingIndexedDocuments.map((document) => [document.document.path, document]));
+        const currentSnapshot = toSnapshot(summaries);
+        const currentSnapshotMap = createSnapshotMap(currentSnapshot);
+        const previousSnapshotMap = createSnapshotMap(previousState?.files ?? []);
+        const settingsChanged = previousState == null ||
+            previousState.chunkSize !== config.chunkSize ||
+            previousState.embeddingProvider !== config.embeddingProvider ||
+            previousState.graphLinkModelVersion !== graphLinkModelVersion;
+        const packSettingsChanged = previousState == null ||
+            previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
+            previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
+            previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
+        const changedPaths = new Set();
+        for (let index = 0; index < summaries.length; index += 1) {
+            const summary = summaries[index];
+            const previous = previousSnapshotMap.get(summary.relativePath);
+            const changed = settingsChanged ||
+                previous == null ||
+                previous.mtimeMs !== summary.updatedAt.getTime() ||
+                previous.size !== summary.size ||
+                !existingByPath.has(summary.relativePath);
+            if (changed) {
+                changedPaths.add(summary.relativePath);
+            }
         }
-        catch {
-            // Pack generation is best-effort. SQLite index remains the primary path.
+        const hasDeletes = previousState
+            ? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
+            : false;
+        const manifestRecovery = await ensureSearchPackManifest(absoluteVaultPath);
+        if (changedPaths.size === 0 &&
+            !hasDeletes &&
+            existingIndexedDocuments.length === summaries.length &&
+            previousState != null) {
+            const result = {
+                ...toIndexResult(existingIndexedDocuments),
+                elapsedMs: elapsedMs(),
+                changedDocumentCount: 0,
+                packs: {
+                    rebuilt: false,
+                    reason: manifestRecovery.repaired ? 'No changes detected; pack manifest repaired' : 'No changes detected'
+                }
+            };
+            emit('complete', 'skip', 'Index skipped: no changes detected', {
+                elapsedMs: result.elapsedMs,
+                manifestRepaired: manifestRecovery.repaired,
+                manifestRecoverySource: manifestRecovery.source
+            });
+            return result;
         }
-        return {
-            documentCount: indexedDocuments.length,
-            chunkCount: indexedDocuments.reduce((total, document) => total + document.chunks.length, 0),
-            linkCount: indexedDocuments.reduce((total, document) => total + document.links.length, 0)
+        const changedSummaries = summaries.filter((summary) => changedPaths.has(summary.relativePath));
+        emit('parse', 'start', 'Parsing changed markdown files', {
+            changedFiles: changedSummaries.length
+        });
+        const changedDocumentsByPath = await readChangedDocuments(absoluteVaultPath, changedSummaries);
+        emit('parse', 'finish', 'Parse complete', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const documents = summaries.flatMap((summary) => {
+            const changed = changedDocumentsByPath.get(summary.relativePath);
+            if (changed) {
+                return [changed];
+            }
+            const existing = existingByPath.get(summary.relativePath);
+            return existing ? [existing.document] : [];
+        });
+        const titleMaps = createTitleMaps(documents);
+        emit('embed', 'start', 'Embedding changed chunks', {
+            changedDocuments: changedDocumentsByPath.size
+        });
+        const changedIndexedDocuments = changedDocumentsByPath.size > 0
+            ? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
+            : [];
+        emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
+            changedIndexedDocuments: changedIndexedDocuments.length
+        });
+        const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
+        const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
+        const indexedDocuments = documents.map((document) => {
+            const changed = changedIndexedByPath.get(document.path);
+            if (changed) {
+                return changed;
+            }
+            const existing = existingByPath.get(document.path);
+            if (!existing) {
+                return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
+            }
+            return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
+        });
+        emit('persist', 'start', 'Persisting index');
+        await index.reset();
+        await index.saveDocuments(indexedDocuments);
+        emit('persist', 'finish', 'Index persisted', {
+            indexedDocuments: indexedDocuments.length
+        });
+        const existingPackManifest = manifestRecovery.repaired || manifestRecovery.source === 'not-needed';
+        const changedCount = changedPaths.size;
+        const documentCount = Math.max(indexedDocuments.length, 1);
+        const changeRatio = changedCount / documentCount;
+        const previousPendingPackChanges = previousState?.pendingPackChanges ?? 0;
+        const pendingPackChanges = previousPendingPackChanges + changedCount;
+        const shouldRebuildPacks = !existingPackManifest ||
+            settingsChanged ||
+            packSettingsChanged ||
+            hasDeletes ||
+            changedCount >= 400 ||
+            changeRatio >= 0.04 ||
+            pendingPackChanges >= 1200;
+        let packResult;
+        const packReason = !existingPackManifest
+            ? 'Missing pack manifest'
+            : manifestRecovery.repaired
+                ? 'Pack manifest repaired from existing packs'
+                : settingsChanged
+                    ? 'Index settings changed'
+                    : packSettingsChanged
+                        ? 'Search pack settings changed'
+                        : hasDeletes
+                            ? 'Document deletions detected'
+                            : changedCount >= 400
+                                ? 'Changed file count threshold reached'
+                                : changeRatio >= 0.04
+                                    ? 'Change ratio threshold reached'
+                                    : pendingPackChanges >= 1200
+                                        ? 'Pending pack changes threshold reached'
+                                        : 'Pack rebuild skipped';
+        if (shouldRebuildPacks) {
+            emit('packs', 'start', 'Rebuilding compressed search packs', {
+                reason: packReason
+            });
+            try {
+                packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
+                emit('packs', 'finish', 'Compressed packs rebuilt', {
+                    reason: packReason,
+                    packCount: packResult.packCount,
+                    recordCount: packResult.recordCount,
+                    durationMs: packResult.durationMs,
+                    compressionRatio: packResult.compression.ratio
+                });
+            }
+            catch {
+                // Pack generation is best-effort. The JSON index remains the primary path.
+                emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
+                    reason: packReason
+                });
+            }
+        }
+        else {
+            emit('packs', 'skip', 'Pack rebuild not required', {
+                reason: packReason
+            });
+        }
+        const packsRebuilt = packResult != null;
+        const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
+        await writeIndexState(absoluteVaultPath, {
+            chunkSize: config.chunkSize,
+            embeddingProvider: config.embeddingProvider,
+            graphLinkModelVersion,
+            searchPackRowChunkSize: config.searchPack.rowChunkSize,
+            searchPackCompressionLevel: config.searchPack.compressionLevel,
+            searchPackUseDictionary: config.searchPack.useDictionary,
+            files: currentSnapshot,
+            pendingPackChanges: packsRebuilt ? 0 : pendingPackChanges
+        });
+        const result = {
+            ...toIndexResult(indexedDocuments),
+            elapsedMs: elapsedMs(),
+            changedDocumentCount: changedDocumentsByPath.size,
+            packs: {
+                rebuilt: packsRebuilt,
+                reason: packResultReason,
+                ...(packResult
+                    ? {
+                        packCount: packResult.packCount,
+                        recordCount: packResult.recordCount,
+                        durationMs: packResult.durationMs,
+                        compression: packResult.compression
+                    }
+                    : {})
+            }
         };
+        emit('complete', 'finish', 'Indexing complete', {
+            documentCount: result.documentCount,
+            chunkCount: result.chunkCount,
+            linkCount: result.linkCount,
+            elapsedMs: result.elapsedMs
+        });
+        return result;
     }
     finally {
         index.close();

package/dist/application/list-agents.js

@@ -1,10 +1,10 @@
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 export const listAgents = async (vaultPath) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listAgents();
+        return await index.listAgents();
     }
     finally {
         index.close();

package/dist/application/list-links.js

@@ -1,10 +1,10 @@
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
+import { openFileIndex } from '../infrastructure/file-index.js';
 export const listLinks = async (vaultPath, agentId) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listLinks(agentId);
+        return await index.listLinks(agentId);
     }
     finally {
         index.close();
@@ -12,9 +12,9 @@ export const listLinks = async (vaultPath, agentId) => {
 };
 export const listBacklinks = async (vaultPath, title, agentId) => {
     const absoluteVaultPath = await ensureVault(vaultPath);
-    const index = openSqliteIndex(absoluteVaultPath);
+    const index = openFileIndex(absoluteVaultPath);
     try {
-        return index.listBacklinks(title, agentId);
+        return await index.listBacklinks(title, agentId);
     }
     finally {
         index.close();