@andespindola/brainlink 0.1.0-beta.12 → 0.1.0-beta.14

package/README.md

@@ -57,9 +57,10 @@ LLMs do not have infinite context. Brainlink gives agents an external memory lay
 6. Brainlink returns compact, source-backed context.
 
 Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable index.
-Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup`. If the main SQLite file is corrupted, Brainlink automatically restores from snapshot (or recreates a clean index when no snapshot exists).
+Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup` plus rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. If the main SQLite file is corrupted, Brainlink automatically restores the newest valid snapshot (or recreates a clean index when no snapshot exists).
 After each index run, Brainlink also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`. If SQLite is unavailable, search falls back to these packs automatically.
 Pack decryption uses a Brainlink key from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when explicitly configured.
+On upgrade, if a legacy SQLite index exists without private packs, Brainlink imports index rows into `.blpk` automatically on first search/context access.
 
 ## Features
 

package/dist/application/analyze-vault.js

@@ -1,5 +1,5 @@
 import { stat } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
+import { existsSync, readdirSync } from 'node:fs';
 import { performance } from 'node:perf_hooks';
 import { join } from 'node:path';
 import { validateGraph, getBrokenLinks, getOrphanNodes, getVaultStats } from '../domain/graph-analysis.js';
@@ -97,7 +97,11 @@ export const doctorVault = async (vaultPath) => {
     const graph = await getGraphSummary(absoluteVaultPath);
     const validation = validateGraph(graph);
     const backupPath = join(absoluteVaultPath, '.brainlink', 'brainlink.db.backup');
+    const snapshotDirectory = join(absoluteVaultPath, '.brainlink', 'brainlink.db.backup.snapshots');
     const hasBackup = existsSync(backupPath);
+    const snapshotCount = existsSync(snapshotDirectory)
+        ? readdirSync(snapshotDirectory).filter((name) => name.endsWith('.db')).length
+        : 0;
     const backupReady = graph.nodes.length === 0 || hasBackup;
     const checks = [
         createCheck('vault', true, `Vault ready at ${absoluteVaultPath}`),
@@ -105,7 +109,9 @@ export const doctorVault = async (vaultPath) => {
         createCheck('index', graph.nodes.length > 0, `${graph.nodes.length} indexed documents found`),
         createCheck('broken-links', validation.brokenLinks.length === 0, `${validation.brokenLinks.length} broken links found`),
         createCheck('index-backup', backupReady, backupReady
-            ? (hasBackup ? 'SQLite recovery snapshot is available' : 'No index yet. Snapshot will be created after first indexing run')
+            ? (hasBackup
+                ? `SQLite recovery snapshot is available (${snapshotCount} rotating snapshots)`
+                : 'No index yet. Snapshot will be created after first indexing run')
             : 'Recovery snapshot missing. Run blink index to create a rollback snapshot')
     ];
     const recommendations = files.length === 0 && graph.nodes.length === 0

package/dist/application/search-knowledge.js

@@ -1,7 +1,7 @@
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { searchInPacks } from '../infrastructure/search-packs.js';
+import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
 import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
@@ -47,6 +47,7 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
     const searchMode = sanitizeSearchMode(mode, config.defaultSearchMode);
+    await ensurePrivatePacksFromLegacyIndex(absoluteVaultPath);
     const cacheKey = searchMode === 'hybrid' ? toCacheKey(absoluteVaultPath, query, limit, agentId) : undefined;
     const indexMtimeMs = cacheKey ? await readIndexMtimeMs(absoluteVaultPath) : 0;
     const cached = cacheKey ? cacheGet(cacheKey, indexMtimeMs) : undefined;

package/dist/infrastructure/search-packs.js

@@ -1,6 +1,8 @@
+import Database from 'better-sqlite3';
 import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { existsSync } from 'node:fs';
 import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
@@ -8,6 +10,7 @@ const rowChunkSize = 5_000;
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
+const toDatabasePath = (vaultPath) => join(vaultPath, '.brainlink', 'brainlink.db');
 const parseRowsFromPack = async (vaultPath, content) => {
     const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
     return raw
@@ -29,6 +32,15 @@ const toRows = (documents) => documents.flatMap((document) => document.chunks.ma
 const writeManifest = async (vaultPath, manifest) => {
     await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
 };
+const parseTags = (value) => {
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : [];
+    }
+    catch {
+        return [];
+    }
+};
 const chunkRows = (rows, size) => {
     const chunks = [];
     for (let index = 0; index < rows.length; index += size) {
@@ -100,14 +112,15 @@ const sortedPackFiles = async (vaultPath) => {
         throw error;
     }
 };
-export const buildSearchPacks = async (vaultPath, documents) => {
+const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting) => {
     const directory = toPackDirectory(vaultPath);
-    const rows = toRows(documents);
     await mkdir(directory, { recursive: true });
-    const current = await readdir(directory);
-    await Promise.all(current
-        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
-        .map((name) => rm(join(directory, name), { force: true })));
+    if (clearExisting) {
+        const current = await readdir(directory);
+        await Promise.all(current
+            .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
+            .map((name) => rm(join(directory, name), { force: true })));
+    }
     const chunks = chunkRows(rows, rowChunkSize);
     await Promise.all(chunks.map(async (chunk, index) => {
         const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
@@ -127,6 +140,100 @@ export const buildSearchPacks = async (vaultPath, documents) => {
         recordCount: rows.length
     };
 };
+const tableExists = (database, table) => {
+    const row = database.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table);
+    return row?.name === table;
+};
+const tableColumns = (database, table) => {
+    const rows = database.prepare(`SELECT name FROM pragma_table_info('${table.replaceAll("'", "''")}')`).all();
+    return new Set(rows.map((row) => row.name));
+};
+const loadRowsFromLegacySqlite = (vaultPath) => {
+    const databasePath = toDatabasePath(vaultPath);
+    if (!existsSync(databasePath)) {
+        return [];
+    }
+    const database = new Database(databasePath, { readonly: true, fileMustExist: true });
+    try {
+        if (!tableExists(database, 'documents') || !tableExists(database, 'chunks')) {
+            return [];
+        }
+        const documentColumns = tableColumns(database, 'documents');
+        const chunkColumns = tableColumns(database, 'chunks');
+        if (!documentColumns.has('id') || !documentColumns.has('title') || !chunkColumns.has('document_id')) {
+            return [];
+        }
+        const agentExpr = documentColumns.has('agent_id') ? 'documents.agent_id' : "'shared'";
+        const pathExpr = documentColumns.has('path') ? 'documents.path' : "documents.title";
+        const tagsExpr = documentColumns.has('tags_json') ? 'documents.tags_json' : "'[]'";
+        const chunkIdExpr = chunkColumns.has('id') ? 'chunks.id' : "documents.id || ':' || chunks.rowid";
+        const chunkContentExpr = chunkColumns.has('content')
+            ? 'chunks.content'
+            : documentColumns.has('content')
+                ? 'documents.content'
+                : "''";
+        const chunkOrderExpr = chunkColumns.has('ordinal') ? 'chunks.ordinal' : 'chunks.rowid';
+        const statement = database.prepare(`
+      SELECT
+        documents.id AS document_id,
+        ${agentExpr} AS agent_id,
+        documents.title AS title,
+        ${pathExpr} AS path,
+        ${chunkIdExpr} AS chunk_id,
+        ${chunkContentExpr} AS content,
+        ${tagsExpr} AS tags_json
+      FROM chunks
+      JOIN documents ON documents.id = chunks.document_id
+      ORDER BY documents.title, ${chunkOrderExpr}
+    `);
+        const rows = statement.all();
+        return rows.map((row) => ({
+            documentId: row.document_id,
+            agentId: typeof row.agent_id === 'string' && row.agent_id.length > 0 ? row.agent_id : 'shared',
+            title: row.title,
+            path: row.path,
+            chunkId: row.chunk_id,
+            content: row.content ?? '',
+            tags: parseTags(row.tags_json)
+        }));
+    }
+    finally {
+        database.close();
+    }
+};
+export const buildSearchPacks = async (vaultPath, documents) => {
+    return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true);
+};
+export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
+    const files = await sortedPackFiles(vaultPath);
+    if (files.some((file) => file.endsWith('.blpk'))) {
+        return { imported: false };
+    }
+    const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
+    if (legacyPackFiles.length > 0) {
+        const rows = [];
+        for (const file of legacyPackFiles) {
+            const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+            rows.push(...parsed);
+        }
+        const report = await writeRowsAsPrivatePacks(vaultPath, rows, true);
+        return {
+            imported: true,
+            source: 'legacy-packs',
+            ...report
+        };
+    }
+    const legacyRows = loadRowsFromLegacySqlite(vaultPath);
+    if (legacyRows.length === 0) {
+        return { imported: false };
+    }
+    const report = await writeRowsAsPrivatePacks(vaultPath, legacyRows, true);
+    return {
+        imported: true,
+        source: 'legacy-sqlite',
+        ...report
+    };
+};
 export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     const normalizedAgent = agentId?.trim();
     const tokens = tokenize(query);

package/dist/infrastructure/sqlite/recovery.js

@@ -1,6 +1,6 @@
 import Database from 'better-sqlite3';
-import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, unlinkSync } from 'node:fs';
-import { dirname } from 'node:path';
+import { copyFileSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { basename, dirname, join } from 'node:path';
 const sqliteCorruptionHints = [
     'database disk image is malformed',
     'file is not a database',
@@ -8,6 +8,7 @@ const sqliteCorruptionHints = [
     'malformed database schema',
     'sqlite quick_check failed'
 ];
+const maxSnapshotFiles = 24;
 const normalizeMessage = (error) => error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
 const isSqliteCorruptionError = (error) => sqliteCorruptionHints.some((hint) => normalizeMessage(error).includes(hint));
 const safeUnlink = (path) => {
@@ -39,15 +40,19 @@ const archiveCorruptedDatabase = (databasePath) => {
     const archivedPath = `${databasePath}.corrupt-${Date.now()}`;
     renameSync(databasePath, archivedPath);
 };
-const restoreFromBackupOrReset = (databasePath, backupPath) => {
-    clearSidecars(databasePath);
-    archiveCorruptedDatabase(databasePath);
-    if (existsSync(backupPath)) {
-        copyFileSync(backupPath, databasePath);
-        clearSidecars(databasePath);
+const snapshotDirectoryPath = (backupPath) => join(dirname(backupPath), `${basename(backupPath)}.snapshots`);
+const snapshotFileName = () => `snapshot-${new Date().toISOString().replace(/[:.]/g, '-')}.db`;
+const cleanupSnapshotOverflow = (backupPath) => {
+    const directory = snapshotDirectoryPath(backupPath);
+    if (!existsSync(directory)) {
         return;
     }
-    rmSync(databasePath, { force: true });
+    const snapshots = readdirSync(directory)
+        .filter((name) => name.endsWith('.db'))
+        .sort((left, right) => right.localeCompare(left));
+    snapshots.slice(maxSnapshotFiles).forEach((name) => {
+        rmSync(join(directory, name), { force: true });
+    });
 };
 const openCheckedDatabase = (databasePath) => {
     const database = new Database(databasePath);
@@ -60,13 +65,88 @@ const openCheckedDatabase = (databasePath) => {
     }
     return database;
 };
+const isValidDatabaseSnapshot = (path) => {
+    if (!existsSync(path)) {
+        return false;
+    }
+    try {
+        const size = statSync(path).size;
+        if (size <= 0) {
+            return false;
+        }
+    }
+    catch {
+        return false;
+    }
+    try {
+        const database = new Database(path);
+        try {
+            assertQuickCheck(database);
+            return true;
+        }
+        finally {
+            database.close();
+        }
+    }
+    catch {
+        return false;
+    }
+};
+const candidateBackupFiles = (backupPath) => {
+    const directory = snapshotDirectoryPath(backupPath);
+    const snapshots = existsSync(directory)
+        ? readdirSync(directory)
+            .filter((name) => name.endsWith('.db'))
+            .sort((left, right) => right.localeCompare(left))
+            .map((name) => join(directory, name))
+        : [];
+    return [backupPath, ...snapshots];
+};
+const ensureSnapshotDirectory = (backupPath) => {
+    mkdirSync(snapshotDirectoryPath(backupPath), { recursive: true, mode: 0o700 });
+};
+const writeRecoveryMarker = (backupPath, restoredFrom) => {
+    const markerPath = join(dirname(backupPath), 'recovery-last-restore.json');
+    const payload = {
+        restoredAt: new Date().toISOString(),
+        restoredFrom
+    };
+    writeFileSync(markerPath, `${JSON.stringify(payload, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+};
+const restoreFromBackupOrReset = (databasePath, backupPath) => {
+    clearSidecars(databasePath);
+    archiveCorruptedDatabase(databasePath);
+    for (const candidate of candidateBackupFiles(backupPath)) {
+        if (!isValidDatabaseSnapshot(candidate)) {
+            continue;
+        }
+        copyFileSync(candidate, databasePath);
+        clearSidecars(databasePath);
+        if (isValidDatabaseSnapshot(databasePath)) {
+            writeRecoveryMarker(backupPath, candidate);
+            return;
+        }
+    }
+    rmSync(databasePath, { force: true });
+};
 export const createRecoverySnapshot = (database, backupPath) => {
     const backupDirectory = dirname(backupPath);
     const tempBackupPath = `${backupPath}.tmp`;
+    const snapshotDirectory = snapshotDirectoryPath(backupPath);
+    const snapshotPath = join(snapshotDirectory, snapshotFileName());
     mkdirSync(backupDirectory, { recursive: true });
+    ensureSnapshotDirectory(backupPath);
     rmSync(tempBackupPath, { force: true });
+    try {
+        database.pragma('wal_checkpoint(PASSIVE)');
+    }
+    catch {
+        // Checkpoint is best-effort.
+    }
     database.prepare('VACUUM INTO ?').run(tempBackupPath);
     renameSync(tempBackupPath, backupPath);
+    copyFileSync(backupPath, snapshotPath);
+    cleanupSnapshotOverflow(backupPath);
 };
 export const openDatabaseWithRecovery = (databasePath, backupPath) => {
     mkdirSync(dirname(databasePath), { recursive: true });

package/docs/AGENT_USAGE.md

@@ -634,9 +634,10 @@ GET  /api/validate
 
 The HTTP API is read-only. Use the CLI for writes and indexing.
 
-Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores from snapshot automatically or recreates a clean index if no snapshot exists yet.
+Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup` and rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores the newest valid snapshot automatically or recreates a clean index if no snapshot exists yet.
 Indexing also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
 Pack decryption keys are resolved from `$BRAINLINK_HOME/keys` (or `BRAINLINK_SEARCH_PACK_KEY` when explicitly set).
+For legacy installations, when SQLite already exists but private packs are missing, Brainlink auto-imports index context rows from `brainlink.db` into `.blpk` on first retrieval.
 
 ## Agent Integration Contract
 

package/docs/ARCHITECTURE.md

@@ -300,9 +300,10 @@ Markdown keeps the system portable, inspectable, Git-friendly, and compatible wi
 
 SQLite gives fast local search, local vector storage and rebuildable retrieval without forcing users to run external infrastructure.
 Hybrid retrieval also uses a short-lived in-memory cache keyed by vault/query/agent and invalidated by index file mtime to reduce repeated query latency.
-Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores from snapshot automatically before reopening the index.
+Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) plus rotating point-in-time snapshots (`.brainlink/brainlink.db.backup.snapshots/`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores the newest valid snapshot automatically before reopening the index.
 Indexing additionally exports private encrypted pack files (`.brainlink/search-packs/*.blpk`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
 Pack encryption keys are resolved from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when configured.
+Legacy upgrades are automatic: when a vault has `brainlink.db` but no `.blpk` packs yet, Brainlink extracts indexed context rows from SQLite and writes private packs on first retrieval flow.
 
 ### CLI First
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.12",
+  "version": "0.1.0-beta.14",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",