@andespindola/brainlink 0.1.0-beta.11 → 0.1.0-beta.13

package/README.md

@@ -57,8 +57,9 @@ LLMs do not have infinite context. Brainlink gives agents an external memory lay
 6. Brainlink returns compact, source-backed context.
 
 Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable index.
-Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup`. If the main SQLite file is corrupted, Brainlink automatically restores from snapshot (or recreates a clean index when no snapshot exists).
-After each index run, Brainlink also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`. If SQLite is unavailable, search falls back to these packs automatically.
+Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup` plus rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. If the main SQLite file is corrupted, Brainlink automatically restores the newest valid snapshot (or recreates a clean index when no snapshot exists).
+After each index run, Brainlink also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`. If SQLite is unavailable, search falls back to these packs automatically.
+Pack decryption uses a Brainlink key from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when explicitly configured.
 
 ## Features
 

package/dist/application/analyze-vault.js

@@ -1,5 +1,5 @@
 import { stat } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
+import { existsSync, readdirSync } from 'node:fs';
 import { performance } from 'node:perf_hooks';
 import { join } from 'node:path';
 import { validateGraph, getBrokenLinks, getOrphanNodes, getVaultStats } from '../domain/graph-analysis.js';
@@ -97,7 +97,11 @@ export const doctorVault = async (vaultPath) => {
     const graph = await getGraphSummary(absoluteVaultPath);
     const validation = validateGraph(graph);
     const backupPath = join(absoluteVaultPath, '.brainlink', 'brainlink.db.backup');
+    const snapshotDirectory = join(absoluteVaultPath, '.brainlink', 'brainlink.db.backup.snapshots');
     const hasBackup = existsSync(backupPath);
+    const snapshotCount = existsSync(snapshotDirectory)
+        ? readdirSync(snapshotDirectory).filter((name) => name.endsWith('.db')).length
+        : 0;
     const backupReady = graph.nodes.length === 0 || hasBackup;
     const checks = [
         createCheck('vault', true, `Vault ready at ${absoluteVaultPath}`),
@@ -105,7 +109,9 @@ export const doctorVault = async (vaultPath) => {
         createCheck('index', graph.nodes.length > 0, `${graph.nodes.length} indexed documents found`),
         createCheck('broken-links', validation.brokenLinks.length === 0, `${validation.brokenLinks.length} broken links found`),
         createCheck('index-backup', backupReady, backupReady
-            ? (hasBackup ? 'SQLite recovery snapshot is available' : 'No index yet. Snapshot will be created after first indexing run')
+            ? (hasBackup
+                ? `SQLite recovery snapshot is available (${snapshotCount} rotating snapshots)`
+                : 'No index yet. Snapshot will be created after first indexing run')
             : 'Recovery snapshot missing. Run blink index to create a rollback snapshot')
     ];
     const recommendations = files.length === 0 && graph.nodes.length === 0

package/dist/infrastructure/private-pack-codec.js

@@ -0,0 +1,73 @@
+import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
+import { brotliCompressSync, brotliDecompressSync } from 'node:zlib';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { getBrainlinkHomePath } from './paths.js';
+const magic = Buffer.from('BLPK2', 'ascii');
+const version = 1;
+const nonceLength = 12;
+const authTagLength = 16;
+const algorithm = 'aes-256-gcm';
+const keyFilePath = (vaultPath) => {
+    const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
+    return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
+};
+const deriveKeyFromSecret = (secret) => createHash('sha256').update(secret, 'utf8').digest();
+const readOrCreateKey = async (vaultPath) => {
+    const envSecret = process.env.BRAINLINK_SEARCH_PACK_KEY?.trim();
+    if (envSecret && envSecret.length > 0) {
+        return deriveKeyFromSecret(envSecret);
+    }
+    const path = keyFilePath(vaultPath);
+    try {
+        const existing = (await readFile(path, 'utf8')).trim();
+        if (existing.length > 0) {
+            return deriveKeyFromSecret(existing);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+    const secret = randomBytes(48).toString('base64url');
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    await writeFile(path, `${secret}\n`, { encoding: 'utf8', mode: 0o600 });
+    return deriveKeyFromSecret(secret);
+};
+const parseHeader = (payload) => {
+    if (payload.length < magic.length + 1 + nonceLength + authTagLength) {
+        throw new Error('Invalid private pack payload: too short.');
+    }
+    const payloadMagic = payload.subarray(0, magic.length);
+    const payloadVersion = payload[magic.length];
+    if (!payloadMagic.equals(magic) || payloadVersion !== version) {
+        throw new Error('Invalid private pack payload: unsupported format.');
+    }
+    const nonceStart = magic.length + 1;
+    const authTagStart = nonceStart + nonceLength;
+    const dataStart = authTagStart + authTagLength;
+    return {
+        nonce: payload.subarray(nonceStart, authTagStart),
+        authTag: payload.subarray(authTagStart, dataStart),
+        ciphertext: payload.subarray(dataStart)
+    };
+};
+export const encodePrivatePack = async (vaultPath, content) => {
+    const key = await readOrCreateKey(vaultPath);
+    const nonce = randomBytes(nonceLength);
+    const compressed = brotliCompressSync(content);
+    const cipher = createCipheriv(algorithm, key, nonce);
+    const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+    return Buffer.concat([magic, Buffer.from([version]), nonce, authTag, ciphertext]);
+};
+export const decodePrivatePack = async (vaultPath, payload) => {
+    const key = await readOrCreateKey(vaultPath);
+    const { nonce, authTag, ciphertext } = parseHeader(payload);
+    const decipher = createDecipheriv(algorithm, key, nonce);
+    decipher.setAuthTag(authTag);
+    const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    return brotliDecompressSync(compressed);
+};
+export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);

package/dist/infrastructure/search-packs.js

@@ -1,18 +1,22 @@
-import { gunzipSync, gzipSync } from 'node:zlib';
+import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
 const rowChunkSize = 5_000;
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
-const parseRowsFromPack = (content) => gunzipSync(content)
-    .toString('utf8')
-    .split('\n')
-    .map((line) => line.trim())
-    .filter((line) => line.length > 0)
-    .map((line) => JSON.parse(line));
+const parseRowsFromPack = async (vaultPath, content) => {
+    const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
+    return raw
+        .toString('utf8')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((line) => JSON.parse(line));
+};
 const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
     documentId: document.document.id,
     agentId: document.document.agentId,
@@ -86,7 +90,7 @@ const sortedPackFiles = async (vaultPath) => {
     try {
         const files = await readdir(toPackDirectory(vaultPath));
         return files
-            .filter((file) => file.endsWith('.jsonl.gz'))
+            .filter((file) => file.endsWith('.blpk') || file.endsWith('.jsonl.gz'))
             .sort((left, right) => left.localeCompare(right));
     }
     catch (error) {
@@ -102,20 +106,21 @@ export const buildSearchPacks = async (vaultPath, documents) => {
     await mkdir(directory, { recursive: true });
     const current = await readdir(directory);
     await Promise.all(current
-        .filter((name) => name.endsWith('.jsonl.gz') || name === manifestFileName)
+        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
         .map((name) => rm(join(directory, name), { force: true })));
     const chunks = chunkRows(rows, rowChunkSize);
     await Promise.all(chunks.map(async (chunk, index) => {
-        const fileName = `pack-${String(index + 1).padStart(4, '0')}.jsonl.gz`;
+        const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
         const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
-        const compressed = gzipSync(Buffer.from(serialized, 'utf8'), { level: 6 });
+        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'));
         await writeFile(join(directory, fileName), compressed);
     }));
     await writeManifest(vaultPath, {
-        version: 1,
+        version: 2,
         createdAt: new Date().toISOString(),
         packCount: chunks.length,
-        recordCount: rows.length
+        recordCount: rows.length,
+        format: 'private-v2'
     });
     return {
         packCount: chunks.length,
@@ -134,7 +139,7 @@ export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     }
     const scored = [];
     for (const file of files) {
-        const rows = parseRowsFromPack(await readFile(join(toPackDirectory(vaultPath), file)));
+        const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
         rows.forEach((row) => {
             if (normalizedAgent && row.agentId !== normalizedAgent) {
                 return;

package/dist/infrastructure/sqlite/recovery.js

@@ -1,6 +1,6 @@
 import Database from 'better-sqlite3';
-import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, unlinkSync } from 'node:fs';
-import { dirname } from 'node:path';
+import { copyFileSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { basename, dirname, join } from 'node:path';
 const sqliteCorruptionHints = [
     'database disk image is malformed',
     'file is not a database',
@@ -8,6 +8,7 @@ const sqliteCorruptionHints = [
     'malformed database schema',
     'sqlite quick_check failed'
 ];
+const maxSnapshotFiles = 24;
 const normalizeMessage = (error) => error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
 const isSqliteCorruptionError = (error) => sqliteCorruptionHints.some((hint) => normalizeMessage(error).includes(hint));
 const safeUnlink = (path) => {
@@ -39,15 +40,19 @@ const archiveCorruptedDatabase = (databasePath) => {
     const archivedPath = `${databasePath}.corrupt-${Date.now()}`;
     renameSync(databasePath, archivedPath);
 };
-const restoreFromBackupOrReset = (databasePath, backupPath) => {
-    clearSidecars(databasePath);
-    archiveCorruptedDatabase(databasePath);
-    if (existsSync(backupPath)) {
-        copyFileSync(backupPath, databasePath);
-        clearSidecars(databasePath);
+const snapshotDirectoryPath = (backupPath) => join(dirname(backupPath), `${basename(backupPath)}.snapshots`);
+const snapshotFileName = () => `snapshot-${new Date().toISOString().replace(/[:.]/g, '-')}.db`;
+const cleanupSnapshotOverflow = (backupPath) => {
+    const directory = snapshotDirectoryPath(backupPath);
+    if (!existsSync(directory)) {
         return;
     }
-    rmSync(databasePath, { force: true });
+    const snapshots = readdirSync(directory)
+        .filter((name) => name.endsWith('.db'))
+        .sort((left, right) => right.localeCompare(left));
+    snapshots.slice(maxSnapshotFiles).forEach((name) => {
+        rmSync(join(directory, name), { force: true });
+    });
 };
 const openCheckedDatabase = (databasePath) => {
     const database = new Database(databasePath);
@@ -60,13 +65,88 @@ const openCheckedDatabase = (databasePath) => {
     }
     return database;
 };
+const isValidDatabaseSnapshot = (path) => {
+    if (!existsSync(path)) {
+        return false;
+    }
+    try {
+        const size = statSync(path).size;
+        if (size <= 0) {
+            return false;
+        }
+    }
+    catch {
+        return false;
+    }
+    try {
+        const database = new Database(path);
+        try {
+            assertQuickCheck(database);
+            return true;
+        }
+        finally {
+            database.close();
+        }
+    }
+    catch {
+        return false;
+    }
+};
+const candidateBackupFiles = (backupPath) => {
+    const directory = snapshotDirectoryPath(backupPath);
+    const snapshots = existsSync(directory)
+        ? readdirSync(directory)
+            .filter((name) => name.endsWith('.db'))
+            .sort((left, right) => right.localeCompare(left))
+            .map((name) => join(directory, name))
+        : [];
+    return [backupPath, ...snapshots];
+};
+const ensureSnapshotDirectory = (backupPath) => {
+    mkdirSync(snapshotDirectoryPath(backupPath), { recursive: true, mode: 0o700 });
+};
+const writeRecoveryMarker = (backupPath, restoredFrom) => {
+    const markerPath = join(dirname(backupPath), 'recovery-last-restore.json');
+    const payload = {
+        restoredAt: new Date().toISOString(),
+        restoredFrom
+    };
+    writeFileSync(markerPath, `${JSON.stringify(payload, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+};
+const restoreFromBackupOrReset = (databasePath, backupPath) => {
+    clearSidecars(databasePath);
+    archiveCorruptedDatabase(databasePath);
+    for (const candidate of candidateBackupFiles(backupPath)) {
+        if (!isValidDatabaseSnapshot(candidate)) {
+            continue;
+        }
+        copyFileSync(candidate, databasePath);
+        clearSidecars(databasePath);
+        if (isValidDatabaseSnapshot(databasePath)) {
+            writeRecoveryMarker(backupPath, candidate);
+            return;
+        }
+    }
+    rmSync(databasePath, { force: true });
+};
 export const createRecoverySnapshot = (database, backupPath) => {
     const backupDirectory = dirname(backupPath);
     const tempBackupPath = `${backupPath}.tmp`;
+    const snapshotDirectory = snapshotDirectoryPath(backupPath);
+    const snapshotPath = join(snapshotDirectory, snapshotFileName());
     mkdirSync(backupDirectory, { recursive: true });
+    ensureSnapshotDirectory(backupPath);
     rmSync(tempBackupPath, { force: true });
+    try {
+        database.pragma('wal_checkpoint(PASSIVE)');
+    }
+    catch {
+        // Checkpoint is best-effort.
+    }
     database.prepare('VACUUM INTO ?').run(tempBackupPath);
     renameSync(tempBackupPath, backupPath);
+    copyFileSync(backupPath, snapshotPath);
+    cleanupSnapshotOverflow(backupPath);
 };
 export const openDatabaseWithRecovery = (databasePath, backupPath) => {
     mkdirSync(dirname(databasePath), { recursive: true });

package/docs/AGENT_USAGE.md

@@ -634,8 +634,9 @@ GET  /api/validate
 
 The HTTP API is read-only. Use the CLI for writes and indexing.
 
-Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores from snapshot automatically or recreates a clean index if no snapshot exists yet.
-Indexing also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
+Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup` and rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores the newest valid snapshot automatically or recreates a clean index if no snapshot exists yet.
+Indexing also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
+Pack decryption keys are resolved from `$BRAINLINK_HOME/keys` (or `BRAINLINK_SEARCH_PACK_KEY` when explicitly set).
 
 ## Agent Integration Contract
 

package/docs/ARCHITECTURE.md

@@ -300,8 +300,9 @@ Markdown keeps the system portable, inspectable, Git-friendly, and compatible wi
 
 SQLite gives fast local search, local vector storage and rebuildable retrieval without forcing users to run external infrastructure.
 Hybrid retrieval also uses a short-lived in-memory cache keyed by vault/query/agent and invalidated by index file mtime to reduce repeated query latency.
-Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores from snapshot automatically before reopening the index.
-Indexing additionally exports compressed pack files (`.brainlink/search-packs/*.jsonl.gz`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
+Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) plus rotating point-in-time snapshots (`.brainlink/brainlink.db.backup.snapshots/`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores the newest valid snapshot automatically before reopening the index.
+Indexing additionally exports private encrypted pack files (`.brainlink/search-packs/*.blpk`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
+Pack encryption keys are resolved from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when configured.
 
 ### CLI First
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.11",
+  "version": "0.1.0-beta.13",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",