@andespindola/brainlink 0.1.0-beta.11 → 0.1.0-beta.12

package/README.md

@@ -58,7 +58,8 @@ LLMs do not have infinite context. Brainlink gives agents an external memory lay
 
 Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable index.
 Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup`. If the main SQLite file is corrupted, Brainlink automatically restores from snapshot (or recreates a clean index when no snapshot exists).
-After each index run, Brainlink also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`. If SQLite is unavailable, search falls back to these packs automatically.
+After each index run, Brainlink also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`. If SQLite is unavailable, search falls back to these packs automatically.
+Pack decryption uses a Brainlink key from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when explicitly configured.
 
 ## Features
 

package/dist/infrastructure/private-pack-codec.js

@@ -0,0 +1,73 @@
+import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
+import { brotliCompressSync, brotliDecompressSync } from 'node:zlib';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { getBrainlinkHomePath } from './paths.js';
+const magic = Buffer.from('BLPK2', 'ascii');
+const version = 1;
+const nonceLength = 12;
+const authTagLength = 16;
+const algorithm = 'aes-256-gcm';
+const keyFilePath = (vaultPath) => {
+    const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
+    return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
+};
+const deriveKeyFromSecret = (secret) => createHash('sha256').update(secret, 'utf8').digest();
+const readOrCreateKey = async (vaultPath) => {
+    const envSecret = process.env.BRAINLINK_SEARCH_PACK_KEY?.trim();
+    if (envSecret && envSecret.length > 0) {
+        return deriveKeyFromSecret(envSecret);
+    }
+    const path = keyFilePath(vaultPath);
+    try {
+        const existing = (await readFile(path, 'utf8')).trim();
+        if (existing.length > 0) {
+            return deriveKeyFromSecret(existing);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+    const secret = randomBytes(48).toString('base64url');
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    await writeFile(path, `${secret}\n`, { encoding: 'utf8', mode: 0o600 });
+    return deriveKeyFromSecret(secret);
+};
+const parseHeader = (payload) => {
+    if (payload.length < magic.length + 1 + nonceLength + authTagLength) {
+        throw new Error('Invalid private pack payload: too short.');
+    }
+    const payloadMagic = payload.subarray(0, magic.length);
+    const payloadVersion = payload[magic.length];
+    if (!payloadMagic.equals(magic) || payloadVersion !== version) {
+        throw new Error('Invalid private pack payload: unsupported format.');
+    }
+    const nonceStart = magic.length + 1;
+    const authTagStart = nonceStart + nonceLength;
+    const dataStart = authTagStart + authTagLength;
+    return {
+        nonce: payload.subarray(nonceStart, authTagStart),
+        authTag: payload.subarray(authTagStart, dataStart),
+        ciphertext: payload.subarray(dataStart)
+    };
+};
+export const encodePrivatePack = async (vaultPath, content) => {
+    const key = await readOrCreateKey(vaultPath);
+    const nonce = randomBytes(nonceLength);
+    const compressed = brotliCompressSync(content);
+    const cipher = createCipheriv(algorithm, key, nonce);
+    const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+    return Buffer.concat([magic, Buffer.from([version]), nonce, authTag, ciphertext]);
+};
+export const decodePrivatePack = async (vaultPath, payload) => {
+    const key = await readOrCreateKey(vaultPath);
+    const { nonce, authTag, ciphertext } = parseHeader(payload);
+    const decipher = createDecipheriv(algorithm, key, nonce);
+    decipher.setAuthTag(authTag);
+    const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    return brotliDecompressSync(compressed);
+};
+export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);

package/dist/infrastructure/search-packs.js

@@ -1,18 +1,22 @@
-import { gunzipSync, gzipSync } from 'node:zlib';
+import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
 const rowChunkSize = 5_000;
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
-const parseRowsFromPack = (content) => gunzipSync(content)
-    .toString('utf8')
-    .split('\n')
-    .map((line) => line.trim())
-    .filter((line) => line.length > 0)
-    .map((line) => JSON.parse(line));
+const parseRowsFromPack = async (vaultPath, content) => {
+    const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
+    return raw
+        .toString('utf8')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((line) => JSON.parse(line));
+};
 const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
     documentId: document.document.id,
     agentId: document.document.agentId,
@@ -86,7 +90,7 @@ const sortedPackFiles = async (vaultPath) => {
     try {
         const files = await readdir(toPackDirectory(vaultPath));
         return files
-            .filter((file) => file.endsWith('.jsonl.gz'))
+            .filter((file) => file.endsWith('.blpk') || file.endsWith('.jsonl.gz'))
             .sort((left, right) => left.localeCompare(right));
     }
     catch (error) {
@@ -102,20 +106,21 @@ export const buildSearchPacks = async (vaultPath, documents) => {
     await mkdir(directory, { recursive: true });
     const current = await readdir(directory);
     await Promise.all(current
-        .filter((name) => name.endsWith('.jsonl.gz') || name === manifestFileName)
+        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
         .map((name) => rm(join(directory, name), { force: true })));
     const chunks = chunkRows(rows, rowChunkSize);
     await Promise.all(chunks.map(async (chunk, index) => {
-        const fileName = `pack-${String(index + 1).padStart(4, '0')}.jsonl.gz`;
+        const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
         const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
-        const compressed = gzipSync(Buffer.from(serialized, 'utf8'), { level: 6 });
+        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'));
         await writeFile(join(directory, fileName), compressed);
     }));
     await writeManifest(vaultPath, {
-        version: 1,
+        version: 2,
         createdAt: new Date().toISOString(),
         packCount: chunks.length,
-        recordCount: rows.length
+        recordCount: rows.length,
+        format: 'private-v2'
     });
     return {
         packCount: chunks.length,
@@ -134,7 +139,7 @@ export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     }
     const scored = [];
     for (const file of files) {
-        const rows = parseRowsFromPack(await readFile(join(toPackDirectory(vaultPath), file)));
+        const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
         rows.forEach((row) => {
             if (normalizedAgent && row.agentId !== normalizedAgent) {
                 return;

package/docs/AGENT_USAGE.md

@@ -635,7 +635,8 @@ GET  /api/validate
 The HTTP API is read-only. Use the CLI for writes and indexing.
 
 Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores from snapshot automatically or recreates a clean index if no snapshot exists yet.
-Indexing also writes compressed search packs at `.brainlink/search-packs/*.jsonl.gz`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
+Indexing also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
+Pack decryption keys are resolved from `$BRAINLINK_HOME/keys` (or `BRAINLINK_SEARCH_PACK_KEY` when explicitly set).
 
 ## Agent Integration Contract
 

package/docs/ARCHITECTURE.md

@@ -301,7 +301,8 @@ Markdown keeps the system portable, inspectable, Git-friendly, and compatible wi
 SQLite gives fast local search, local vector storage and rebuildable retrieval without forcing users to run external infrastructure.
 Hybrid retrieval also uses a short-lived in-memory cache keyed by vault/query/agent and invalidated by index file mtime to reduce repeated query latency.
 Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores from snapshot automatically before reopening the index.
-Indexing additionally exports compressed pack files (`.brainlink/search-packs/*.jsonl.gz`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
+Indexing additionally exports private encrypted pack files (`.brainlink/search-packs/*.blpk`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
+Pack encryption keys are resolved from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when configured.
 
 ### CLI First
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.11",
+  "version": "0.1.0-beta.12",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",