@andespindola/brainlink 0.1.0-beta.13 → 0.1.0-beta.14

package/README.md

@@ -60,6 +60,7 @@ Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable
 Brainlink now keeps an automatic rollback snapshot at `.brainlink/brainlink.db.backup` plus rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. If the main SQLite file is corrupted, Brainlink automatically restores the newest valid snapshot (or recreates a clean index when no snapshot exists).
 After each index run, Brainlink also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`. If SQLite is unavailable, search falls back to these packs automatically.
 Pack decryption uses a Brainlink key from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when explicitly configured.
+On upgrade, if a legacy SQLite index exists without private packs, Brainlink imports index rows into `.blpk` automatically on first search/context access.
 
 ## Features
 

package/dist/application/search-knowledge.js

@@ -1,7 +1,7 @@
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import { ensureVault } from '../infrastructure/file-system-vault.js';
-import { searchInPacks } from '../infrastructure/search-packs.js';
+import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
 import { openSqliteIndex } from '../infrastructure/sqlite-index.js';
 import { createEmbeddingProvider } from '../domain/embeddings.js';
 import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
@@ -47,6 +47,7 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
     const absoluteVaultPath = await ensureVault(vaultPath);
     const config = await loadBrainlinkConfig();
     const searchMode = sanitizeSearchMode(mode, config.defaultSearchMode);
+    await ensurePrivatePacksFromLegacyIndex(absoluteVaultPath);
     const cacheKey = searchMode === 'hybrid' ? toCacheKey(absoluteVaultPath, query, limit, agentId) : undefined;
     const indexMtimeMs = cacheKey ? await readIndexMtimeMs(absoluteVaultPath) : 0;
     const cached = cacheKey ? cacheGet(cacheKey, indexMtimeMs) : undefined;

package/dist/infrastructure/search-packs.js

@@ -1,6 +1,8 @@
+import Database from 'better-sqlite3';
 import { gunzipSync } from 'node:zlib';
 import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { existsSync } from 'node:fs';
 import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
 const packsDirectoryName = 'search-packs';
 const manifestFileName = 'manifest.json';
@@ -8,6 +10,7 @@ const rowChunkSize = 5_000;
 const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
 const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
 const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
+const toDatabasePath = (vaultPath) => join(vaultPath, '.brainlink', 'brainlink.db');
 const parseRowsFromPack = async (vaultPath, content) => {
     const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
     return raw
@@ -29,6 +32,15 @@ const toRows = (documents) => documents.flatMap((document) => document.chunks.ma
 const writeManifest = async (vaultPath, manifest) => {
     await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
 };
+const parseTags = (value) => {
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : [];
+    }
+    catch {
+        return [];
+    }
+};
 const chunkRows = (rows, size) => {
     const chunks = [];
     for (let index = 0; index < rows.length; index += size) {
@@ -100,14 +112,15 @@ const sortedPackFiles = async (vaultPath) => {
         throw error;
     }
 };
-export const buildSearchPacks = async (vaultPath, documents) => {
+const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting) => {
     const directory = toPackDirectory(vaultPath);
-    const rows = toRows(documents);
     await mkdir(directory, { recursive: true });
-    const current = await readdir(directory);
-    await Promise.all(current
-        .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
-        .map((name) => rm(join(directory, name), { force: true })));
+    if (clearExisting) {
+        const current = await readdir(directory);
+        await Promise.all(current
+            .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
+            .map((name) => rm(join(directory, name), { force: true })));
+    }
     const chunks = chunkRows(rows, rowChunkSize);
     await Promise.all(chunks.map(async (chunk, index) => {
         const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
@@ -127,6 +140,100 @@ export const buildSearchPacks = async (vaultPath, documents) => {
         recordCount: rows.length
     };
 };
+const tableExists = (database, table) => {
+    const row = database.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table);
+    return row?.name === table;
+};
+const tableColumns = (database, table) => {
+    const rows = database.prepare(`SELECT name FROM pragma_table_info('${table.replaceAll("'", "''")}')`).all();
+    return new Set(rows.map((row) => row.name));
+};
+const loadRowsFromLegacySqlite = (vaultPath) => {
+    const databasePath = toDatabasePath(vaultPath);
+    if (!existsSync(databasePath)) {
+        return [];
+    }
+    const database = new Database(databasePath, { readonly: true, fileMustExist: true });
+    try {
+        if (!tableExists(database, 'documents') || !tableExists(database, 'chunks')) {
+            return [];
+        }
+        const documentColumns = tableColumns(database, 'documents');
+        const chunkColumns = tableColumns(database, 'chunks');
+        if (!documentColumns.has('id') || !documentColumns.has('title') || !chunkColumns.has('document_id')) {
+            return [];
+        }
+        const agentExpr = documentColumns.has('agent_id') ? 'documents.agent_id' : "'shared'";
+        const pathExpr = documentColumns.has('path') ? 'documents.path' : "documents.title";
+        const tagsExpr = documentColumns.has('tags_json') ? 'documents.tags_json' : "'[]'";
+        const chunkIdExpr = chunkColumns.has('id') ? 'chunks.id' : "documents.id || ':' || chunks.rowid";
+        const chunkContentExpr = chunkColumns.has('content')
+            ? 'chunks.content'
+            : documentColumns.has('content')
+                ? 'documents.content'
+                : "''";
+        const chunkOrderExpr = chunkColumns.has('ordinal') ? 'chunks.ordinal' : 'chunks.rowid';
+        const statement = database.prepare(`
+      SELECT
+        documents.id AS document_id,
+        ${agentExpr} AS agent_id,
+        documents.title AS title,
+        ${pathExpr} AS path,
+        ${chunkIdExpr} AS chunk_id,
+        ${chunkContentExpr} AS content,
+        ${tagsExpr} AS tags_json
+      FROM chunks
+      JOIN documents ON documents.id = chunks.document_id
+      ORDER BY documents.title, ${chunkOrderExpr}
+    `);
+        const rows = statement.all();
+        return rows.map((row) => ({
+            documentId: row.document_id,
+            agentId: typeof row.agent_id === 'string' && row.agent_id.length > 0 ? row.agent_id : 'shared',
+            title: row.title,
+            path: row.path,
+            chunkId: row.chunk_id,
+            content: row.content ?? '',
+            tags: parseTags(row.tags_json)
+        }));
+    }
+    finally {
+        database.close();
+    }
+};
+export const buildSearchPacks = async (vaultPath, documents) => {
+    return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true);
+};
+export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
+    const files = await sortedPackFiles(vaultPath);
+    if (files.some((file) => file.endsWith('.blpk'))) {
+        return { imported: false };
+    }
+    const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
+    if (legacyPackFiles.length > 0) {
+        const rows = [];
+        for (const file of legacyPackFiles) {
+            const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+            rows.push(...parsed);
+        }
+        const report = await writeRowsAsPrivatePacks(vaultPath, rows, true);
+        return {
+            imported: true,
+            source: 'legacy-packs',
+            ...report
+        };
+    }
+    const legacyRows = loadRowsFromLegacySqlite(vaultPath);
+    if (legacyRows.length === 0) {
+        return { imported: false };
+    }
+    const report = await writeRowsAsPrivatePacks(vaultPath, legacyRows, true);
+    return {
+        imported: true,
+        source: 'legacy-sqlite',
+        ...report
+    };
+};
 export const searchInPacks = async (vaultPath, query, limit, agentId) => {
     const normalizedAgent = agentId?.trim();
     const tokens = tokenize(query);

package/docs/AGENT_USAGE.md

@@ -637,6 +637,7 @@ The HTTP API is read-only. Use the CLI for writes and indexing.
 Brainlink maintains an automatic SQLite rollback snapshot at `.brainlink/brainlink.db.backup` and rotating snapshots in `.brainlink/brainlink.db.backup.snapshots/`. When `.brainlink/brainlink.db` is corrupted, Brainlink restores the newest valid snapshot automatically or recreates a clean index if no snapshot exists yet.
 Indexing also writes private encrypted search packs at `.brainlink/search-packs/*.blpk`; when SQLite cannot be opened, Brainlink falls back to pack-based search automatically.
 Pack decryption keys are resolved from `$BRAINLINK_HOME/keys` (or `BRAINLINK_SEARCH_PACK_KEY` when explicitly set).
+For legacy installations, when SQLite already exists but private packs are missing, Brainlink auto-imports index context rows from `brainlink.db` into `.blpk` on first retrieval.
 
 ## Agent Integration Contract
 

package/docs/ARCHITECTURE.md

@@ -303,6 +303,7 @@ Hybrid retrieval also uses a short-lived in-memory cache keyed by vault/query/ag
 Brainlink also writes a local rollback snapshot (`.brainlink/brainlink.db.backup`) plus rotating point-in-time snapshots (`.brainlink/brainlink.db.backup.snapshots/`) after successful indexing. On corruption detection (`quick_check`/SQLite malformed errors), Brainlink restores the newest valid snapshot automatically before reopening the index.
 Indexing additionally exports private encrypted pack files (`.brainlink/search-packs/*.blpk`) from indexed chunks. Search falls back to these packs when SQLite is unavailable, preserving retrieval continuity in degraded mode.
 Pack encryption keys are resolved from `$BRAINLINK_HOME/keys` or from `BRAINLINK_SEARCH_PACK_KEY` when configured.
+Legacy upgrades are automatic: when a vault has `brainlink.db` but no `.blpk` packs yet, Brainlink extracts indexed context rows from SQLite and writes private packs on first retrieval flow.
 
 ### CLI First
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.13",
+  "version": "0.1.0-beta.14",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",