@andespindola/brainlink 0.1.0-alpha.0

package/dist/infrastructure/sqlite/graph-reader.js

@@ -0,0 +1,108 @@
+import { sanitizeAgentId } from '../../domain/agents.js';
+const toGraphLink = (row) => ({
+    agentId: row.agent_id,
+    fromTitle: row.from_title,
+    fromPath: row.from_path,
+    toTitle: row.to_title,
+    toPath: row.to_path
+});
+const normalizeAgentFilter = (agentId) => agentId ? sanitizeAgentId(agentId) : undefined;
+export const createGraphReader = (database) => ({
+    listLinks: (agentId) => {
+        const normalizedAgentId = normalizeAgentFilter(agentId);
+        const agentFilter = normalizedAgentId ? 'WHERE source.agent_id = ?' : '';
+        const rows = database
+            .prepare(`
+          SELECT
+            source.agent_id AS agent_id,
+            source.title AS from_title,
+            source.path AS from_path,
+            COALESCE(target.title, links.to_title) AS to_title,
+            target.path AS to_path
+          FROM links
+          JOIN documents source ON source.id = links.from_document_id
+          LEFT JOIN documents target ON target.id = links.to_document_id
+          ${agentFilter}
+          ORDER BY source.title, to_title
+        `)
+            .all(...(normalizedAgentId ? [normalizedAgentId] : []));
+        return rows.map(toGraphLink);
+    },
+    listBacklinks: (title, agentId) => {
+        const normalizedAgentId = normalizeAgentFilter(agentId);
+        const agentFilter = normalizedAgentId ? 'AND source.agent_id = ?' : '';
+        const rows = database
+            .prepare(`
+          SELECT
+            source.agent_id AS agent_id,
+            source.title AS from_title,
+            source.path AS from_path,
+            COALESCE(target.title, links.to_title) AS to_title,
+            target.path AS to_path
+          FROM links
+          JOIN documents source ON source.id = links.from_document_id
+          LEFT JOIN documents target ON target.id = links.to_document_id
+          WHERE (lower(links.to_title) = lower(?) OR lower(target.title) = lower(?))
+          ${agentFilter}
+          ORDER BY source.title
+        `)
+            .all(...(normalizedAgentId ? [title, title, normalizedAgentId] : [title, title]));
+        return rows.map(toGraphLink);
+    },
+    getGraph: (agentId) => {
+        const normalizedAgentId = normalizeAgentFilter(agentId);
+        const documentAgentFilter = normalizedAgentId ? 'WHERE agent_id = ?' : '';
+        const edgeAgentFilter = normalizedAgentId ? 'WHERE source.agent_id = ?' : '';
+        const nodeRows = database
+            .prepare(`
+          SELECT id, agent_id, title, path, content, tags_json
+          FROM documents
+          ${documentAgentFilter}
+          ORDER BY title
+        `)
+            .all(...(normalizedAgentId ? [normalizedAgentId] : []));
+        const edgeRows = database
+            .prepare(`
+          SELECT
+            links.from_document_id AS source,
+            links.to_document_id AS target,
+            links.to_title AS target_title
+          FROM links
+          JOIN documents source ON source.id = links.from_document_id
+          ${edgeAgentFilter}
+          ORDER BY links.from_document_id, links.to_title
+        `)
+            .all(...(normalizedAgentId ? [normalizedAgentId] : []));
+        const nodes = nodeRows.map((row) => ({
+            id: row.id,
+            agentId: row.agent_id,
+            title: row.title,
+            path: row.path,
+            content: row.content,
+            tags: JSON.parse(row.tags_json)
+        }));
+        const edges = edgeRows.map((row) => ({
+            source: row.source,
+            target: row.target,
+            targetTitle: row.target_title
+        }));
+        return {
+            nodes,
+            edges
+        };
+    },
+    listAgents: () => {
+        const rows = database
+            .prepare(`
+          SELECT agent_id AS id, count(*) AS document_count
+          FROM documents
+          GROUP BY agent_id
+          ORDER BY agent_id
+        `)
+            .all();
+        return rows.map((row) => ({
+            id: row.id,
+            documentCount: row.document_count
+        }));
+    }
+});

package/dist/infrastructure/sqlite/schema.js

@@ -0,0 +1,87 @@
+const schemaVersion = 4;
+const getStoredSchemaVersion = (database) => {
+    const hasMetadata = database
+        .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'metadata'")
+        .get();
+    if (!hasMetadata) {
+        return 0;
+    }
+    const row = database.prepare("SELECT value FROM metadata WHERE key = 'schema_version'").get();
+    return Number.parseInt(row?.value ?? '0', 10);
+};
+const dropDerivedSchema = (database) => {
+    database.exec(`
+    DROP TABLE IF EXISTS embedding_buckets;
+    DROP TABLE IF EXISTS chunks_fts;
+    DROP TABLE IF EXISTS links;
+    DROP TABLE IF EXISTS chunks;
+    DROP TABLE IF EXISTS documents;
+  `);
+};
+export const createSchema = (database) => {
+    const storedSchemaVersion = getStoredSchemaVersion(database);
+    if (storedSchemaVersion > 0 && storedSchemaVersion < schemaVersion) {
+        dropDerivedSchema(database);
+    }
+    database.exec(`
+    CREATE TABLE IF NOT EXISTS metadata (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS documents (
+      id TEXT PRIMARY KEY,
+      agent_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      path TEXT NOT NULL UNIQUE,
+      content TEXT NOT NULL,
+      tags_json TEXT NOT NULL,
+      frontmatter_json TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS chunks (
+      id TEXT PRIMARY KEY,
+      document_id TEXT NOT NULL,
+      ordinal INTEGER NOT NULL,
+      content TEXT NOT NULL,
+      token_count INTEGER NOT NULL,
+      embedding_provider TEXT NOT NULL,
+      embedding_json TEXT NOT NULL,
+      FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
+    );
+
+    CREATE TABLE IF NOT EXISTS embedding_buckets (
+      bucket TEXT NOT NULL,
+      chunk_id TEXT NOT NULL,
+      PRIMARY KEY (bucket, chunk_id),
+      FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_embedding_buckets_bucket ON embedding_buckets(bucket);
+
+    CREATE TABLE IF NOT EXISTS links (
+      from_document_id TEXT NOT NULL,
+      to_title TEXT NOT NULL,
+      to_document_id TEXT,
+      FOREIGN KEY (from_document_id) REFERENCES documents(id) ON DELETE CASCADE,
+      FOREIGN KEY (to_document_id) REFERENCES documents(id) ON DELETE SET NULL
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+      chunk_id UNINDEXED,
+      document_id UNINDEXED,
+      agent_id UNINDEXED,
+      title,
+      content
+    );
+  `);
+    database
+        .prepare(`
+      INSERT INTO metadata (key, value)
+      VALUES ('schema_version', ?)
+      ON CONFLICT(key) DO UPDATE SET value = excluded.value
+    `)
+        .run(String(schemaVersion));
+};

package/dist/infrastructure/sqlite/search-reader.js

@@ -0,0 +1,156 @@
+import { sanitizeAgentId } from '../../domain/agents.js';
+import { cosineSimilarity, createEmbeddingBuckets } from '../../domain/embeddings.js';
+const toFtsQuery = (query) => query
+    .toLowerCase()
+    .match(/[\p{L}\p{N}_-]+/gu)
+    ?.map((term) => `"${term.replaceAll('"', '""')}"*`)
+    .join(' OR ') ?? '';
+const normalizeAgentFilter = (agentId) => agentId ? sanitizeAgentId(agentId) : undefined;
+const parseJsonArray = (value) => {
+    if (!value) {
+        return [];
+    }
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+};
+const toTextScore = (index, total) => total === 0 ? 0 : 1 - index / (total + 1);
+const toSearchResult = (row, score, textScore, semanticScore, searchMode) => ({
+    documentId: row.document_id,
+    agentId: row.agent_id,
+    title: row.title,
+    path: row.path,
+    chunkId: row.chunk_id,
+    content: row.content,
+    score,
+    textScore,
+    semanticScore,
+    searchMode,
+    tags: parseJsonArray(row.tags_json).filter((value) => typeof value === 'string')
+});
+const sortByScore = (results) => [...results].sort((left, right) => right.score - left.score || left.title.localeCompare(right.title));
+const mergeHybridResults = (ftsResults, semanticResults, limit) => {
+    const rows = new Map();
+    [...semanticResults, ...ftsResults].forEach((result) => {
+        const current = rows.get(result.chunkId);
+        const textScore = Math.max(current?.textScore ?? 0, result.textScore);
+        const semanticScore = Math.max(current?.semanticScore ?? 0, result.semanticScore);
+        const score = textScore * 0.62 + semanticScore * 0.38;
+        rows.set(result.chunkId, {
+            ...result,
+            score,
+            textScore,
+            semanticScore,
+            searchMode: 'hybrid'
+        });
+    });
+    return sortByScore(Array.from(rows.values())).slice(0, limit);
+};
+const placeholders = (count) => Array.from({ length: count }, () => '?').join(', ');
+const readAllSemanticRows = (database, normalizedAgentId) => {
+    const semanticAgentFilter = normalizedAgentId ? 'WHERE documents.agent_id = ?' : '';
+    return database
+        .prepare(`
+      SELECT
+        documents.id AS document_id,
+        documents.agent_id AS agent_id,
+        documents.title AS title,
+        documents.path AS path,
+        chunks.id AS chunk_id,
+        chunks.content AS content,
+        documents.tags_json AS tags_json,
+        chunks.embedding_json AS embedding_json
+      FROM chunks
+      JOIN documents ON documents.id = chunks.document_id
+      ${semanticAgentFilter}
+    `)
+        .all(...(normalizedAgentId ? [normalizedAgentId] : []));
+};
+const readBucketedSemanticRows = (database, normalizedAgentId, queryEmbedding, limit) => {
+    const buckets = createEmbeddingBuckets(queryEmbedding);
+    if (buckets.length === 0) {
+        return [];
+    }
+    const agentFilter = normalizedAgentId ? 'AND documents.agent_id = ?' : '';
+    const params = normalizedAgentId ? [...buckets, normalizedAgentId, limit] : [...buckets, limit];
+    return database
+        .prepare(`
+      SELECT
+        documents.id AS document_id,
+        documents.agent_id AS agent_id,
+        documents.title AS title,
+        documents.path AS path,
+        chunks.id AS chunk_id,
+        chunks.content AS content,
+        documents.tags_json AS tags_json,
+        chunks.embedding_json AS embedding_json,
+        count(*) AS score
+      FROM embedding_buckets
+      JOIN chunks ON chunks.id = embedding_buckets.chunk_id
+      JOIN documents ON documents.id = chunks.document_id
+      WHERE embedding_buckets.bucket IN (${placeholders(buckets.length)})
+      ${agentFilter}
+      GROUP BY chunks.id
+      ORDER BY score DESC, chunks.token_count ASC, documents.title ASC
+      LIMIT ?
+    `)
+        .all(...params);
+};
+const readSemanticRows = (database, normalizedAgentId, queryEmbedding, limit) => {
+    const candidateLimit = Math.max(limit * 96, 768);
+    const bucketedRows = readBucketedSemanticRows(database, normalizedAgentId, queryEmbedding, candidateLimit);
+    return bucketedRows.length > 0 ? bucketedRows : readAllSemanticRows(database, normalizedAgentId);
+};
+export const createSearchReader = (database) => ({
+    search: (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
+        const normalizedQuery = query.trim();
+        if (!normalizedQuery || limit <= 0) {
+            return [];
+        }
+        const normalizedAgentId = normalizeAgentFilter(agentId);
+        const ftsQuery = toFtsQuery(query);
+        const expandedLimit = Math.max(limit * 4, 24);
+        const ftsAgentFilter = normalizedAgentId ? 'AND documents.agent_id = ?' : '';
+        const ftsParams = normalizedAgentId ? [ftsQuery, normalizedAgentId, expandedLimit] : [ftsQuery, expandedLimit];
+        const ftsRows = mode === 'semantic' || !ftsQuery
+            ? []
+            : database
+                .prepare(`
+                SELECT
+                  documents.id AS document_id,
+                  documents.agent_id AS agent_id,
+                  documents.title AS title,
+                  documents.path AS path,
+                  chunks_fts.chunk_id AS chunk_id,
+                  chunks_fts.content AS content,
+                  bm25(chunks_fts) * -1 AS score,
+                  documents.tags_json AS tags_json
+                FROM chunks_fts
+                JOIN documents ON documents.id = chunks_fts.document_id
+                WHERE chunks_fts MATCH ?
+                ${ftsAgentFilter}
+                ORDER BY bm25(chunks_fts)
+                LIMIT ?
+              `)
+                .all(...ftsParams);
+        const ftsResults = ftsRows.map((row, index) => toSearchResult(row, toTextScore(index, ftsRows.length), toTextScore(index, ftsRows.length), 0, 'fts'));
+        const semanticRows = mode === 'fts' || queryEmbedding.length === 0 ? [] : readSemanticRows(database, normalizedAgentId, queryEmbedding, expandedLimit);
+        const semanticResults = sortByScore(semanticRows
+            .map((row) => {
+            const semanticScore = Math.max(0, cosineSimilarity(queryEmbedding, parseJsonArray(row.embedding_json).filter((value) => typeof value === 'number')));
+            return toSearchResult(row, semanticScore, 0, semanticScore, 'semantic');
+        })
+            .filter((result) => result.semanticScore > 0)).slice(0, expandedLimit);
+        if (mode === 'fts') {
+            return ftsResults.slice(0, limit);
+        }
+        if (mode === 'semantic') {
+            return semanticResults.slice(0, limit);
+        }
+        return mergeHybridResults(ftsResults, semanticResults, limit);
+    }
+});

package/dist/infrastructure/sqlite/types.js

@@ -0,0 +1 @@
+export {};

package/dist/infrastructure/sqlite-index.js

@@ -0,0 +1,20 @@
+import Database from 'better-sqlite3';
+import { chmodSync } from 'node:fs';
+import { join } from 'node:path';
+import { createIndexWriter } from './sqlite/document-writer.js';
+import { createGraphReader } from './sqlite/graph-reader.js';
+import { createSchema } from './sqlite/schema.js';
+import { createSearchReader } from './sqlite/search-reader.js';
+export const openSqliteIndex = (vaultPath) => {
+    const databasePath = join(vaultPath, '.brainlink', 'brainlink.db');
+    const database = new Database(databasePath);
+    chmodSync(databasePath, 0o600);
+    database.exec('PRAGMA foreign_keys = ON;');
+    createSchema(database);
+    return {
+        ...createIndexWriter(database),
+        ...createSearchReader(database),
+        ...createGraphReader(database),
+        close: () => database.close()
+    };
+};