@getlore/cli 0.2.0

package/dist/core/vector-store-lance.js

@@ -0,0 +1,384 @@
+/**
+ * Lore - Vector Store
+ *
+ * LanceDB-based vector storage for semantic search across sources and chunks.
+ * Adapted from granola-extractor with expanded schema for projects and citations.
+ */
+import * as lancedb from '@lancedb/lancedb';
+import { existsSync } from 'fs';
+let db = null;
+let currentDbPath = null;
+export async function getDatabase(dbPath) {
+    // Reconnect if path changed or no connection
+    if (!db || currentDbPath !== dbPath) {
+        db = await lancedb.connect(dbPath);
+        currentDbPath = dbPath;
+    }
+    return db;
+}
+/**
+ * Reset the database connection to force a fresh read
+ */
+export function resetDatabaseConnection() {
+    db = null;
+    currentDbPath = null;
+}
+export async function closeDatabase() {
+    if (db) {
+        db = null;
+    }
+}
+const SOURCES_TABLE = 'sources';
+const CHUNKS_TABLE = 'chunks';
+const PROJECTS_TABLE = 'projects';
+const DECISIONS_TABLE = 'decisions';
+// ============================================================================
+// Index Management
+// ============================================================================
+export async function indexExists(dbPath) {
+    if (!existsSync(dbPath))
+        return false;
+    try {
+        const database = await getDatabase(dbPath);
+        const tables = await database.tableNames();
+        // Only sources table is required - chunks may be empty
+        return tables.includes(SOURCES_TABLE);
+    }
+    catch {
+        return false;
+    }
+}
+export async function initializeTables(dbPath) {
+    const database = await getDatabase(dbPath);
+    const existingTables = await database.tableNames();
+    // Drop existing tables if they exist (for reindexing)
+    for (const table of [SOURCES_TABLE, CHUNKS_TABLE]) {
+        if (existingTables.includes(table)) {
+            await database.dropTable(table);
+        }
+    }
+}
+// ============================================================================
+// Source Storage
+// ============================================================================
+/**
+ * Add a single source to the existing index (for retain operations)
+ */
+export async function addSource(dbPath, source, vector) {
+    const database = await getDatabase(dbPath);
+    const record = {
+        id: source.id,
+        title: source.title,
+        source_type: source.source_type,
+        content_type: source.content_type,
+        projects: source.projects,
+        tags: source.tags,
+        created_at: source.created_at,
+        summary: source.summary,
+        themes_json: source.themes_json,
+        quotes_json: source.quotes_json,
+        has_full_content: source.has_full_content,
+        vector,
+    };
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        await table.add([record]);
+        console.error(`[addSource] Added to existing table: ${source.id}`);
+    }
+    catch (error) {
+        // Table doesn't exist, create it
+        console.error(`[addSource] Creating new table for: ${source.id}, error was: ${error}`);
+        await database.createTable(SOURCES_TABLE, [record]);
+    }
+}
+/**
+ * Add chunks to the existing index
+ */
+export async function addChunks(dbPath, chunks) {
+    if (chunks.length === 0)
+        return;
+    const database = await getDatabase(dbPath);
+    const records = chunks.map((chunk) => ({
+        id: chunk.id,
+        source_id: chunk.source_id,
+        content: chunk.content,
+        type: chunk.type,
+        theme_name: chunk.theme_name || '',
+        speaker: chunk.speaker || '',
+        timestamp: chunk.timestamp || '',
+        vector: chunk.vector,
+    }));
+    try {
+        const table = await database.openTable(CHUNKS_TABLE);
+        await table.add(records);
+    }
+    catch {
+        // Table doesn't exist, create it
+        await database.createTable(CHUNKS_TABLE, records);
+    }
+}
+export async function storeSources(dbPath, sources) {
+    const database = await getDatabase(dbPath);
+    const records = sources.map(({ source, vector }) => ({
+        id: source.id,
+        title: source.title,
+        source_type: source.source_type,
+        content_type: source.content_type,
+        projects: source.projects,
+        tags: source.tags,
+        created_at: source.created_at,
+        summary: source.summary,
+        themes_json: source.themes_json,
+        quotes_json: source.quotes_json,
+        has_full_content: source.has_full_content,
+        vector,
+    }));
+    await database.createTable(SOURCES_TABLE, records, { mode: 'overwrite' });
+}
+export async function storeChunks(dbPath, chunks) {
+    if (chunks.length === 0) {
+        // Nothing to store - skip creating empty table
+        return;
+    }
+    const database = await getDatabase(dbPath);
+    const records = chunks.map((chunk) => ({
+        id: chunk.id,
+        source_id: chunk.source_id,
+        content: chunk.content,
+        type: chunk.type,
+        theme_name: chunk.theme_name || '',
+        speaker: chunk.speaker || '',
+        timestamp: chunk.timestamp || '',
+        vector: chunk.vector,
+    }));
+    await database.createTable(CHUNKS_TABLE, records, { mode: 'overwrite' });
+}
+// ============================================================================
+// Search Operations
+// ============================================================================
+/**
+ * Calculate time-weighted score
+ * Recent sources get a boost, but semantic relevance is still primary
+ */
+function calculateTimeWeightedScore(semanticScore, createdAt, recencyBoost = 0.15) {
+    const now = Date.now();
+    const created = new Date(createdAt).getTime();
+    const ageInDays = (now - created) / (1000 * 60 * 60 * 24);
+    // Decay function: sources lose up to recencyBoost over 90 days
+    // After 90 days, no additional penalty
+    const ageFactor = Math.min(ageInDays / 90, 1);
+    const recencyScore = 1 - ageFactor * recencyBoost;
+    // Combine: semantic is primary (85%), recency is secondary (15%)
+    return semanticScore * (1 - recencyBoost) + semanticScore * recencyScore * recencyBoost;
+}
+export async function searchSources(dbPath, queryVector, options = {}) {
+    const { limit = 10, project, source_type, content_type, recency_boost = 0.15 } = options;
+    const database = await getDatabase(dbPath);
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        const query = table.search(queryVector).limit(limit * 3); // Over-fetch for filtering and re-ranking
+        const results = await query.toArray();
+        const filtered = results
+            .filter((row) => {
+            if (source_type && row.source_type !== source_type)
+                return false;
+            if (content_type && row.content_type !== content_type)
+                return false;
+            if (project) {
+                const projects = JSON.parse(row.projects);
+                if (!projects.some((p) => p.toLowerCase().includes(project.toLowerCase()))) {
+                    return false;
+                }
+            }
+            return true;
+        })
+            .map((row) => {
+            const semanticScore = row._distance !== undefined ? 1 / (1 + row._distance) : 0;
+            const timeWeightedScore = calculateTimeWeightedScore(semanticScore, row.created_at, recency_boost);
+            return {
+                id: row.id,
+                title: row.title,
+                source_type: row.source_type,
+                content_type: row.content_type,
+                projects: JSON.parse(row.projects),
+                tags: JSON.parse(row.tags),
+                created_at: row.created_at,
+                summary: row.summary,
+                themes: JSON.parse(row.themes_json),
+                quotes: JSON.parse(row.quotes_json),
+                score: timeWeightedScore,
+            };
+        });
+        // Re-sort by time-weighted score
+        filtered.sort((a, b) => b.score - a.score);
+        return filtered.slice(0, limit);
+    }
+    catch (error) {
+        console.error('Error searching sources:', error);
+        return [];
+    }
+}
+export async function searchChunks(dbPath, queryVector, options = {}) {
+    const { limit = 20, type, theme_name, source_id } = options;
+    const database = await getDatabase(dbPath);
+    try {
+        const table = await database.openTable(CHUNKS_TABLE);
+        const query = table.search(queryVector).limit(limit * 2);
+        const results = await query.toArray();
+        return results
+            .filter((row) => {
+            if (type && row.type !== type)
+                return false;
+            if (theme_name && row.theme_name !== theme_name)
+                return false;
+            if (source_id && row.source_id !== source_id)
+                return false;
+            return true;
+        })
+            .slice(0, limit)
+            .map((row) => ({
+            id: row.id,
+            source_id: row.source_id,
+            content: row.content,
+            type: row.type,
+            theme_name: row.theme_name,
+            speaker: row.speaker,
+            timestamp: row.timestamp,
+            score: row._distance !== undefined ? 1 / (1 + row._distance) : 0,
+        }));
+    }
+    catch {
+        // Table may not exist if no chunks have been stored
+        return [];
+    }
+}
+// ============================================================================
+// Retrieval Operations
+// ============================================================================
+export async function getAllSources(dbPath, options = {}) {
+    const { project, source_type, limit } = options;
+    const database = await getDatabase(dbPath);
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        const results = await table.query().toArray();
+        let filtered = results.filter((row) => {
+            if (source_type && row.source_type !== source_type)
+                return false;
+            if (project) {
+                const projects = JSON.parse(row.projects);
+                if (!projects.some((p) => p.toLowerCase().includes(project.toLowerCase()))) {
+                    return false;
+                }
+            }
+            return true;
+        });
+        // Sort by date descending
+        filtered.sort((a, b) => new Date(b.created_at).getTime() -
+            new Date(a.created_at).getTime());
+        if (limit) {
+            filtered = filtered.slice(0, limit);
+        }
+        return filtered.map((row) => ({
+            id: row.id,
+            title: row.title,
+            source_type: row.source_type,
+            content_type: row.content_type,
+            projects: JSON.parse(row.projects),
+            created_at: row.created_at,
+            summary: row.summary,
+        }));
+    }
+    catch (error) {
+        console.error('Error getting all sources:', error);
+        return [];
+    }
+}
+export async function getSourceById(dbPath, sourceId) {
+    const database = await getDatabase(dbPath);
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        const results = await table.query().where(`id = '${sourceId}'`).toArray();
+        if (results.length === 0)
+            return null;
+        const row = results[0];
+        return {
+            id: row.id,
+            title: row.title,
+            source_type: row.source_type,
+            content_type: row.content_type,
+            projects: JSON.parse(row.projects),
+            tags: JSON.parse(row.tags),
+            created_at: row.created_at,
+            summary: row.summary,
+            themes: JSON.parse(row.themes_json),
+            quotes: JSON.parse(row.quotes_json),
+        };
+    }
+    catch (error) {
+        console.error('Error getting source by ID:', error);
+        return null;
+    }
+}
+// ============================================================================
+// Statistics
+// ============================================================================
+export async function getThemeStats(dbPath, project) {
+    const database = await getDatabase(dbPath);
+    const stats = new Map();
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        const results = await table.query().toArray();
+        for (const row of results) {
+            // Filter by project if specified
+            if (project) {
+                const projects = JSON.parse(row.projects);
+                if (!projects.some((p) => p.toLowerCase().includes(project.toLowerCase()))) {
+                    continue;
+                }
+            }
+            const themes = JSON.parse(row.themes_json);
+            for (const theme of themes) {
+                const existing = stats.get(theme.name) || { source_count: 0, quote_count: 0 };
+                existing.source_count++;
+                existing.quote_count += theme.evidence.length;
+                stats.set(theme.name, existing);
+            }
+        }
+    }
+    catch (error) {
+        console.error('Error getting theme stats:', error);
+    }
+    return stats;
+}
+export async function getProjectStats(dbPath) {
+    const database = await getDatabase(dbPath);
+    const projectMap = new Map();
+    try {
+        const table = await database.openTable(SOURCES_TABLE);
+        const results = await table.query().toArray();
+        for (const row of results) {
+            const projects = JSON.parse(row.projects);
+            const quotes = JSON.parse(row.quotes_json);
+            const created_at = row.created_at;
+            for (const project of projects) {
+                const existing = projectMap.get(project) || {
+                    source_count: 0,
+                    quote_count: 0,
+                    latest_activity: created_at,
+                };
+                existing.source_count++;
+                existing.quote_count += quotes.length;
+                if (new Date(created_at) > new Date(existing.latest_activity)) {
+                    existing.latest_activity = created_at;
+                }
+                projectMap.set(project, existing);
+            }
+        }
+    }
+    catch (error) {
+        console.error('Error getting project stats:', error);
+    }
+    return Array.from(projectMap.entries())
+        .map(([project, stats]) => ({ project, ...stats }))
+        .sort((a, b) => new Date(b.latest_activity).getTime() - new Date(a.latest_activity).getTime());
+}

package/dist/core/vector-store-supabase.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Lore - Vector Store (Supabase + pgvector)
+ *
+ * Cloud-hosted vector storage for semantic search across sources and chunks.
+ * Replaces LanceDB for multi-machine, multi-agent support.
+ */
+import { SupabaseClient } from '@supabase/supabase-js';
+import type { SourceRecord, ChunkRecord, Quote, Theme, SourceType, ContentType } from './types.js';
+export declare function indexExists(_dbPath: string): Promise<boolean>;
+export declare function initializeTables(_dbPath: string): Promise<void>;
+export declare function resetDatabaseConnection(): void;
+export declare function closeDatabase(): Promise<void>;
+export declare function getDatabase(_dbPath: string): Promise<SupabaseClient>;
+export declare function addSource(_dbPath: string, source: SourceRecord, vector: number[]): Promise<void>;
+export declare function addChunks(_dbPath: string, chunks: ChunkRecord[]): Promise<void>;
+export declare function storeSources(_dbPath: string, sources: Array<{
+    source: SourceRecord;
+    vector: number[];
+}>): Promise<void>;
+export declare function storeChunks(_dbPath: string, chunks: ChunkRecord[]): Promise<void>;
+export declare function searchSources(_dbPath: string, queryVector: number[], options?: {
+    limit?: number;
+    project?: string;
+    source_type?: SourceType;
+    content_type?: ContentType;
+    recency_boost?: number;
+}): Promise<Array<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    tags: string[];
+    created_at: string;
+    summary: string;
+    themes: Theme[];
+    quotes: Quote[];
+    score: number;
+}>>;
+export declare function searchChunks(_dbPath: string, queryVector: number[], options?: {
+    limit?: number;
+    type?: ChunkRecord['type'];
+    theme_name?: string;
+    source_id?: string;
+}): Promise<Array<{
+    id: string;
+    source_id: string;
+    content: string;
+    type: string;
+    theme_name: string;
+    speaker: string;
+    timestamp: string;
+    score: number;
+}>>;
+export declare function getAllSources(_dbPath: string, options?: {
+    project?: string;
+    source_type?: SourceType;
+    limit?: number;
+}): Promise<Array<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    created_at: string;
+    summary: string;
+}>>;
+export declare function getSourceById(_dbPath: string, sourceId: string): Promise<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    tags: string[];
+    created_at: string;
+    summary: string;
+    themes: Theme[];
+    quotes: Quote[];
+} | null>;
+export declare function getThemeStats(_dbPath: string, project?: string): Promise<Map<string, {
+    source_count: number;
+    quote_count: number;
+}>>;
+export declare function getProjectStats(_dbPath: string): Promise<Array<{
+    project: string;
+    source_count: number;
+    quote_count: number;
+    latest_activity: string;
+}>>;