@getlore/cli 0.2.0

package/dist/mcp/handlers/retain.js

@@ -0,0 +1,92 @@
+/**
+ * Retain Handler - Save insights, decisions, and notes
+ *
+ * This is the "push" mechanism for adding knowledge explicitly.
+ * Retained items are immediately added to the vector store for instant searchability.
+ * Auto-pushes to git remote if configured.
+ */
+import { writeFile, mkdir } from 'fs/promises';
+import path from 'path';
+import { randomUUID } from 'crypto';
+import { addSource } from '../../core/vector-store.js';
+import { generateEmbedding, createSearchableText } from '../../core/embedder.js';
+import { gitCommitAndPush } from '../../core/git.js';
+export async function handleRetain(dbPath, dataDir, args, options = {}) {
+    const { content, project, type, source_context, tags = [] } = args;
+    const { autoPush = true } = options;
+    const id = randomUUID();
+    const timestamp = new Date().toISOString();
+    // Create the retained knowledge entry
+    const entry = {
+        id,
+        content,
+        project,
+        type,
+        source_context: source_context || 'Explicitly retained via MCP',
+        tags,
+        created_at: timestamp,
+    };
+    // Save to disk
+    const retainedDir = path.join(dataDir, 'retained', project);
+    await mkdir(retainedDir, { recursive: true });
+    const filename = `${type}-${id.slice(0, 8)}.json`;
+    const filepath = path.join(retainedDir, filename);
+    await writeFile(filepath, JSON.stringify(entry, null, 2));
+    // Add to vector store immediately for instant searchability
+    try {
+        // Generate embedding for the content
+        const searchableText = createSearchableText({
+            type: type === 'decision' ? 'theme' : 'summary',
+            text: content,
+            project,
+        });
+        const vector = await generateEmbedding(searchableText);
+        // Create source record
+        const sourceRecord = {
+            id,
+            title: `${type.charAt(0).toUpperCase() + type.slice(1)}: ${content.substring(0, 50)}...`,
+            source_type: 'retained',
+            content_type: type === 'decision' ? 'decision' : 'note',
+            projects: JSON.stringify([project]),
+            tags: JSON.stringify(tags),
+            created_at: timestamp,
+            summary: content,
+            themes_json: JSON.stringify([]),
+            quotes_json: JSON.stringify([]),
+            has_full_content: true,
+            vector: [],
+        };
+        await addSource(dbPath, sourceRecord, vector);
+        // Auto-push to git if enabled
+        let pushed = false;
+        if (autoPush) {
+            const pushResult = await gitCommitAndPush(dataDir, `Retain ${type}: ${content.substring(0, 50)}...`);
+            pushed = pushResult.success && (pushResult.message?.includes('pushed') || false);
+        }
+        return {
+            success: true,
+            id,
+            message: `Retained ${type} for project "${project}"`,
+            indexed: true,
+            synced: pushed,
+        };
+    }
+    catch (error) {
+        // Still saved to disk, just not indexed yet
+        console.error('Failed to index retained item:', error);
+        // Still try to push even if indexing failed
+        let pushed = false;
+        if (autoPush) {
+            const pushResult = await gitCommitAndPush(dataDir, `Retain ${type}: ${content.substring(0, 50)}...`);
+            pushed = pushResult.success && (pushResult.message?.includes('pushed') || false);
+        }
+        return {
+            success: true,
+            id,
+            message: `Retained ${type} for project "${project}"`,
+            indexed: false,
+            synced: pushed,
+            note: 'Saved to disk but indexing failed. Run "lore sync" to index.',
+        };
+    }
+}

package/dist/mcp/handlers/search.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Search Handler - Semantic and hybrid search across sources
+ *
+ * Supports multiple search modes:
+ * - semantic: Vector similarity only (conceptual queries)
+ * - keyword: Full-text search only (exact terms)
+ * - hybrid: RRF fusion of semantic + keyword (default)
+ * - regex: Local file grep (pattern matching)
+ *
+ * By default, excludes sources from archived projects.
+ * Use include_archived: true to search everything.
+ */
+import type { SourceType, ContentType, Quote, SearchMode } from '../../core/types.js';
+interface SearchArgs {
+    query: string;
+    project?: string;
+    source_type?: SourceType;
+    content_type?: ContentType;
+    limit?: number;
+    include_archived?: boolean;
+    mode?: SearchMode;
+}
+interface SearchResultSource {
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    created_at: string;
+    summary: string;
+    relevance_score: number;
+    matching_quotes: Quote[];
+    themes: string[];
+    /** Rank in semantic search (only for hybrid/semantic modes) */
+    semantic_rank?: number;
+    /** Rank in keyword search (only for hybrid/keyword modes) */
+    keyword_rank?: number;
+    /** Matching lines (only for regex mode) */
+    matching_lines?: Array<{
+        line_number: number;
+        snippet: string;
+    }>;
+}
+interface SearchResult {
+    sources: SearchResultSource[];
+    total_found: number;
+    query: string;
+    mode: SearchMode;
+    archived_excluded?: number;
+}
+export declare function handleSearch(dbPath: string, dataDir: string, args: SearchArgs): Promise<SearchResult>;
+export {};

package/dist/mcp/handlers/search.js

@@ -0,0 +1,145 @@
+/**
+ * Search Handler - Semantic and hybrid search across sources
+ *
+ * Supports multiple search modes:
+ * - semantic: Vector similarity only (conceptual queries)
+ * - keyword: Full-text search only (exact terms)
+ * - hybrid: RRF fusion of semantic + keyword (default)
+ * - regex: Local file grep (pattern matching)
+ *
+ * By default, excludes sources from archived projects.
+ * Use include_archived: true to search everything.
+ */
+import { searchSources, getSourceById } from '../../core/vector-store.js';
+import { generateEmbedding } from '../../core/embedder.js';
+import { searchLocalFiles, getMatchSnippet } from '../../core/local-search.js';
+import { loadArchivedProjects } from './archive-project.js';
+export async function handleSearch(dbPath, dataDir, args) {
+    const { query, project, source_type, content_type, limit = 10, include_archived = false, mode = 'hybrid', } = args;
+    // Handle regex mode separately - uses local file search
+    if (mode === 'regex') {
+        return handleRegexSearch(dbPath, dataDir, {
+            query,
+            project,
+            limit,
+            include_archived,
+        });
+    }
+    // Generate embedding for query (needed for semantic/hybrid modes)
+    const queryVector = await generateEmbedding(query);
+    // Search sources (fetch more to account for archived filtering)
+    const fetchLimit = include_archived ? limit : limit * 2;
+    const results = await searchSources(dbPath, queryVector, {
+        limit: fetchLimit,
+        project,
+        source_type,
+        content_type,
+        mode,
+        queryText: query,
+    });
+    // Filter out archived projects unless explicitly requested
+    let filteredResults = results;
+    let archivedExcluded = 0;
+    if (!include_archived) {
+        const archivedProjects = await loadArchivedProjects(dataDir);
+        const archivedNames = new Set(archivedProjects.map((p) => p.project.toLowerCase()));
+        filteredResults = results.filter((result) => {
+            const isArchived = result.projects.some((p) => archivedNames.has(p.toLowerCase()));
+            if (isArchived)
+                archivedExcluded++;
+            return !isArchived;
+        });
+    }
+    // Format results with relevant quotes highlighted
+    const sources = filteredResults.slice(0, limit).map((result) => {
+        // Find quotes most relevant to the query (simple keyword match for now)
+        const queryWords = query.toLowerCase().split(/\s+/);
+        const matchingQuotes = result.quotes
+            .filter((q) => queryWords.some((word) => q.text.toLowerCase().includes(word)))
+            .slice(0, 3);
+        return {
+            id: result.id,
+            title: result.title,
+            source_type: result.source_type,
+            content_type: result.content_type,
+            projects: result.projects,
+            created_at: result.created_at,
+            summary: result.summary,
+            relevance_score: result.score,
+            matching_quotes: matchingQuotes,
+            themes: result.themes.map((t) => t.name),
+            semantic_rank: result.semantic_rank,
+            keyword_rank: result.keyword_rank,
+        };
+    });
+    return {
+        sources,
+        total_found: sources.length,
+        query,
+        mode,
+        archived_excluded: archivedExcluded > 0 ? archivedExcluded : undefined,
+    };
+}
+/**
+ * Handle regex search using local file grep
+ */
+async function handleRegexSearch(dbPath, dataDir, args) {
+    const { query, project, limit = 10, include_archived = false } = args;
+    // Search local files
+    const localResults = await searchLocalFiles(dataDir, query, {
+        maxTotalResults: limit * 2, // Fetch extra for filtering
+        maxMatchesPerFile: 5,
+        ignoreCase: false,
+    });
+    // Get source details from database to enrich results
+    const sources = [];
+    let archivedExcluded = 0;
+    const archivedProjects = include_archived
+        ? []
+        : await loadArchivedProjects(dataDir);
+    const archivedNames = new Set(archivedProjects.map((p) => p.project.toLowerCase()));
+    for (const localResult of localResults) {
+        if (sources.length >= limit)
+            break;
+        // Get source metadata from database
+        const sourceData = await getSourceById(dbPath, localResult.source_id);
+        if (!sourceData)
+            continue;
+        // Filter by project if specified
+        if (project && !sourceData.projects.includes(project))
+            continue;
+        // Filter out archived projects
+        if (!include_archived) {
+            const isArchived = sourceData.projects.some((p) => archivedNames.has(p.toLowerCase()));
+            if (isArchived) {
+                archivedExcluded++;
+                continue;
+            }
+        }
+        // Format matching lines
+        const matchingLines = localResult.matches.slice(0, 3).map((m) => ({
+            line_number: m.line_number,
+            snippet: getMatchSnippet(m.line_content, m.match_start, m.match_end, 80),
+        }));
+        sources.push({
+            id: sourceData.id,
+            title: sourceData.title,
+            source_type: sourceData.source_type,
+            content_type: sourceData.content_type,
+            projects: sourceData.projects,
+            created_at: sourceData.created_at,
+            summary: sourceData.summary,
+            relevance_score: localResult.matches.length / 10, // Simple score based on match count
+            matching_quotes: [],
+            themes: sourceData.themes.map((t) => t.name),
+            matching_lines: matchingLines,
+        });
+    }
+    return {
+        sources,
+        total_found: sources.length,
+        query,
+        mode: 'regex',
+        archived_excluded: archivedExcluded > 0 ? archivedExcluded : undefined,
+    };
+}

package/dist/mcp/handlers/sync.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Sync Handler - Universal file sync with two-phase processing
+ *
+ * Phase 1: Discovery (NO LLM calls - essentially free)
+ *   - Scan configured directories
+ *   - Compute content hashes
+ *   - Check Supabase for existing hashes
+ *
+ * Phase 2: Processing (only for NEW files)
+ *   - Claude extracts metadata
+ *   - Generate embeddings
+ *   - Store in Supabase + local data dir
+ */
+interface SyncArgs {
+    git_pull?: boolean;
+    git_push?: boolean;
+    index_new?: boolean;
+    dry_run?: boolean;
+    use_legacy?: boolean;
+}
+interface SyncResult {
+    git_pulled: boolean;
+    git_pushed: boolean;
+    git_error?: string;
+    sources_found: number;
+    sources_indexed: number;
+    already_indexed: number;
+    discovery?: {
+        sources_scanned: number;
+        total_files: number;
+        new_files: number;
+        edited_files: number;
+        existing_files: number;
+        errors: number;
+    };
+    processing?: {
+        processed: number;
+        errors: number;
+        titles: string[];
+    };
+}
+export declare function handleSync(dbPath: string, dataDir: string, args: SyncArgs, options?: {
+    hookContext?: {
+        mode: 'mcp' | 'cli';
+    };
+}): Promise<SyncResult>;
+export {};

package/dist/mcp/handlers/sync.js

@@ -0,0 +1,211 @@
+/**
+ * Sync Handler - Universal file sync with two-phase processing
+ *
+ * Phase 1: Discovery (NO LLM calls - essentially free)
+ *   - Scan configured directories
+ *   - Compute content hashes
+ *   - Check Supabase for existing hashes
+ *
+ * Phase 2: Processing (only for NEW files)
+ *   - Claude extracts metadata
+ *   - Generate embeddings
+ *   - Store in Supabase + local data dir
+ */
+import { readdir, readFile } from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+import { getAllSources, addSource, resetDatabaseConnection, } from '../../core/vector-store.js';
+import { generateEmbedding, createSearchableText } from '../../core/embedder.js';
+import { gitPull, gitCommitAndPush } from '../../core/git.js';
+import { loadSyncConfig, getEnabledSources } from '../../sync/config.js';
+import { discoverAllSources, summarizeDiscovery } from '../../sync/discover.js';
+import { processFiles } from '../../sync/process.js';
+// ============================================================================
+// Legacy Disk-Based Sync (for backward compatibility)
+// ============================================================================
+async function loadSourceFromDisk(sourcesDir, sourceId) {
+    const sourceDir = path.join(sourcesDir, sourceId);
+    try {
+        const metadata = JSON.parse(await readFile(path.join(sourceDir, 'metadata.json'), 'utf-8'));
+        const content = await readFile(path.join(sourceDir, 'content.md'), 'utf-8');
+        let insights = { summary: '', themes: [] };
+        try {
+            const insightsFile = JSON.parse(await readFile(path.join(sourceDir, 'insights.json'), 'utf-8'));
+            insights.summary = insightsFile.summary || '';
+            insights.themes = insightsFile.themes || [];
+        }
+        catch {
+            insights.summary = content.substring(0, 500) + (content.length > 500 ? '...' : '');
+        }
+        return {
+            source: {
+                ...metadata,
+                content,
+            },
+            insights,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+async function indexSource(dbPath, source, insights) {
+    const summary = insights.summary || source.content.substring(0, 500);
+    const searchableText = createSearchableText({
+        type: 'summary',
+        text: summary,
+        project: source.projects[0],
+    });
+    const vector = await generateEmbedding(searchableText);
+    const sourceRecord = {
+        id: source.id,
+        title: source.title,
+        source_type: source.source_type,
+        content_type: source.content_type,
+        projects: JSON.stringify(source.projects),
+        tags: JSON.stringify(source.tags),
+        created_at: source.created_at,
+        summary,
+        themes_json: JSON.stringify(insights.themes || []),
+        quotes_json: JSON.stringify([]),
+        has_full_content: true,
+        vector: [],
+    };
+    await addSource(dbPath, sourceRecord, vector, {
+        content_hash: source.content_hash,
+        source_path: source.source_path,
+    });
+}
+async function legacyDiskSync(dbPath, dataDir) {
+    const sourcesDir = path.join(dataDir, 'sources');
+    const result = { sources_found: 0, sources_indexed: 0, already_indexed: 0 };
+    if (!existsSync(sourcesDir)) {
+        return result;
+    }
+    try {
+        const diskSources = await readdir(sourcesDir, { withFileTypes: true });
+        const diskIds = diskSources
+            .filter((d) => d.isDirectory() && !d.name.startsWith('.'))
+            .map((d) => d.name);
+        result.sources_found = diskIds.length;
+        const indexedSources = await getAllSources(dbPath, {});
+        const indexedIds = new Set(indexedSources.map((s) => s.id));
+        result.already_indexed = indexedIds.size;
+        const unsyncedIds = diskIds.filter((id) => !indexedIds.has(id));
+        for (const sourceId of unsyncedIds) {
+            const data = await loadSourceFromDisk(sourcesDir, sourceId);
+            if (data) {
+                await indexSource(dbPath, data.source, data.insights);
+                result.sources_indexed++;
+            }
+        }
+    }
+    catch {
+        // Sources directory doesn't exist or other error
+    }
+    return result;
+}
+// ============================================================================
+// Universal Sync (new system)
+// ============================================================================
+async function universalSync(dataDir, dryRun, hookContext) {
+    // Load sync configuration
+    const config = await loadSyncConfig();
+    const enabledSources = getEnabledSources(config);
+    if (enabledSources.length === 0) {
+        return {
+            discovery: {
+                sources_scanned: 0,
+                total_files: 0,
+                new_files: 0,
+                edited_files: 0,
+                existing_files: 0,
+                errors: 0,
+            },
+            processing: undefined,
+        };
+    }
+    // Phase 1: Discovery
+    const discoveryResults = await discoverAllSources(enabledSources);
+    const summary = summarizeDiscovery(discoveryResults);
+    const discovery = {
+        sources_scanned: summary.totalSources,
+        total_files: summary.totalFiles,
+        new_files: summary.newFiles,
+        edited_files: summary.editedFiles,
+        existing_files: summary.existingFiles,
+        errors: summary.errors,
+    };
+    // If dry run or no new/edited files, stop here
+    const totalToProcess = summary.newFiles + summary.editedFiles;
+    if (dryRun || totalToProcess === 0) {
+        return { discovery, processing: undefined };
+    }
+    // Phase 2: Process new and edited files
+    const allNewFiles = discoveryResults.flatMap(r => r.newFiles);
+    const allEditedFiles = discoveryResults.flatMap(r => r.editedFiles);
+    const allFilesToProcess = [...allNewFiles, ...allEditedFiles];
+    const processResult = await processFiles(allFilesToProcess, dataDir, {
+        gitPush: false, // We'll handle git at the end
+        hookContext,
+    });
+    const processing = {
+        processed: processResult.processed.length,
+        errors: processResult.errors.length,
+        titles: processResult.processed.map(p => p.metadata.title),
+    };
+    return { discovery, processing };
+}
+// ============================================================================
+// Main Handler
+// ============================================================================
+export async function handleSync(dbPath, dataDir, args, options = {}) {
+    const doPull = args.git_pull !== false;
+    const doPush = args.git_push !== false;
+    const indexNew = args.index_new !== false;
+    const dryRun = args.dry_run === true;
+    const useLegacy = args.use_legacy === true;
+    resetDatabaseConnection();
+    const result = {
+        git_pulled: false,
+        git_pushed: false,
+        sources_found: 0,
+        sources_indexed: 0,
+        already_indexed: 0,
+    };
+    // 1. Git pull
+    if (doPull) {
+        const pullResult = await gitPull(dataDir);
+        result.git_pulled = pullResult.success && (pullResult.message?.includes('Pulled') || false);
+        if (pullResult.error) {
+            result.git_error = pullResult.error;
+        }
+    }
+    // 2. Sync sources
+    if (indexNew) {
+        // Check if we have sync config
+        const config = await loadSyncConfig();
+        const hasUniversalSources = getEnabledSources(config).length > 0;
+        if (hasUniversalSources && !useLegacy) {
+            // Use new universal sync
+            const { discovery, processing } = await universalSync(dataDir, dryRun, options.hookContext);
+            result.discovery = discovery;
+            result.processing = processing;
+        }
+        // Always run legacy disk sync for backward compatibility
+        // (picks up sources added via old `lore ingest` command)
+        const legacyResult = await legacyDiskSync(dbPath, dataDir);
+        result.sources_found = legacyResult.sources_found;
+        result.sources_indexed = legacyResult.sources_indexed;
+        result.already_indexed = legacyResult.already_indexed;
+    }
+    // 3. Git push
+    if (doPush && !dryRun) {
+        const totalNew = (result.processing?.processed || 0) + result.sources_indexed;
+        if (totalNew > 0) {
+            const pushResult = await gitCommitAndPush(dataDir, `Sync: Added ${totalNew} source(s)`);
+            result.git_pushed = pushResult.success && (pushResult.message?.includes('pushed') || false);
+        }
+    }
+    return result;
+}

package/dist/mcp/server.d.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+/**
+ * Lore - MCP Server
+ *
+ * Exposes knowledge repository tools via Model Context Protocol.
+ * Supports both simple query tools and agentic research capabilities.
+ *
+ * Auto-syncs: Periodically checks for new sources and syncs them.
+ */
+export {};