@getlore/cli 0.2.0

package/dist/mcp/server.js

@@ -0,0 +1,268 @@
+#!/usr/bin/env node
+/**
+ * Lore - MCP Server
+ *
+ * Exposes knowledge repository tools via Model Context Protocol.
+ * Supports both simple query tools and agentic research capabilities.
+ *
+ * Auto-syncs: Periodically checks for new sources and syncs them.
+ */
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import chokidar from 'chokidar';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { readdir } from 'fs/promises';
+import path from 'path';
+import { toolDefinitions } from './tools.js';
+import { handleSearch } from './handlers/search.js';
+import { handleGetSource } from './handlers/get-source.js';
+import { handleListSources } from './handlers/list-sources.js';
+import { handleRetain } from './handlers/retain.js';
+import { handleIngest } from './handlers/ingest.js';
+import { handleResearch } from './handlers/research.js';
+import { handleListProjects } from './handlers/list-projects.js';
+import { handleSync } from './handlers/sync.js';
+import { handleArchiveProject } from './handlers/archive-project.js';
+import { indexExists, getAllSources } from '../core/vector-store.js';
+import { expandPath } from '../sync/config.js';
+import { getExtensionRegistry } from '../extensions/registry.js';
+import { getExtensionsDir } from '../extensions/config.js';
+import { bridgeConfigToEnv } from '../core/config.js';
+const execAsync = promisify(exec);
+// Configuration from environment
+const LORE_DATA_DIR = expandPath(process.env.LORE_DATA_DIR || '~/.lore');
+const DB_PATH = path.join(LORE_DATA_DIR, 'lore.lance');
+const SYNC_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+// Auto-sync is DISABLED by default - use `lore watch` for visible sync
+// Set LORE_AUTO_SYNC=true to enable background sync in MCP server
+const AUTO_SYNC = process.env.LORE_AUTO_SYNC === 'true';
+const AUTO_GIT_PULL = process.env.LORE_AUTO_GIT_PULL !== 'false';
+const AUTO_GIT_PUSH = process.env.LORE_AUTO_GIT_PUSH !== 'false';
+const AUTO_INDEX = process.env.LORE_AUTO_INDEX !== 'false';
+const WATCH_EXTENSIONS = process.env.LORE_EXTENSION_WATCH === 'true' || process.argv.includes('--watch');
+/**
+ * Try to git pull, handling conflicts gracefully
+ */
+async function tryGitPull() {
+    try {
+        // Check if we're in a git repo
+        await execAsync('git rev-parse --git-dir', { cwd: LORE_DATA_DIR });
+        // Stash any local changes (shouldn't be any, but just in case)
+        await execAsync('git stash', { cwd: LORE_DATA_DIR }).catch(() => { });
+        // Pull with rebase to avoid merge commits
+        const { stdout } = await execAsync('git pull --rebase', { cwd: LORE_DATA_DIR });
+        const pulled = !stdout.includes('Already up to date');
+        return { pulled };
+    }
+    catch (error) {
+        // Not a git repo or pull failed - that's okay, continue without sync
+        return { pulled: false, error: String(error) };
+    }
+}
+/**
+ * Find sources on disk that aren't in the index
+ */
+async function findUnsyncedSources() {
+    const sourcesDir = path.join(LORE_DATA_DIR, 'sources');
+    try {
+        // Get source IDs from disk
+        const diskSources = await readdir(sourcesDir, { withFileTypes: true });
+        const diskIds = diskSources
+            .filter(d => d.isDirectory() && !d.name.startsWith('.'))
+            .map(d => d.name);
+        // Get source IDs from index
+        const indexedSources = await getAllSources(DB_PATH, {});
+        const indexedIds = new Set(indexedSources.map(s => s.id));
+        // Find the difference
+        return diskIds.filter(id => !indexedIds.has(id));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Periodic sync check
+ */
+async function syncCheck() {
+    try {
+        // Use the sync handler for actual sync
+        const result = await handleSync(DB_PATH, LORE_DATA_DIR, {
+            git_pull: AUTO_GIT_PULL,
+            index_new: AUTO_INDEX,
+        }, { hookContext: { mode: 'mcp' } });
+        if (result.git_pulled) {
+            console.error('[lore] Git pulled new changes');
+        }
+        if (result.sources_indexed > 0) {
+            console.error(`[lore] Auto-indexed ${result.sources_indexed} new source(s)`);
+        }
+        else if (!AUTO_INDEX) {
+            // Check for unsynced without indexing
+            const unsynced = await findUnsyncedSources();
+            if (unsynced.length > 0) {
+                console.error(`[lore] Found ${unsynced.length} unsynced sources. Use 'sync' tool or set LORE_AUTO_INDEX=true.`);
+            }
+        }
+    }
+    catch (error) {
+        console.error('[lore] Sync check error:', error);
+    }
+}
+async function main() {
+    // Bridge config.json values into process.env (env vars take precedence)
+    try {
+        await bridgeConfigToEnv();
+    }
+    catch {
+        // Config not set up yet — fine
+    }
+    // Check if index exists
+    const hasIndex = await indexExists(DB_PATH);
+    if (!hasIndex) {
+        console.error(`[lore] No index found at ${DB_PATH}. Run 'lore ingest' to add sources.`);
+    }
+    // Auto-sync (disabled by default - use `lore watch` instead)
+    if (AUTO_SYNC) {
+        // Initial sync check
+        await syncCheck();
+        // Periodic sync check
+        setInterval(syncCheck, SYNC_INTERVAL_MS);
+        console.error(`[lore] Auto-sync enabled (every ${SYNC_INTERVAL_MS / 60000} minutes)`);
+    }
+    else {
+        console.error(`[lore] Auto-sync disabled. Use 'lore watch' for visible sync, or set LORE_AUTO_SYNC=true`);
+    }
+    const server = new Server({
+        name: 'lore',
+        version: '0.1.0',
+    }, {
+        capabilities: {
+            tools: {},
+        },
+    });
+    const extensionRegistry = await getExtensionRegistry({
+        logger: (message) => console.error(message),
+    });
+    if (WATCH_EXTENSIONS) {
+        const extensionsDir = getExtensionsDir();
+        const watchPatterns = [
+            path.join(extensionsDir, '**/package.json'),
+            path.join(extensionsDir, '**/dist/**'),
+        ];
+        let reloadTimer = null;
+        const scheduleReload = (event, filePath) => {
+            if (reloadTimer) {
+                clearTimeout(reloadTimer);
+            }
+            reloadTimer = setTimeout(async () => {
+                reloadTimer = null;
+                try {
+                    const relativePath = path.relative(extensionsDir, filePath);
+                    console.error(`[extensions] Change detected (${event}: ${relativePath})`);
+                    await extensionRegistry.reload();
+                }
+                catch (error) {
+                    console.error('[extensions] Reload failed:', error);
+                }
+            }, 500);
+        };
+        const watcher = chokidar.watch(watchPatterns, {
+            ignoreInitial: true,
+        });
+        watcher.on('all', (event, filePath) => {
+            if (!filePath) {
+                return;
+            }
+            scheduleReload(event, filePath);
+        });
+        console.error('[extensions] Watching for changes in installed extensions');
+    }
+    // List available tools (core tools only - extensions are event-driven middleware)
+    server.setRequestHandler(ListToolsRequestSchema, async () => {
+        return { tools: toolDefinitions };
+    });
+    // Handle tool calls (core tools only)
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const { name, arguments: args } = request.params;
+        try {
+            let result;
+            switch (name) {
+                // Simple query tools (cheap, fast)
+                case 'search':
+                    result = await handleSearch(DB_PATH, LORE_DATA_DIR, args);
+                    break;
+                case 'get_source':
+                    result = await handleGetSource(DB_PATH, LORE_DATA_DIR, args);
+                    break;
+                case 'list_sources':
+                    result = await handleListSources(DB_PATH, args);
+                    break;
+                case 'list_projects':
+                    result = await handleListProjects(DB_PATH);
+                    break;
+                // Push-based retention
+                case 'retain':
+                    result = await handleRetain(DB_PATH, LORE_DATA_DIR, args, {
+                        autoPush: AUTO_GIT_PUSH,
+                    });
+                    break;
+                // Direct document ingestion
+                case 'ingest':
+                    result = await handleIngest(DB_PATH, LORE_DATA_DIR, args, {
+                        autoPush: AUTO_GIT_PUSH,
+                        hookContext: { mode: 'mcp' },
+                    });
+                    break;
+                // Agentic research tool (uses Claude Agent SDK internally)
+                case 'research':
+                    result = await handleResearch(DB_PATH, LORE_DATA_DIR, args, {
+                        hookContext: { mode: 'mcp' },
+                    });
+                    break;
+                // Sync tool
+                case 'sync':
+                    result = await handleSync(DB_PATH, LORE_DATA_DIR, args, {
+                        hookContext: { mode: 'mcp' },
+                    });
+                    break;
+                // Project management
+                case 'archive_project':
+                    result = await handleArchiveProject(DB_PATH, LORE_DATA_DIR, args, {
+                        autoPush: AUTO_GIT_PUSH,
+                    });
+                    break;
+                default:
+                    throw new Error(`Unknown tool: ${name}`);
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify(result, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({ error: errorMessage }),
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+    // Start server
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+}
+main().catch((error) => {
+    console.error('Lore server error:', error);
+    process.exit(1);
+});

package/dist/mcp/tools.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Lore - MCP Tool Definitions
+ *
+ * Defines the tools exposed by the Lore MCP server.
+ * Two categories:
+ * 1. Simple tools - Direct database queries, cheap and fast
+ * 2. Agentic tools - Use Claude Agent SDK for complex research
+ *
+ * Descriptions are written for agent consumption. Agents without a skill file
+ * should understand Lore purely from these tool descriptions.
+ */
+export declare const toolDefinitions: {
+    name: string;
+    description: string;
+    inputSchema: Record<string, unknown>;
+}[];

package/dist/mcp/tools.js

@@ -0,0 +1,297 @@
+/**
+ * Lore - MCP Tool Definitions
+ *
+ * Defines the tools exposed by the Lore MCP server.
+ * Two categories:
+ * 1. Simple tools - Direct database queries, cheap and fast
+ * 2. Agentic tools - Use Claude Agent SDK for complex research
+ *
+ * Descriptions are written for agent consumption. Agents without a skill file
+ * should understand Lore purely from these tool descriptions.
+ */
+import { z } from 'zod';
+// Helper to convert Zod schema to JSON Schema
+function zodToJsonSchema(schema) {
+    // Simplified conversion - in production use zod-to-json-schema
+    if (schema instanceof z.ZodObject) {
+        const shape = schema.shape;
+        const properties = {};
+        const required = [];
+        for (const [key, value] of Object.entries(shape)) {
+            const zodValue = value;
+            properties[key] = zodToJsonSchema(zodValue);
+            if (!(zodValue instanceof z.ZodOptional)) {
+                required.push(key);
+            }
+        }
+        return {
+            type: 'object',
+            properties,
+            required: required.length > 0 ? required : undefined,
+        };
+    }
+    if (schema instanceof z.ZodString) {
+        return { type: 'string', description: schema.description };
+    }
+    if (schema instanceof z.ZodNumber) {
+        return { type: 'number', description: schema.description };
+    }
+    if (schema instanceof z.ZodBoolean) {
+        return { type: 'boolean', description: schema.description };
+    }
+    if (schema instanceof z.ZodOptional) {
+        return zodToJsonSchema(schema.unwrap());
+    }
+    if (schema instanceof z.ZodEnum) {
+        return { type: 'string', enum: schema.options, description: schema.description };
+    }
+    if (schema instanceof z.ZodArray) {
+        return { type: 'array', items: zodToJsonSchema(schema.element) };
+    }
+    return { type: 'string' };
+}
+// ============================================================================
+// Simple Query Tools
+// ============================================================================
+const SearchSchema = z.object({
+    query: z.string().describe('Search query'),
+    project: z.string().optional().describe('Filter to specific project'),
+    source_type: z
+        .string()
+        .optional()
+        .describe('Filter by source type (matches the source_type passed during ingest, e.g. "meeting", "slack", "github-issue")'),
+    content_type: z
+        .enum(['interview', 'meeting', 'conversation', 'document', 'note', 'analysis'])
+        .optional()
+        .describe('Filter by content type'),
+    limit: z.number().optional().describe('Max results (default 10)'),
+    include_archived: z
+        .boolean()
+        .optional()
+        .describe('Include sources from archived projects (default: false)'),
+    mode: z
+        .enum(['semantic', 'keyword', 'hybrid', 'regex'])
+        .optional()
+        .describe('Search mode: semantic (vector), keyword (full-text), hybrid (RRF fusion, default), regex (local grep)'),
+});
+const GetSourceSchema = z.object({
+    source_id: z.string().describe('ID of the source document'),
+    include_content: z.boolean().optional().describe('Include full original content/transcript (default: false - set to true for raw text)'),
+});
+const ListSourcesSchema = z.object({
+    project: z.string().optional().describe('Filter to specific project'),
+    source_type: z
+        .string()
+        .optional()
+        .describe('Filter by source type (matches the source_type passed during ingest, e.g. "meeting", "slack", "github-issue")'),
+    limit: z.number().optional().describe('Max results (default 20)'),
+});
+const RetainSchema = z.object({
+    content: z.string().describe('The insight, decision, or note to retain'),
+    project: z.string().describe('Project this belongs to'),
+    type: z
+        .enum(['insight', 'decision', 'requirement', 'note'])
+        .describe('Type of knowledge being retained'),
+    source_context: z
+        .string()
+        .optional()
+        .describe('Where this came from (e.g., "user interview with Sarah")'),
+    tags: z.array(z.string()).optional().describe('Optional tags for categorization'),
+});
+// ============================================================================
+// Agentic Research Tool
+// ============================================================================
+const ResearchSchema = z.object({
+    task: z
+        .string()
+        .describe('Research task description (e.g., "Find all user feedback about export performance")'),
+    project: z.string().optional().describe('Focus research on specific project'),
+    include_sources: z
+        .boolean()
+        .optional()
+        .describe('Include source document references (default: true)'),
+});
+// ============================================================================
+// Ingest Tool
+// ============================================================================
+const IngestSchema = z.object({
+    content: z.string().describe('The document content to ingest'),
+    title: z.string().describe('Title for the document'),
+    project: z.string().describe('Project this document belongs to'),
+    source_type: z
+        .string()
+        .optional()
+        .describe('Content category. Use one of these canonical values: meeting, interview, document, notes, analysis, conversation, article, slack, email, github-issue, github-pr, notion, spec, rfc, transcript, pdf, image, video, audio. Defaults to "document". Variants are auto-normalized (e.g. "Slack Thread" → "slack", "blog post" → "article").'),
+    date: z.string().optional().describe('Date of the document (ISO format, defaults to now)'),
+    participants: z
+        .array(z.string())
+        .optional()
+        .describe('People involved (for meetings/interviews)'),
+    tags: z.array(z.string()).optional().describe('Optional tags for categorization'),
+    source_url: z
+        .string()
+        .optional()
+        .describe('Original URL for citation linking (e.g., Slack permalink, Notion page URL, GitHub issue URL). Stored for traceability.'),
+    source_name: z
+        .string()
+        .optional()
+        .describe('Human-readable origin label (e.g., "Slack #product-team", "GitHub issue #42", "Notion: Sprint Planning")'),
+});
+// ============================================================================
+// Sync Tool
+// ============================================================================
+const SyncSchema = z.object({
+    git_pull: z
+        .boolean()
+        .optional()
+        .describe('Pull latest changes from git remote (default: true)'),
+    git_push: z
+        .boolean()
+        .optional()
+        .describe('Push local changes to git remote (default: true)'),
+    index_new: z
+        .boolean()
+        .optional()
+        .describe('Index any new sources found on disk (default: true)'),
+    dry_run: z
+        .boolean()
+        .optional()
+        .describe('Show what would be synced without actually processing (default: false)'),
+    use_legacy: z
+        .boolean()
+        .optional()
+        .describe('Use only legacy disk-based sync, skip universal sync (default: false)'),
+});
+// ============================================================================
+// Project Management Tools
+// ============================================================================
+const ArchiveProjectSchema = z.object({
+    project: z.string().describe('Name of the project to archive'),
+    reason: z
+        .string()
+        .optional()
+        .describe('Reason for archiving (e.g., "Pivoted to new approach", "Project completed")'),
+    successor_project: z
+        .string()
+        .optional()
+        .describe('Name of the project that supersedes this one, if any'),
+});
+// ============================================================================
+// Tool Definitions for MCP
+// ============================================================================
+export const toolDefinitions = [
+    // Simple tools
+    {
+        name: 'search',
+        description: `Search the Lore knowledge base. Returns source summaries with relevance scores, matching quotes, and themes.
+
+SEARCH MODES (pick based on your query):
+- hybrid (default): Best for most queries. Combines vector similarity + full-text search via RRF fusion.
+- semantic: Vector similarity only. Use for conceptual queries ("pain points", "user frustrations") where exact terms don't matter.
+- keyword: Full-text search only. Use for exact terms, identifiers, or proper nouns ("TS-01", "OAuth", specific names).
+- regex: Pattern matching in local files. Use for code patterns or complex text matching ("OAuth.*config").
+
+WHEN TO USE THIS TOOL:
+- Quick lookups of specific information
+- Finding sources related to a topic
+- Gathering context before answering a question
+- Any query where you expect 1-3 relevant sources to answer it
+
+USE 'research' INSTEAD when the question requires cross-referencing multiple sources, detecting patterns across documents, or synthesizing findings with citations. 'research' costs more API calls — avoid it for simple lookups.`,
+        inputSchema: zodToJsonSchema(SearchSchema),
+    },
+    {
+        name: 'get_source',
+        description: `Retrieve full details of a specific source document by ID. Returns metadata, summary, quotes, themes, and optionally the complete original content.
+
+Set include_content=true to get the full raw text (transcript, document body, etc.). By default only metadata and summary are returned.
+
+USE THIS AFTER 'search' returns a relevant source_id and you need the full document for detailed analysis or quoting.`,
+        inputSchema: zodToJsonSchema(GetSourceSchema),
+    },
+    {
+        name: 'list_sources',
+        description: `List all sources in the knowledge base, optionally filtered by project or type. Returns summaries sorted by date (newest first).
+
+Use this to browse what exists in a project, understand the scope of available knowledge, or check if content has already been ingested before calling 'ingest'.`,
+        inputSchema: zodToJsonSchema(ListSourcesSchema),
+    },
+    {
+        name: 'list_projects',
+        description: `List all projects with source counts and latest activity dates. Use this to discover what knowledge domains exist before searching or ingesting content.`,
+        inputSchema: {
+            type: 'object',
+            properties: {},
+        },
+    },
+    {
+        name: 'retain',
+        description: `Save a discrete insight, decision, requirement, or note to the knowledge base. These are short, synthesized pieces of knowledge — NOT full documents.
+
+Examples of what to retain:
+- A decision: "We chose JWT over session cookies because of mobile app requirements"
+- An insight: "3 out of 5 users mentioned export speed as their top frustration"
+- A requirement: "Must support SSO for enterprise customers"
+
+USE 'ingest' INSTEAD for full documents, meeting notes, transcripts, or any content longer than a few paragraphs.`,
+        inputSchema: zodToJsonSchema(RetainSchema),
+    },
+    // Agentic tool
+    {
+        name: 'research',
+        description: `Run a comprehensive research query across the knowledge base. An internal agent iteratively searches, reads sources, cross-references findings, and synthesizes a research package with full citations.
+
+Returns: summary, key findings, supporting quotes with citations, conflicts detected between sources, and suggested follow-up queries.
+
+WHEN TO USE:
+- Questions that span multiple sources ("What do we know about authentication?")
+- Detecting patterns or contradictions across documents
+- Building a cited research package for decision-making
+- Open-ended exploration of a topic
+
+COST: This tool makes multiple LLM calls internally (typically 3-8 search + read cycles). For simple lookups, use 'search' instead — it's 10x cheaper and faster.`,
+        inputSchema: zodToJsonSchema(ResearchSchema),
+    },
+    // Ingest tool
+    {
+        name: 'ingest',
+        description: `Push content into the Lore knowledge base. This is the primary way to add documents from external systems (Slack threads, Notion pages, GitHub issues, meeting notes, emails, etc.).
+
+IDEMPOTENT: Content is deduplicated by SHA256 hash. Calling ingest with identical content returns {deduplicated: true} immediately — no LLM calls, no disk writes. Safe to call repeatedly.
+
+WHAT HAPPENS:
+1. Content hash checked for deduplication
+2. Document saved to disk
+3. LLM extracts summary, themes, and key quotes
+4. Embedding generated for semantic search
+5. Indexed in Supabase for instant retrieval
+
+BEST PRACTICES:
+- Always pass source_url when available (enables citation linking back to the original)
+- Use source_name for human-readable origin context (e.g., "Slack #product-team")
+- source_type is a free-form hint — use whatever describes the content (slack, email, notion, github-issue, etc.)
+- Use 'retain' instead for short discrete insights/decisions (not full documents)`,
+        inputSchema: zodToJsonSchema(IngestSchema),
+    },
+    // Sync tool
+    {
+        name: 'sync',
+        description: `Sync the knowledge base from configured source directories. Two-phase process:
+
+Phase 1 (Discovery — free, no LLM calls): Scans configured directories, computes content hashes, identifies new files.
+Phase 2 (Processing — only new files): Extracts metadata via LLM, generates embeddings, stores in Supabase.
+
+Use this when source directories have been updated externally, or to refresh the index after manual file changes. Source directories are configured via 'lore sources add' CLI command.
+
+Note: For pushing content from agents, use 'ingest' instead — it's the direct path.`,
+        inputSchema: zodToJsonSchema(SyncSchema),
+    },
+    // Project management
+    {
+        name: 'archive_project',
+        description: `Archive a project and exclude its sources from default search results. Archived sources are preserved for historical reference and can be included with include_archived=true in search.
+
+Only use when explicitly requested — this is a curation action, not an automatic cleanup.`,
+        inputSchema: zodToJsonSchema(ArchiveProjectSchema),
+    },
+];

package/dist/sync/config.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Lore - Sync Configuration
+ *
+ * Loads and validates sync-sources.json from ~/.config/lore/sync-sources.json
+ * This config is machine-specific (NOT in lore-data).
+ */
+export interface SyncSource {
+    name: string;
+    path: string;
+    glob: string;
+    project: string;
+    enabled: boolean;
+}
+export interface SyncConfig {
+    version: number;
+    sources: SyncSource[];
+}
+export declare function getConfigPath(): string;
+export declare function expandPath(p: string): string;
+export declare function loadSyncConfig(): Promise<SyncConfig>;
+export declare function saveSyncConfig(config: SyncConfig): Promise<void>;
+export declare function initializeSyncConfig(): Promise<SyncConfig>;
+export declare function addSyncSource(source: SyncSource): Promise<SyncConfig>;
+export declare function updateSyncSource(name: string, updates: Partial<SyncSource>): Promise<SyncConfig>;
+export declare function removeSyncSource(name: string): Promise<SyncConfig>;
+export declare function getEnabledSources(config: SyncConfig): SyncSource[];