@getlore/cli 0.6.0 → 0.8.0

package/dist/cli/commands/docs.js

@@ -106,7 +106,7 @@ export function registerDocsCommand(program, defaultDataDir) {
         .option('-d, --data-dir <dir>', 'Data directory', defaultDataDir)
         .option('--no-push', 'Skip git push')
         .action(async (content, options) => {
-        const { handleRetain } = await import('../../mcp/handlers/retain.js');
+        const { handleIngest } = await import('../../mcp/handlers/ingest.js');
         const dataDir = options.dataDir;
         const dbPath = path.join(dataDir, 'lore.lance');
         const validTypes = ['insight', 'decision', 'requirement', 'note'];
@@ -114,21 +114,28 @@ export function registerDocsCommand(program, defaultDataDir) {
             console.error(`Invalid type: ${options.type}. Must be one of: ${validTypes.join(', ')}`);
             process.exit(1);
         }
-        const result = await handleRetain(dbPath, dataDir, {
+        // Map CLI type to source_type
+        const sourceTypeMap = {
+            decision: 'notes',
+            requirement: 'notes',
+            insight: 'notes',
+            note: 'notes',
+        };
+        const result = await handleIngest(dbPath, dataDir, {
             content,
             project: options.project,
-            type: options.type,
-            source_context: options.context,
+            title: `${options.type.charAt(0).toUpperCase() + options.type.slice(1)}: ${content.slice(0, 50)}${content.length > 50 ? '...' : ''}`,
+            source_type: sourceTypeMap[options.type] || 'notes',
             tags: options.tags?.split(',').map((t) => t.trim()),
-        }, { autoPush: options.push !== false });
+        }, { autoPush: options.push !== false, hookContext: { mode: 'cli' } });
         if (result.success) {
-            console.log(`\n✓ ${result.message}`);
+            console.log(`\n✓ Created ${options.type} for project "${options.project}"`);
             console.log(`  ID: ${result.id}`);
             console.log(`  Indexed: ${result.indexed ? 'yes' : 'no'}`);
             console.log(`  Synced: ${result.synced ? 'yes' : 'no'}`);
         }
         else {
-            console.error(`\nFailed to create: ${result.message}`);
+            console.error(`\nFailed to create ${options.type}`);
             process.exit(1);
         }
     });

package/dist/core/data-repo.js

@@ -19,7 +19,6 @@ import path from 'path';
 export async function initDataRepo(dirPath) {
     await mkdir(dirPath, { recursive: true });
     await mkdir(path.join(dirPath, 'sources'), { recursive: true });
-    await mkdir(path.join(dirPath, 'retained'), { recursive: true });
     // Create .gitignore if missing
     const gitignorePath = path.join(dirPath, '.gitignore');
     if (!existsSync(gitignorePath)) {
@@ -34,8 +33,7 @@ Your personal knowledge repository for Lore.
 
 ## Structure
 
-- \`sources/\` - Ingested documents
-- \`retained/\` - Explicitly saved insights
+- \`sources/\` - Ingested content
 
 Vector embeddings are stored in Supabase (cloud) for multi-machine access.
 `);

package/dist/core/types.d.ts

@@ -8,7 +8,7 @@
  */
 export type SourceType = string;
 export type SearchMode = 'semantic' | 'keyword' | 'hybrid' | 'regex';
-export type ContentType = 'interview' | 'meeting' | 'conversation' | 'document' | 'note' | 'analysis' | 'survey' | 'research' | 'decision' | 'insight' | 'requirement';
+export type ContentType = 'interview' | 'meeting' | 'conversation' | 'document' | 'note' | 'analysis' | 'survey' | 'research';
 export interface SourceDocument {
     id: string;
     source_type: SourceType;
@@ -154,13 +154,6 @@ export interface SearchArgs {
     limit?: number;
     mode?: SearchMode;
 }
-export interface RetainArgs {
-    content: string;
-    project: string;
-    type: 'insight' | 'decision' | 'requirement' | 'note';
-    source_context?: string;
-    tags?: string[];
-}
 export interface ResearchArgs {
     task: string;
     project?: string;

package/dist/core/vector-store.js

@@ -111,9 +111,7 @@ export async function addSource(_dbPath, source, vector, extras) {
     if (extras?.source_name) {
         record.source_name = extras.source_name;
     }
-    const { error } = await client.from('sources').upsert(record, {
-        ignoreDuplicates: true,
-    });
+    const { error } = await client.from('sources').upsert(record);
     if (error) {
         // Duplicate content_hash for this user — document already exists, skip silently
         if (error.code === '23505') {

package/dist/extensions/proposals.d.ts

@@ -2,13 +2,12 @@
  * Proposal-based write system for extensions
  */
 export interface ProposedChange {
-    type: 'create_source' | 'update_source' | 'delete_source' | 'retain_insight' | 'add_tags';
+    type: 'create_source' | 'update_source' | 'delete_source' | 'add_tags';
     title?: string;
     content?: string;
     project?: string;
     sourceId?: string;
     changes?: Record<string, unknown>;
-    insight?: string;
     tags?: string[];
     reason: string;
 }

package/dist/extensions/proposals.js

@@ -7,7 +7,6 @@ import { mkdir, readFile, readdir, writeFile } from 'fs/promises';
 import os from 'os';
 import path from 'path';
 import { handleIngest } from '../mcp/handlers/ingest.js';
-import { handleRetain } from '../mcp/handlers/retain.js';
 import { getDatabase, getSourceById } from '../core/vector-store.js';
 export function getPendingDir() {
     return path.join(os.homedir(), '.config', 'lore', 'pending');
@@ -84,18 +83,6 @@ async function applyProposalChange(proposal, dbPath, dataDir) {
             }, { hookContext: { mode: 'cli' } });
             return;
         }
-        case 'retain_insight': {
-            if (!change.insight) {
-                throw new Error('retain_insight requires insight');
-            }
-            const project = change.project || proposal.extensionName;
-            await handleRetain(dbPath, dataDir, {
-                content: change.insight,
-                project,
-                type: 'insight',
-            }, {});
-            return;
-        }
         case 'update_source': {
             if (!change.sourceId || !change.changes) {
                 throw new Error('update_source requires sourceId and changes');

package/dist/extensions/registry.js

@@ -83,7 +83,7 @@ export function createProposeFunction(extensionName, permissions) {
     return async (change) => {
         // Enforce permissions
         const perms = permissions || {};
-        if (change.type === 'create_source' || change.type === 'retain_insight') {
+        if (change.type === 'create_source') {
             if (!perms.proposeCreate) {
                 throw new Error(`Extension "${extensionName}" does not have permission to propose creating documents. Add permissions.proposeCreate = true to the extension.`);
             }

package/dist/mcp/handlers/ingest.d.ts

@@ -7,7 +7,7 @@
  */
 interface IngestArgs {
     content: string;
-    title: string;
+    title?: string;
     project: string;
     source_type?: string;
     date?: string;

package/dist/mcp/handlers/ingest.js

@@ -123,9 +123,11 @@ function mapContentType(sourceType) {
     }
 }
 export async function handleIngest(dbPath, dataDir, args, options = {}) {
-    const { content, title, project, source_type: raw_source_type, date, participants = [], tags = [], source_url, source_name, } = args;
+    const { content, project, source_type: raw_source_type, date, participants = [], tags = [], source_url, source_name, } = args;
     const { autoPush = true, hookContext } = options;
     const source_type = normalizeSourceType(raw_source_type);
+    // Auto-generate title if not provided
+    const title = args.title || `${source_type.charAt(0).toUpperCase() + source_type.slice(1)}: ${content.slice(0, 50)}${content.length > 50 ? '...' : ''}`;
     // Content hash deduplication — skip everything if already ingested
     const contentHash = createHash('sha256').update(content).digest('hex');
     try {
@@ -170,12 +172,17 @@ export async function handleIngest(dbPath, dataDir, args, options = {}) {
     await writeFile(path.join(sourceDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
     // Save content.md
     await writeFile(path.join(sourceDir, 'content.md'), content);
-    // Extract insights using LLM
+    // Extract insights using LLM (skip for short content)
     let summary = content.slice(0, 200) + (content.length > 200 ? '...' : '');
     let themes = [];
     let quotes = [];
-    try {
-        if (content.trim().length > 100) {
+    const isShortContent = content.trim().length <= 500;
+    if (isShortContent) {
+        // Short content fast path — use content as its own summary, skip LLM extraction
+        summary = content;
+    }
+    else {
+        try {
             const insights = await extractInsights(content, title, id, { contentType });
             summary = insights.summary;
             themes = insights.themes.map((t) => ({ name: t.name, quotes: [] }));
@@ -183,10 +190,10 @@ export async function handleIngest(dbPath, dataDir, args, options = {}) {
             // Save insights.json
             await writeFile(path.join(sourceDir, 'insights.json'), JSON.stringify({ summary, themes, quotes }, null, 2));
         }
-    }
-    catch (error) {
-        console.error('Failed to extract insights:', error);
-        // Continue with basic summary
+        catch (error) {
+            console.error('Failed to extract insights:', error);
+            // Continue with basic summary
+        }
     }
     // Add to vector store immediately
     try {

package/dist/mcp/handlers/research-agent.d.ts

@@ -8,6 +8,7 @@
  * 4. Synthesizes findings into a comprehensive research package
  */
 import type { ResearchPackage } from '../../core/types.js';
+import type { ProgressCallback } from './research.js';
 interface ResearchAgentArgs {
     task: string;
     project?: string;
@@ -17,5 +18,5 @@ interface ResearchAgentArgs {
 /**
  * Run the agentic research
  */
-export declare function runResearchAgent(dbPath: string, dataDir: string, args: ResearchAgentArgs): Promise<ResearchPackage>;
+export declare function runResearchAgent(dbPath: string, dataDir: string, args: ResearchAgentArgs, onProgress?: ProgressCallback): Promise<ResearchPackage>;
 export {};

package/dist/mcp/handlers/research-agent.js

@@ -27,9 +27,9 @@ function createLoreToolsServer(dbPath, dataDir, archivedProjects) {
             tool('search', 'Semantic search across all sources in the knowledge repository. Returns summaries with relevant quotes. Use this to find information related to a topic.', {
                 query: z.string().describe('Semantic search query - describe what you\'re looking for'),
                 source_type: z
-                    .enum(['granola', 'claude-code', 'claude-desktop', 'chatgpt', 'markdown', 'document'])
+                    .string()
                     .optional()
-                    .describe('Filter by source type (e.g., "granola" for meeting transcripts)'),
+                    .describe('Filter by source type (e.g., "meeting", "slack", "document")'),
                 content_type: z
                     .enum(['interview', 'meeting', 'conversation', 'document', 'note', 'analysis'])
                     .optional()
@@ -91,6 +91,8 @@ ${quotes}`;
                         .slice(0, 10)
                         .map((q) => `- [${q.speaker || 'unknown'}] "${q.text}"`)
                         .join('\n');
+                    const sourceUrlLine = source.source_url ? `\n**Source URL:** ${source.source_url}` : '';
+                    const sourceNameLine = source.source_name ? `\n**Source:** ${source.source_name}` : '';
                     return {
                         content: [
                             {
@@ -99,7 +101,7 @@ ${quotes}`;
 
 **Type:** ${source.source_type} / ${source.content_type}
 **Created:** ${source.created_at}
-**Projects:** ${source.projects.join(', ') || 'none'}
+**Projects:** ${source.projects.join(', ') || 'none'}${sourceUrlLine}${sourceNameLine}
 
 ## Summary
 ${source.summary}
@@ -122,9 +124,9 @@ ${quotes || 'No quotes extracted'}`,
             // List sources - browse available sources
             tool('list_sources', 'List all sources in the repository. Use this to understand what knowledge is available before searching.', {
                 source_type: z
-                    .enum(['granola', 'claude-code', 'claude-desktop', 'chatgpt', 'markdown', 'document'])
+                    .string()
                     .optional()
-                    .describe('Filter by source type'),
+                    .describe('Filter by source type (e.g., "meeting", "slack", "document")'),
                 project: z.string().optional().describe('Filter to specific project'),
                 limit: z.number().optional().describe('Max results (default 20)'),
             }, async (args) => {
@@ -232,7 +234,7 @@ Now begin your research. Use the tools iteratively until you have comprehensive
 /**
  * Run the agentic research
  */
-export async function runResearchAgent(dbPath, dataDir, args) {
+export async function runResearchAgent(dbPath, dataDir, args, onProgress) {
     const { task, project, include_sources = true } = args;
     // Load archived projects to filter (extract just the project names)
     const archivedProjectsData = await loadArchivedProjects(dataDir);
@@ -245,6 +247,8 @@ export async function runResearchAgent(dbPath, dataDir, args) {
     let lastAssistantMessage = '';
     try {
         // Run the agent
+        let turnCount = 0;
+        await onProgress?.(5, undefined, 'Starting research agent...');
         for await (const message of query({
             prompt: `Research task: ${task}${project ? ` (project: ${project})` : ''}`,
             options: {
@@ -261,8 +265,9 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                 permissionMode: 'acceptEdits', // Auto-approve tool calls
             },
         })) {
-            // Capture assistant messages (intermediate)
+            // Capture assistant messages and extract tool call details
             if (message.type === 'assistant') {
+                turnCount++;
                 const msg = message;
                 if (msg.message?.content) {
                     const content = msg.message.content;
@@ -270,9 +275,30 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                         lastAssistantMessage = content;
                     }
                     else if (Array.isArray(content)) {
-                        const textBlocks = content.filter((b) => b.type === 'text');
-                        if (textBlocks.length > 0) {
-                            lastAssistantMessage = textBlocks.map((b) => b.text).join('\n');
+                        // Extract tool_use blocks to report what the agent is doing
+                        for (const block of content) {
+                            if (block.type === 'tool_use') {
+                                const input = block.input;
+                                const toolShort = block.name.replace('mcp__lore-tools__', '');
+                                if (toolShort === 'search' && input.query) {
+                                    await onProgress?.(0, undefined, `Searching: "${input.query}"`);
+                                }
+                                else if (toolShort === 'get_source' && input.source_id) {
+                                    await onProgress?.(0, undefined, `Reading source: ${input.source_id}`);
+                                }
+                                else if (toolShort === 'list_sources') {
+                                    const filter = input.project ? ` (project: ${input.project})` : '';
+                                    await onProgress?.(0, undefined, `Listing sources${filter}`);
+                                }
+                            }
+                            else if (block.type === 'text' && block.text) {
+                                lastAssistantMessage = block.text;
+                                // Send a brief snippet of agent reasoning
+                                const snippet = block.text.substring(0, 120).replace(/\n/g, ' ');
+                                if (snippet.length > 10) {
+                                    await onProgress?.(0, undefined, `Agent thinking: ${snippet}...`);
+                                }
+                            }
                         }
                     }
                 }
@@ -282,16 +308,22 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                 const msg = message;
                 if (msg.subtype === 'success' && msg.result) {
                     lastAssistantMessage = msg.result;
+                    await onProgress?.(0, undefined, `Research complete (${msg.num_turns} turns)`);
                     console.error(`[research-agent] Completed in ${msg.num_turns} turns`);
                 }
                 else if (msg.subtype?.startsWith('error')) {
                     console.error(`[research-agent] Error: ${msg.subtype}`, msg.errors);
                 }
             }
-            // Log tool usage for debugging
+            // Log tool results via the summary message
             if (message.type === 'tool_use_summary') {
                 const msg = message;
-                console.error(`[research-agent] Tool: ${msg.tool_name || 'unknown'}`);
+                if (msg.summary) {
+                    // The summary often contains "Found X results" or similar
+                    const summarySnippet = msg.summary.substring(0, 150).replace(/\n/g, ' ');
+                    await onProgress?.(0, undefined, `Result: ${summarySnippet}`);
+                }
+                console.error(`[research-agent] Tool complete (turn ${turnCount})`);
             }
         }
         // Parse the final result from the agent's output

package/dist/mcp/handlers/research.d.ts

@@ -6,17 +6,36 @@
  * 2. SIMPLE (fallback): Single-pass search + GPT-4o-mini synthesis
  *
  * Set LORE_RESEARCH_MODE=simple to use the fallback mode.
+ *
+ * MCP integration: Research runs asynchronously. The `research` tool returns
+ * immediately with a job_id. Use `research_status` to poll for results.
  */
 import type { ResearchPackage } from '../../core/types.js';
+/**
+ * Start research asynchronously and return a job ID immediately.
+ */
+export declare function startResearchJob(dbPath: string, dataDir: string, args: ResearchArgs, options?: {
+    hookContext?: {
+        mode: 'mcp' | 'cli';
+    };
+    onProgress?: ProgressCallback;
+}): {
+    job_id: string;
+    status: string;
+    message: string;
+};
+export declare function getResearchJobStatus(jobId: string): Promise<Record<string, unknown>>;
 interface ResearchArgs {
     task: string;
     project?: string;
     content_type?: string;
     include_sources?: boolean;
 }
+export type ProgressCallback = (progress: number, total?: number, message?: string) => Promise<void>;
 export declare function handleResearch(dbPath: string, dataDir: string, args: ResearchArgs, options?: {
     hookContext?: {
         mode: 'mcp' | 'cli';
     };
+    onProgress?: ProgressCallback;
 }): Promise<ResearchPackage>;
 export {};

package/dist/mcp/handlers/research.js

@@ -6,13 +6,145 @@
  * 2. SIMPLE (fallback): Single-pass search + GPT-4o-mini synthesis
  *
  * Set LORE_RESEARCH_MODE=simple to use the fallback mode.
+ *
+ * MCP integration: Research runs asynchronously. The `research` tool returns
+ * immediately with a job_id. Use `research_status` to poll for results.
  */
 import OpenAI from 'openai';
+import { randomUUID } from 'crypto';
 import { searchSources } from '../../core/vector-store.js';
 import { generateEmbedding } from '../../core/embedder.js';
 import { loadArchivedProjects } from './archive-project.js';
 import { runResearchAgent } from './research-agent.js';
 import { getExtensionRegistry } from '../../extensions/registry.js';
+const jobStore = new Map();
+// Clean up old jobs after 10 minutes
+const JOB_TTL_MS = 10 * 60 * 1000;
+function cleanOldJobs() {
+    const now = Date.now();
+    for (const [id, job] of jobStore) {
+        const startTime = new Date(job.startedAt).getTime();
+        if (now - startTime > JOB_TTL_MS) {
+            jobStore.delete(id);
+        }
+    }
+}
+/**
+ * Start research asynchronously and return a job ID immediately.
+ */
+export function startResearchJob(dbPath, dataDir, args, options = {}) {
+    cleanOldJobs();
+    const jobId = randomUUID();
+    const now = new Date().toISOString();
+    const job = {
+        id: jobId,
+        task: args.task,
+        project: args.project,
+        status: 'running',
+        startedAt: now,
+        lastActivityAt: now,
+        activity: ['Starting research...'],
+    };
+    jobStore.set(jobId, job);
+    // Fire and forget — runs in the background
+    handleResearch(dbPath, dataDir, args, {
+        ...options,
+        onProgress: async (_p, _t, message) => {
+            const j = jobStore.get(jobId);
+            if (j && message) {
+                j.activity.push(message);
+                j.lastActivityAt = new Date().toISOString();
+            }
+        },
+    })
+        .then((result) => {
+        const j = jobStore.get(jobId);
+        if (j) {
+            j.status = 'complete';
+            j.completedAt = new Date().toISOString();
+            j.result = result;
+            j.activity.push('Research complete');
+        }
+    })
+        .catch((err) => {
+        const j = jobStore.get(jobId);
+        if (j) {
+            j.status = 'error';
+            j.completedAt = new Date().toISOString();
+            j.error = err instanceof Error ? err.message : String(err);
+            j.activity.push(`Failed: ${j.error}`);
+        }
+    })
+        .catch((err) => {
+        // Final safety net for errors in the handlers above
+        console.error(`[research] Critical error in job ${jobId}:`, err);
+    });
+    return {
+        job_id: jobId,
+        status: 'running',
+        message: `Research started for: "${args.task}". Poll research_status with job_id "${jobId}" every 15-20 seconds. This typically takes 2-8 minutes — do not abandon early.`,
+    };
+}
+/**
+ * Check status of a research job.
+ * Long-polls for up to POLL_WAIT_MS, returning early if the job completes.
+ */
+const POLL_WAIT_MS = 20_000;
+const POLL_INTERVAL_MS = 1_000;
+export async function getResearchJobStatus(jobId) {
+    let job = jobStore.get(jobId);
+    if (!job) {
+        return { status: 'not_found', job_id: jobId };
+    }
+    // If already done, return immediately
+    if (job.status !== 'running') {
+        return formatJobResponse(job);
+    }
+    // Long-poll: wait up to POLL_WAIT_MS for completion, checking every second
+    const deadline = Date.now() + POLL_WAIT_MS;
+    while (Date.now() < deadline) {
+        await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+        // Re-fetch to avoid stale reference if job was cleaned up
+        job = jobStore.get(jobId);
+        if (!job) {
+            return { status: 'not_found', job_id: jobId };
+        }
+        if (job.status !== 'running') {
+            return formatJobResponse(job);
+        }
+    }
+    return formatJobResponse(job);
+}
+function formatJobResponse(job) {
+    const elapsed = Math.round((Date.now() - new Date(job.startedAt).getTime()) / 1000);
+    if (job.status === 'complete') {
+        return {
+            status: 'complete',
+            job_id: job.id,
+            task: job.task,
+            elapsed_seconds: elapsed,
+            result: job.result,
+        };
+    }
+    if (job.status === 'error') {
+        return {
+            status: 'error',
+            job_id: job.id,
+            task: job.task,
+            elapsed_seconds: elapsed,
+            error: job.error,
+        };
+    }
+    return {
+        status: 'running',
+        job_id: job.id,
+        task: job.task,
+        elapsed_seconds: elapsed,
+        total_steps: job.activity.length,
+        activity: job.activity,
+        message: `Research is still running (${elapsed}s elapsed, ${job.activity.length} steps completed). This is normal — deep research takes 2-8 minutes. Keep polling.`,
+    };
+}
 // Lazy initialization for OpenAI (only used in simple mode)
 let openaiClient = null;
 function getOpenAI() {
@@ -109,12 +241,15 @@ Respond with only the JSON object.`;
 }
 export async function handleResearch(dbPath, dataDir, args, options = {}) {
     const { task, project, include_sources = true } = args;
+    const { onProgress } = options;
     // Check if we should use agentic mode (default) or simple mode (fallback)
     const useAgenticMode = process.env.LORE_RESEARCH_MODE !== 'simple';
     if (useAgenticMode) {
         console.error('[research] Using agentic mode (Claude Agent SDK)');
+        await onProgress?.(0, undefined, 'Starting agentic research...');
         try {
-            const result = await runResearchAgent(dbPath, dataDir, args);
+            const result = await runResearchAgent(dbPath, dataDir, args, onProgress);
+            await onProgress?.(100, 100, 'Research complete');
             await runResearchCompletedHook(result, {
                 mode: options.hookContext?.mode || 'mcp',
                 dataDir,
@@ -124,11 +259,14 @@ export async function handleResearch(dbPath, dataDir, args, options = {}) {
         }
         catch (error) {
             console.error('[research] Agentic mode failed, falling back to simple mode:', error);
+            await onProgress?.(0, undefined, 'Agentic mode failed, falling back to simple mode...');
             // Fall through to simple mode
         }
     }
     console.error('[research] Using simple mode (single-pass synthesis)');
-    const result = await handleResearchSimple(dbPath, dataDir, args);
+    await onProgress?.(0, undefined, 'Starting simple research...');
+    const result = await handleResearchSimple(dbPath, dataDir, args, onProgress);
+    await onProgress?.(100, 100, 'Research complete');
     await runResearchCompletedHook(result, {
         mode: options.hookContext?.mode || 'mcp',
         dataDir,
@@ -140,7 +278,7 @@ export async function handleResearch(dbPath, dataDir, args, options = {}) {
  * Simple research mode - single pass search + synthesis
  * This is the fallback when agentic mode fails or is disabled
  */
-async function handleResearchSimple(dbPath, dataDir, args) {
+async function handleResearchSimple(dbPath, dataDir, args, onProgress) {
     const { task, project, include_sources = true } = args;
     // Use sensible defaults for simple mode
     const sourceLimit = 10;
@@ -149,7 +287,9 @@ async function handleResearchSimple(dbPath, dataDir, args) {
     const archivedProjects = await loadArchivedProjects(dataDir);
     const archivedNames = new Set(archivedProjects.map((p) => p.project.toLowerCase()));
     // Step 1: Search for relevant sources (fetch extra to account for archived filtering)
+    await onProgress?.(10, 100, 'Generating embeddings...');
     const queryVector = await generateEmbedding(task);
+    await onProgress?.(30, 100, 'Searching sources...');
     const rawSources = await searchSources(dbPath, queryVector, {
         limit: sourceLimit * 2,
         project,
@@ -172,6 +312,7 @@ async function handleResearchSimple(dbPath, dataDir, args) {
         }
     }
     // Step 3: Synthesize findings with LLM (conflict-aware)
+    await onProgress?.(60, 100, 'Synthesizing findings...');
     // Note: Decisions are now extracted at query time by the agentic research mode
     const synthesis = await synthesizeFindings(task, sources.map((s) => ({
         id: s.id,

package/dist/mcp/handlers/sync.d.ts

@@ -44,5 +44,6 @@ export declare function handleSync(dbPath: string, dataDir: string, args: SyncAr
     hookContext?: {
         mode: 'mcp' | 'cli';
     };
+    onProgress?: (progress: number, total?: number, message?: string) => Promise<void>;
 }): Promise<SyncResult>;
 export {};

package/dist/mcp/handlers/sync.js

@@ -233,8 +233,10 @@ export async function handleSync(dbPath, dataDir, args, options = {}) {
         already_indexed: 0,
         reconciled: 0,
     };
+    const { onProgress } = options;
     // 1. Git pull
     if (doPull) {
+        await onProgress?.(5, undefined, 'Pulling from git...');
         const pullResult = await gitPull(dataDir);
         result.git_pulled = pullResult.success && (pullResult.message?.includes('Pulled') || false);
         if (pullResult.error) {
@@ -248,17 +250,20 @@ export async function handleSync(dbPath, dataDir, args, options = {}) {
         const hasUniversalSources = getEnabledSources(config).length > 0;
         if (hasUniversalSources && !useLegacy) {
             // Use new universal sync
+            await onProgress?.(20, undefined, 'Discovering new files...');
             const { discovery, processing } = await universalSync(dataDir, dryRun, options.hookContext);
             result.discovery = discovery;
             result.processing = processing;
         }
         // Always run legacy disk sync for backward compatibility
         // (picks up sources added via old `lore ingest` command)
+        await onProgress?.(60, undefined, 'Running legacy sync...');
         const legacyResult = await legacyDiskSync(dbPath, dataDir);
         result.sources_found = legacyResult.sources_found;
         result.sources_indexed = legacyResult.sources_indexed;
         result.already_indexed = legacyResult.already_indexed;
         // Reconcile: ensure every Supabase source has local content.md
+        await onProgress?.(80, undefined, 'Reconciling local content...');
         result.reconciled = await reconcileLocalContent(dataDir);
     }
     // 3. Git push

package/dist/mcp/server.js

@@ -19,9 +19,8 @@ import { toolDefinitions } from './tools.js';
 import { handleSearch } from './handlers/search.js';
 import { handleGetSource } from './handlers/get-source.js';
 import { handleListSources } from './handlers/list-sources.js';
-import { handleRetain } from './handlers/retain.js';
 import { handleIngest } from './handlers/ingest.js';
-import { handleResearch } from './handlers/research.js';
+import { startResearchJob, getResearchJobStatus } from './handlers/research.js';
 import { handleListProjects } from './handlers/list-projects.js';
 import { handleSync } from './handlers/sync.js';
 import { handleArchiveProject } from './handlers/archive-project.js';
@@ -136,7 +135,7 @@ async function main() {
     }
     const server = new Server({
         name: 'lore',
-        version: '0.1.0',
+        version: '0.8.0',
     }, {
         capabilities: {
             tools: {},
@@ -184,8 +183,25 @@ async function main() {
         return { tools: toolDefinitions };
     });
     // Handle tool calls (core tools only)
-    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
         const { name, arguments: args } = request.params;
+        // Build a progress callback for long-running tools.
+        // If the client sent a progressToken, we send notifications/progress back;
+        // otherwise, onProgress is a no-op.
+        const progressToken = request.params._meta?.progressToken;
+        const onProgress = progressToken
+            ? async (progress, total, message) => {
+                try {
+                    await extra.sendNotification({
+                        method: 'notifications/progress',
+                        params: { progressToken, progress, ...(total != null ? { total } : {}), ...(message ? { message } : {}) },
+                    });
+                }
+                catch {
+                    // Progress notifications are best-effort
+                }
+            }
+            : undefined;
         try {
             let result;
             switch (name) {
@@ -202,12 +218,6 @@ async function main() {
                 case 'list_projects':
                     result = await handleListProjects(DB_PATH);
                     break;
-                // Push-based retention
-                case 'retain':
-                    result = await handleRetain(DB_PATH, LORE_DATA_DIR, args, {
-                        autoPush: AUTO_GIT_PUSH,
-                    });
-                    break;
                 // Direct document ingestion
                 case 'ingest':
                     result = await handleIngest(DB_PATH, LORE_DATA_DIR, args, {
@@ -215,16 +225,22 @@ async function main() {
                         hookContext: { mode: 'mcp' },
                     });
                     break;
-                // Agentic research tool (uses Claude Agent SDK internally)
+                // Agentic research tool — runs async, returns job_id immediately
                 case 'research':
-                    result = await handleResearch(DB_PATH, LORE_DATA_DIR, args, {
+                    result = startResearchJob(DB_PATH, LORE_DATA_DIR, args, {
                         hookContext: { mode: 'mcp' },
+                        onProgress,
                     });
                     break;
+                // Poll for research results (long-polls up to 20s)
+                case 'research_status':
+                    result = await getResearchJobStatus(args?.job_id);
+                    break;
                 // Sync tool
                 case 'sync':
                     result = await handleSync(DB_PATH, LORE_DATA_DIR, args, {
                         hookContext: { mode: 'mcp' },
+                        onProgress,
                     });
                     break;
                 // Project management

package/dist/mcp/tools.js

@@ -86,18 +86,6 @@ const ListSourcesSchema = z.object({
         .describe('Filter by source type (matches the source_type passed during ingest, e.g. "meeting", "slack", "github-issue")'),
     limit: z.number().optional().describe('Max results (default 20)'),
 });
-const RetainSchema = z.object({
-    content: z.string().describe('The insight, decision, or note to retain'),
-    project: z.string().describe('Project this belongs to'),
-    type: z
-        .enum(['insight', 'decision', 'requirement', 'note'])
-        .describe('Type of knowledge being retained'),
-    source_context: z
-        .string()
-        .optional()
-        .describe('Where this came from (e.g., "user interview with Sarah")'),
-    tags: z.array(z.string()).optional().describe('Optional tags for categorization'),
-});
 // ============================================================================
 // Agentic Research Tool
 // ============================================================================
@@ -116,7 +104,7 @@ const ResearchSchema = z.object({
 // ============================================================================
 const IngestSchema = z.object({
     content: z.string().describe('The document content to ingest'),
-    title: z.string().describe('Title for the document'),
+    title: z.string().optional().describe('Title for the document. Auto-generated from content if not provided.'),
     project: z.string().describe('Project this document belongs to'),
     source_type: z
         .string()
@@ -224,24 +212,12 @@ Use this to browse what exists in a project, understand the scope of available k
             properties: {},
         },
     },
-    {
-        name: 'retain',
-        description: `Save a discrete insight, decision, requirement, or note to the knowledge base. These are short, synthesized pieces of knowledge — NOT full documents.
-
-Examples of what to retain:
-- A decision: "We chose JWT over session cookies because of mobile app requirements"
-- An insight: "3 out of 5 users mentioned export speed as their top frustration"
-- A requirement: "Must support SSO for enterprise customers"
-
-USE 'ingest' INSTEAD for full documents, meeting notes, transcripts, or any content longer than a few paragraphs.`,
-        inputSchema: zodToJsonSchema(RetainSchema),
-    },
     // Agentic tool
     {
         name: 'research',
         description: `Run a comprehensive research query across the knowledge base. An internal agent iteratively searches, reads sources, cross-references findings, and synthesizes a research package with full citations.
 
-Returns: summary, key findings, supporting quotes with citations, conflicts detected between sources, and suggested follow-up queries.
+ASYNC: This tool returns immediately with a job_id. You MUST then poll 'research_status' with that job_id to get results. Research typically takes 2-8 minutes depending on the amount of data. Poll every 15-20 seconds. Do NOT assume it is stuck — check the 'activity' array in the status response to see what the agent is doing.
 
 WHEN TO USE:
 - Questions that span multiple sources ("What do we know about authentication?")
@@ -249,9 +225,23 @@ WHEN TO USE:
 - Building a cited research package for decision-making
 - Open-ended exploration of a topic
 
-COST: This tool makes multiple LLM calls internally (typically 3-8 search + read cycles). For simple lookups, use 'search' instead — it's 10x cheaper and faster.`,
+COST: This tool makes multiple LLM calls internally (typically 10-30 search + read cycles). For simple lookups, use 'search' instead — it's 10x cheaper and faster.`,
         inputSchema: zodToJsonSchema(ResearchSchema),
     },
+    // Research status (polling for async results)
+    {
+        name: 'research_status',
+        description: `Check the status of a running research job. Returns the full research package when complete.
+
+Call this after 'research' returns a job_id. Research typically takes 2-8 minutes. Poll every 15-20 seconds. The response includes an 'activity' array showing exactly what the research agent is doing (searches, sources being read, reasoning). As long as 'total_steps' is increasing or 'elapsed_seconds' is under 8 minutes, the research is progressing normally — do NOT abandon it.`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                job_id: { type: 'string', description: 'The job_id returned by the research tool' },
+            },
+            required: ['job_id'],
+        },
+    },
     // Ingest tool
     {
         name: 'ingest',
@@ -262,7 +252,7 @@ IDEMPOTENT: Content is deduplicated by SHA256 hash. Calling ingest with identica
 WHAT HAPPENS:
 1. Content hash checked for deduplication
 2. Document saved to disk
-3. LLM extracts summary, themes, and key quotes
+3. LLM extracts summary, themes, and key quotes (skipped for short content ≤500 chars)
 4. Embedding generated for semantic search
 5. Indexed in Supabase for instant retrieval
 
@@ -270,7 +260,7 @@ BEST PRACTICES:
 - Always pass source_url when available (enables citation linking back to the original)
 - Use source_name for human-readable origin context (e.g., "Slack #product-team")
 - source_type is a free-form hint — use whatever describes the content (slack, email, notion, github-issue, etc.)
-- Use 'retain' instead for short discrete insights/decisions (not full documents)`,
+- For short insights, decisions, or notes — just pass the content. Title and source_type are optional.`,
         inputSchema: zodToJsonSchema(IngestSchema),
     },
     // Sync tool

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getlore/cli",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Research knowledge repository with semantic search, citations, and project lineage tracking",
   "type": "module",
   "main": "dist/index.js",

package/plugins/claude-code/skills/lore/SKILL.md

@@ -36,8 +36,7 @@ After setup, Lore works autonomously.
 | `get_source` | Low | Full document retrieval by ID |
 | `list_sources` | Low | Browse what exists in a project |
 | `list_projects` | Low | Discover available knowledge domains |
-| `retain` | Low | Save discrete insights/decisions |
-| `ingest` | Medium | Push full documents into the knowledge base |
+| `ingest` | Low-Medium | Push content — documents, insights, or decisions |
 | `research` | High | Cross-reference multiple sources, synthesize findings |
 | `sync` | Variable | Refresh from configured source directories |
 
@@ -50,6 +49,11 @@ Use `ingest` to push content into Lore when:
 
 Always pass `source_url` (original URL for linking) and `source_name` (human-readable label like "GitHub PR #123") when available. Ingestion is idempotent — safe to call repeatedly with the same content.
 
+For short insights, decisions, or notes — title and source_type are optional:
+```
+ingest(content: "We chose JWT for auth", project: "auth-system")
+```
+
 ## When to Search
 
 Before making recommendations or answering questions about past work:
@@ -57,13 +61,6 @@ Before making recommendations or answering questions about past work:
 2. Only use `research` if the question genuinely needs cross-referencing multiple sources
 3. Use `get_source(id, include_content: true)` when you need the full text
 
-## When to Retain
-
-Use `retain` for short synthesized knowledge (not full documents):
-- Decisions made during a session
-- Key insights distilled from analysis
-- Requirements extracted from conversations
-
 ## Example: Grounding a Decision
 
 ```
@@ -73,11 +70,9 @@ search("database migration approach", project: "backend-rewrite")
 # 2. If results are relevant, get full context
 get_source("abc-123", include_content: true)
 
-# 3. After making a decision, retain it
-retain(
+# 3. After making a decision, save it
+ingest(
   content: "Chose pgvector over Pinecone for embeddings — lower latency, simpler ops, sufficient scale",
-  project: "backend-rewrite",
-  type: "decision",
-  source_context: "Architecture review session"
+  project: "backend-rewrite"
 )
 ```

package/plugins/codex/SKILL.md

@@ -35,8 +35,7 @@ After setup, Lore works autonomously.
 | `get_source` | Low | Full document retrieval by ID |
 | `list_sources` | Low | Browse what exists in a project |
 | `list_projects` | Low | Discover available knowledge domains |
-| `retain` | Low | Save discrete insights/decisions |
-| `ingest` | Medium | Push full documents into the knowledge base |
+| `ingest` | Low-Medium | Push content — documents, insights, or decisions |
 | `research` | High | Cross-reference multiple sources, synthesize findings |
 | `sync` | Variable | Refresh from configured source directories |
 
@@ -51,6 +50,4 @@ Before making recommendations or answering questions about past work:
 2. Only use `research` for multi-source synthesis (10x more expensive)
 3. Use `get_source(id, include_content: true)` for full text
 
-## When to Retain
-
-Use `retain` for short synthesized knowledge (decisions, insights, requirements) — not full documents.
+For short insights or decisions, just pass the content — title and source_type are optional and auto-generated from content.

package/plugins/gemini/GEMINI.md

@@ -30,8 +30,7 @@ After setup, Lore works autonomously.
 | `get_source` | Low | Full document retrieval by ID |
 | `list_sources` | Low | Browse what exists in a project |
 | `list_projects` | Low | Discover available knowledge domains |
-| `retain` | Low | Save discrete insights/decisions |
-| `ingest` | Medium | Push full documents into the knowledge base |
+| `ingest` | Low-Medium | Push content — documents, insights, or decisions |
 | `research` | High | Cross-reference multiple sources, synthesize findings |
 | `sync` | Variable | Refresh from configured source directories |
 
@@ -46,6 +45,4 @@ Before making recommendations or answering questions about past work:
 2. Only use `research` for multi-source synthesis (10x more expensive)
 3. Use `get_source(id, include_content: true)` for full text
 
-## When to Retain
-
-Use `retain` for short synthesized knowledge (decisions, insights, requirements) — not full documents.
+For short insights or decisions, just pass the content — title and source_type are optional and auto-generated from content.

package/skills/generic-agent.md

@@ -26,7 +26,6 @@ After setup, Lore works autonomously.
 
 - **Sources**: Full documents (meeting notes, interviews, Slack threads, specs, etc.)
 - **Projects**: Organizational grouping for sources
-- **Insights**: Short retained knowledge (decisions, requirements, observations)
 - **Citations**: Every piece of knowledge links back to its original source
 
 ## Tools Reference
@@ -47,10 +46,19 @@ The primary way to add content. Accepts any document with metadata.
 }
 ```
 
+For short insights, decisions, or notes — title and source_type are optional:
+```json
+{
+  "content": "We chose JWT over session cookies because of mobile app requirements",
+  "project": "auth-system"
+}
+```
+
 - **Idempotent**: Duplicate content returns `{deduplicated: true}` with no processing cost.
 - **source_type**: Free-form string. Common values: `meeting`, `interview`, `document`, `notes`, `analysis`, `conversation`, `slack`, `email`, `github-issue`, `notion`.
 - **source_url**: Always pass when available — enables citation linking.
 - **source_name**: Human-readable origin label.
+- Short content (≤500 chars) skips LLM extraction for speed.
 
 ### `search` — Find relevant sources
 Fast lookup. Returns summaries with relevance scores.
@@ -79,18 +87,6 @@ List sources filtered by project or type. Sorted by date (newest first).
 ### `list_projects` — Discover projects
 Lists all projects with source counts and activity dates.
 
-### `retain` — Save discrete knowledge
-For short insights, decisions, or requirements — not full documents.
-
-```json
-{
-  "content": "Users consistently report export takes >30s for large datasets",
-  "project": "my-project",
-  "type": "insight",
-  "source_context": "User interview synthesis — Jan batch"
-}
-```
-
 ### `research` — Deep research with citations
 Runs an internal agent that iteratively searches, reads, and synthesizes findings.
 
@@ -101,7 +97,7 @@ Runs an internal agent that iteratively searches, reads, and synthesizes finding
 }
 ```
 
-**Cost warning**: Makes 3-8 internal LLM calls. Use `search` for simple lookups.
+**Async**: Returns a `job_id` immediately. Poll `research_status` for results (typically 2-8 minutes). Makes 10-30 internal LLM calls. Use `search` for simple lookups.
 
 ### `sync` — Refresh from source directories
 Scans configured directories for new files. Use `ingest` for agent-pushed content instead.
@@ -114,6 +110,6 @@ Excludes from default search. Only use when explicitly requested.
 1. **Search before you answer**: If a question might have documented context, search Lore first.
 2. **Ingest what matters**: After meaningful conversations or when processing external content, ingest it.
 3. **Always pass source_url**: Enables citation linking back to the original.
-4. **Use retain for synthesis**: After analyzing multiple sources, retain the key insight.
+4. **Ingest handles both long and short content**: For short insights, decisions, or notes — just pass the content. Title and source_type are optional.
 5. **Prefer search over research**: `search` is 10x cheaper. Only use `research` for multi-source synthesis.
 6. **Cite your sources**: When presenting Lore results, reference the source title and date.

package/skills/openclaw.md

@@ -37,12 +37,12 @@ Before answering questions about past decisions, user feedback, project history,
 
 3. **Use `get_source`** with `include_content=true` when you need the full original text of a specific document.
 
-## When to Retain Insights
+## Short Content
 
-Use `retain` (not `ingest`) for short, discrete pieces of knowledge:
-- Key decisions: "We chose X because Y"
-- Synthesized insights: "3/5 users mentioned Z as their top issue"
-- Requirements: "Must support SSO for enterprise"
+For short insights, decisions, or notes — title and source_type are optional:
+```
+ingest(content: "We chose X because Y", project: "my-project")
+```
 
 ## Citation Best Practices