@getlore/cli 0.6.0 → 0.7.0

package/dist/mcp/handlers/research-agent.d.ts

@@ -8,6 +8,7 @@
  * 4. Synthesizes findings into a comprehensive research package
  */
 import type { ResearchPackage } from '../../core/types.js';
+import type { ProgressCallback } from './research.js';
 interface ResearchAgentArgs {
     task: string;
     project?: string;
@@ -17,5 +18,5 @@ interface ResearchAgentArgs {
 /**
  * Run the agentic research
  */
-export declare function runResearchAgent(dbPath: string, dataDir: string, args: ResearchAgentArgs): Promise<ResearchPackage>;
+export declare function runResearchAgent(dbPath: string, dataDir: string, args: ResearchAgentArgs, onProgress?: ProgressCallback): Promise<ResearchPackage>;
 export {};

package/dist/mcp/handlers/research-agent.js

@@ -232,7 +232,7 @@ Now begin your research. Use the tools iteratively until you have comprehensive
 /**
  * Run the agentic research
  */
-export async function runResearchAgent(dbPath, dataDir, args) {
+export async function runResearchAgent(dbPath, dataDir, args, onProgress) {
     const { task, project, include_sources = true } = args;
     // Load archived projects to filter (extract just the project names)
     const archivedProjectsData = await loadArchivedProjects(dataDir);
@@ -245,6 +245,8 @@ export async function runResearchAgent(dbPath, dataDir, args) {
     let lastAssistantMessage = '';
     try {
         // Run the agent
+        let turnCount = 0;
+        await onProgress?.(5, undefined, 'Starting research agent...');
         for await (const message of query({
             prompt: `Research task: ${task}${project ? ` (project: ${project})` : ''}`,
             options: {
@@ -261,8 +263,9 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                 permissionMode: 'acceptEdits', // Auto-approve tool calls
             },
         })) {
-            // Capture assistant messages (intermediate)
+            // Capture assistant messages and extract tool call details
             if (message.type === 'assistant') {
+                turnCount++;
                 const msg = message;
                 if (msg.message?.content) {
                     const content = msg.message.content;
@@ -270,9 +273,30 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                         lastAssistantMessage = content;
                     }
                     else if (Array.isArray(content)) {
-                        const textBlocks = content.filter((b) => b.type === 'text');
-                        if (textBlocks.length > 0) {
-                            lastAssistantMessage = textBlocks.map((b) => b.text).join('\n');
+                        // Extract tool_use blocks to report what the agent is doing
+                        for (const block of content) {
+                            if (block.type === 'tool_use') {
+                                const input = block.input;
+                                const toolShort = block.name.replace('mcp__lore-tools__', '');
+                                if (toolShort === 'search' && input.query) {
+                                    await onProgress?.(0, undefined, `Searching: "${input.query}"`);
+                                }
+                                else if (toolShort === 'get_source' && input.source_id) {
+                                    await onProgress?.(0, undefined, `Reading source: ${input.source_id}`);
+                                }
+                                else if (toolShort === 'list_sources') {
+                                    const filter = input.project ? ` (project: ${input.project})` : '';
+                                    await onProgress?.(0, undefined, `Listing sources${filter}`);
+                                }
+                            }
+                            else if (block.type === 'text' && block.text) {
+                                lastAssistantMessage = block.text;
+                                // Send a brief snippet of agent reasoning
+                                const snippet = block.text.substring(0, 120).replace(/\n/g, ' ');
+                                if (snippet.length > 10) {
+                                    await onProgress?.(0, undefined, `Agent thinking: ${snippet}...`);
+                                }
+                            }
                         }
                     }
                 }
@@ -282,16 +306,22 @@ export async function runResearchAgent(dbPath, dataDir, args) {
                 const msg = message;
                 if (msg.subtype === 'success' && msg.result) {
                     lastAssistantMessage = msg.result;
+                    await onProgress?.(0, undefined, `Research complete (${msg.num_turns} turns)`);
                     console.error(`[research-agent] Completed in ${msg.num_turns} turns`);
                 }
                 else if (msg.subtype?.startsWith('error')) {
                     console.error(`[research-agent] Error: ${msg.subtype}`, msg.errors);
                 }
             }
-            // Log tool usage for debugging
+            // Log tool results via the summary message
             if (message.type === 'tool_use_summary') {
                 const msg = message;
-                console.error(`[research-agent] Tool: ${msg.tool_name || 'unknown'}`);
+                if (msg.summary) {
+                    // The summary often contains "Found X results" or similar
+                    const summarySnippet = msg.summary.substring(0, 150).replace(/\n/g, ' ');
+                    await onProgress?.(0, undefined, `Result: ${summarySnippet}`);
+                }
+                console.error(`[research-agent] Tool complete (turn ${turnCount})`);
             }
         }
         // Parse the final result from the agent's output

package/dist/mcp/handlers/research.d.ts

@@ -6,17 +6,36 @@
  * 2. SIMPLE (fallback): Single-pass search + GPT-4o-mini synthesis
  *
  * Set LORE_RESEARCH_MODE=simple to use the fallback mode.
+ *
+ * MCP integration: Research runs asynchronously. The `research` tool returns
+ * immediately with a job_id. Use `research_status` to poll for results.
  */
 import type { ResearchPackage } from '../../core/types.js';
+/**
+ * Start research asynchronously and return a job ID immediately.
+ */
+export declare function startResearchJob(dbPath: string, dataDir: string, args: ResearchArgs, options?: {
+    hookContext?: {
+        mode: 'mcp' | 'cli';
+    };
+    onProgress?: ProgressCallback;
+}): {
+    job_id: string;
+    status: string;
+    message: string;
+};
+export declare function getResearchJobStatus(jobId: string): Promise<Record<string, unknown>>;
 interface ResearchArgs {
     task: string;
     project?: string;
     content_type?: string;
     include_sources?: boolean;
 }
+export type ProgressCallback = (progress: number, total?: number, message?: string) => Promise<void>;
 export declare function handleResearch(dbPath: string, dataDir: string, args: ResearchArgs, options?: {
     hookContext?: {
         mode: 'mcp' | 'cli';
     };
+    onProgress?: ProgressCallback;
 }): Promise<ResearchPackage>;
 export {};

package/dist/mcp/handlers/research.js

@@ -6,13 +6,145 @@
  * 2. SIMPLE (fallback): Single-pass search + GPT-4o-mini synthesis
  *
  * Set LORE_RESEARCH_MODE=simple to use the fallback mode.
+ *
+ * MCP integration: Research runs asynchronously. The `research` tool returns
+ * immediately with a job_id. Use `research_status` to poll for results.
  */
 import OpenAI from 'openai';
+import { randomUUID } from 'crypto';
 import { searchSources } from '../../core/vector-store.js';
 import { generateEmbedding } from '../../core/embedder.js';
 import { loadArchivedProjects } from './archive-project.js';
 import { runResearchAgent } from './research-agent.js';
 import { getExtensionRegistry } from '../../extensions/registry.js';
+const jobStore = new Map();
+// Clean up old jobs after 10 minutes
+const JOB_TTL_MS = 10 * 60 * 1000;
+function cleanOldJobs() {
+    const now = Date.now();
+    for (const [id, job] of jobStore) {
+        const startTime = new Date(job.startedAt).getTime();
+        if (now - startTime > JOB_TTL_MS) {
+            jobStore.delete(id);
+        }
+    }
+}
+/**
+ * Start research asynchronously and return a job ID immediately.
+ */
+export function startResearchJob(dbPath, dataDir, args, options = {}) {
+    cleanOldJobs();
+    const jobId = randomUUID();
+    const now = new Date().toISOString();
+    const job = {
+        id: jobId,
+        task: args.task,
+        project: args.project,
+        status: 'running',
+        startedAt: now,
+        lastActivityAt: now,
+        activity: ['Starting research...'],
+    };
+    jobStore.set(jobId, job);
+    // Fire and forget — runs in the background
+    handleResearch(dbPath, dataDir, args, {
+        ...options,
+        onProgress: async (_p, _t, message) => {
+            const j = jobStore.get(jobId);
+            if (j && message) {
+                j.activity.push(message);
+                j.lastActivityAt = new Date().toISOString();
+            }
+        },
+    })
+        .then((result) => {
+        const j = jobStore.get(jobId);
+        if (j) {
+            j.status = 'complete';
+            j.completedAt = new Date().toISOString();
+            j.result = result;
+            j.activity.push('Research complete');
+        }
+    })
+        .catch((err) => {
+        const j = jobStore.get(jobId);
+        if (j) {
+            j.status = 'error';
+            j.completedAt = new Date().toISOString();
+            j.error = err instanceof Error ? err.message : String(err);
+            j.activity.push(`Failed: ${j.error}`);
+        }
+    })
+        .catch((err) => {
+        // Final safety net for errors in the handlers above
+        console.error(`[research] Critical error in job ${jobId}:`, err);
+    });
+    return {
+        job_id: jobId,
+        status: 'running',
+        message: `Research started for: "${args.task}". Poll research_status with job_id "${jobId}" every 15-20 seconds. This typically takes 2-8 minutes — do not abandon early.`,
+    };
+}
+/**
+ * Check status of a research job.
+ * Long-polls for up to POLL_WAIT_MS, returning early if the job completes.
+ */
+const POLL_WAIT_MS = 20_000;
+const POLL_INTERVAL_MS = 1_000;
+export async function getResearchJobStatus(jobId) {
+    let job = jobStore.get(jobId);
+    if (!job) {
+        return { status: 'not_found', job_id: jobId };
+    }
+    // If already done, return immediately
+    if (job.status !== 'running') {
+        return formatJobResponse(job);
+    }
+    // Long-poll: wait up to POLL_WAIT_MS for completion, checking every second
+    const deadline = Date.now() + POLL_WAIT_MS;
+    while (Date.now() < deadline) {
+        await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+        // Re-fetch to avoid stale reference if job was cleaned up
+        job = jobStore.get(jobId);
+        if (!job) {
+            return { status: 'not_found', job_id: jobId };
+        }
+        if (job.status !== 'running') {
+            return formatJobResponse(job);
+        }
+    }
+    return formatJobResponse(job);
+}
+function formatJobResponse(job) {
+    const elapsed = Math.round((Date.now() - new Date(job.startedAt).getTime()) / 1000);
+    if (job.status === 'complete') {
+        return {
+            status: 'complete',
+            job_id: job.id,
+            task: job.task,
+            elapsed_seconds: elapsed,
+            result: job.result,
+        };
+    }
+    if (job.status === 'error') {
+        return {
+            status: 'error',
+            job_id: job.id,
+            task: job.task,
+            elapsed_seconds: elapsed,
+            error: job.error,
+        };
+    }
+    return {
+        status: 'running',
+        job_id: job.id,
+        task: job.task,
+        elapsed_seconds: elapsed,
+        total_steps: job.activity.length,
+        activity: job.activity,
+        message: `Research is still running (${elapsed}s elapsed, ${job.activity.length} steps completed). This is normal — deep research takes 2-8 minutes. Keep polling.`,
+    };
+}
 // Lazy initialization for OpenAI (only used in simple mode)
 let openaiClient = null;
 function getOpenAI() {
@@ -109,12 +241,15 @@ Respond with only the JSON object.`;
 }
 export async function handleResearch(dbPath, dataDir, args, options = {}) {
     const { task, project, include_sources = true } = args;
+    const { onProgress } = options;
     // Check if we should use agentic mode (default) or simple mode (fallback)
     const useAgenticMode = process.env.LORE_RESEARCH_MODE !== 'simple';
     if (useAgenticMode) {
         console.error('[research] Using agentic mode (Claude Agent SDK)');
+        await onProgress?.(0, undefined, 'Starting agentic research...');
         try {
-            const result = await runResearchAgent(dbPath, dataDir, args);
+            const result = await runResearchAgent(dbPath, dataDir, args, onProgress);
+            await onProgress?.(100, 100, 'Research complete');
             await runResearchCompletedHook(result, {
                 mode: options.hookContext?.mode || 'mcp',
                 dataDir,
@@ -124,11 +259,14 @@ export async function handleResearch(dbPath, dataDir, args, options = {}) {
         }
         catch (error) {
             console.error('[research] Agentic mode failed, falling back to simple mode:', error);
+            await onProgress?.(0, undefined, 'Agentic mode failed, falling back to simple mode...');
             // Fall through to simple mode
         }
     }
     console.error('[research] Using simple mode (single-pass synthesis)');
-    const result = await handleResearchSimple(dbPath, dataDir, args);
+    await onProgress?.(0, undefined, 'Starting simple research...');
+    const result = await handleResearchSimple(dbPath, dataDir, args, onProgress);
+    await onProgress?.(100, 100, 'Research complete');
     await runResearchCompletedHook(result, {
         mode: options.hookContext?.mode || 'mcp',
         dataDir,
@@ -140,7 +278,7 @@ export async function handleResearch(dbPath, dataDir, args, options = {}) {
  * Simple research mode - single pass search + synthesis
  * This is the fallback when agentic mode fails or is disabled
  */
-async function handleResearchSimple(dbPath, dataDir, args) {
+async function handleResearchSimple(dbPath, dataDir, args, onProgress) {
     const { task, project, include_sources = true } = args;
     // Use sensible defaults for simple mode
     const sourceLimit = 10;
@@ -149,7 +287,9 @@ async function handleResearchSimple(dbPath, dataDir, args) {
     const archivedProjects = await loadArchivedProjects(dataDir);
     const archivedNames = new Set(archivedProjects.map((p) => p.project.toLowerCase()));
     // Step 1: Search for relevant sources (fetch extra to account for archived filtering)
+    await onProgress?.(10, 100, 'Generating embeddings...');
     const queryVector = await generateEmbedding(task);
+    await onProgress?.(30, 100, 'Searching sources...');
     const rawSources = await searchSources(dbPath, queryVector, {
         limit: sourceLimit * 2,
         project,
@@ -172,6 +312,7 @@ async function handleResearchSimple(dbPath, dataDir, args) {
         }
     }
     // Step 3: Synthesize findings with LLM (conflict-aware)
+    await onProgress?.(60, 100, 'Synthesizing findings...');
     // Note: Decisions are now extracted at query time by the agentic research mode
     const synthesis = await synthesizeFindings(task, sources.map((s) => ({
         id: s.id,

package/dist/mcp/handlers/sync.d.ts

@@ -44,5 +44,6 @@ export declare function handleSync(dbPath: string, dataDir: string, args: SyncAr
     hookContext?: {
         mode: 'mcp' | 'cli';
     };
+    onProgress?: (progress: number, total?: number, message?: string) => Promise<void>;
 }): Promise<SyncResult>;
 export {};

package/dist/mcp/handlers/sync.js

@@ -233,8 +233,10 @@ export async function handleSync(dbPath, dataDir, args, options = {}) {
         already_indexed: 0,
         reconciled: 0,
     };
+    const { onProgress } = options;
     // 1. Git pull
     if (doPull) {
+        await onProgress?.(5, undefined, 'Pulling from git...');
         const pullResult = await gitPull(dataDir);
         result.git_pulled = pullResult.success && (pullResult.message?.includes('Pulled') || false);
         if (pullResult.error) {
@@ -248,17 +250,20 @@ export async function handleSync(dbPath, dataDir, args, options = {}) {
         const hasUniversalSources = getEnabledSources(config).length > 0;
         if (hasUniversalSources && !useLegacy) {
             // Use new universal sync
+            await onProgress?.(20, undefined, 'Discovering new files...');
             const { discovery, processing } = await universalSync(dataDir, dryRun, options.hookContext);
             result.discovery = discovery;
             result.processing = processing;
         }
         // Always run legacy disk sync for backward compatibility
         // (picks up sources added via old `lore ingest` command)
+        await onProgress?.(60, undefined, 'Running legacy sync...');
         const legacyResult = await legacyDiskSync(dbPath, dataDir);
         result.sources_found = legacyResult.sources_found;
         result.sources_indexed = legacyResult.sources_indexed;
         result.already_indexed = legacyResult.already_indexed;
         // Reconcile: ensure every Supabase source has local content.md
+        await onProgress?.(80, undefined, 'Reconciling local content...');
         result.reconciled = await reconcileLocalContent(dataDir);
     }
     // 3. Git push

package/dist/mcp/server.js

@@ -21,7 +21,7 @@ import { handleGetSource } from './handlers/get-source.js';
 import { handleListSources } from './handlers/list-sources.js';
 import { handleRetain } from './handlers/retain.js';
 import { handleIngest } from './handlers/ingest.js';
-import { handleResearch } from './handlers/research.js';
+import { startResearchJob, getResearchJobStatus } from './handlers/research.js';
 import { handleListProjects } from './handlers/list-projects.js';
 import { handleSync } from './handlers/sync.js';
 import { handleArchiveProject } from './handlers/archive-project.js';
@@ -136,7 +136,7 @@ async function main() {
     }
     const server = new Server({
         name: 'lore',
-        version: '0.1.0',
+        version: '0.7.0',
     }, {
         capabilities: {
             tools: {},
@@ -184,8 +184,25 @@ async function main() {
         return { tools: toolDefinitions };
     });
     // Handle tool calls (core tools only)
-    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
         const { name, arguments: args } = request.params;
+        // Build a progress callback for long-running tools.
+        // If the client sent a progressToken, we send notifications/progress back;
+        // otherwise, onProgress is a no-op.
+        const progressToken = request.params._meta?.progressToken;
+        const onProgress = progressToken
+            ? async (progress, total, message) => {
+                try {
+                    await extra.sendNotification({
+                        method: 'notifications/progress',
+                        params: { progressToken, progress, ...(total != null ? { total } : {}), ...(message ? { message } : {}) },
+                    });
+                }
+                catch {
+                    // Progress notifications are best-effort
+                }
+            }
+            : undefined;
         try {
             let result;
             switch (name) {
@@ -215,16 +232,22 @@ async function main() {
                         hookContext: { mode: 'mcp' },
                     });
                     break;
-                // Agentic research tool (uses Claude Agent SDK internally)
+                // Agentic research tool — runs async, returns job_id immediately
                 case 'research':
-                    result = await handleResearch(DB_PATH, LORE_DATA_DIR, args, {
+                    result = startResearchJob(DB_PATH, LORE_DATA_DIR, args, {
                         hookContext: { mode: 'mcp' },
+                        onProgress,
                     });
                     break;
+                // Poll for research results (long-polls up to 20s)
+                case 'research_status':
+                    result = await getResearchJobStatus(args?.job_id);
+                    break;
                 // Sync tool
                 case 'sync':
                     result = await handleSync(DB_PATH, LORE_DATA_DIR, args, {
                         hookContext: { mode: 'mcp' },
+                        onProgress,
                     });
                     break;
                 // Project management

package/dist/mcp/tools.js

@@ -241,7 +241,7 @@ USE 'ingest' INSTEAD for full documents, meeting notes, transcripts, or any cont
         name: 'research',
         description: `Run a comprehensive research query across the knowledge base. An internal agent iteratively searches, reads sources, cross-references findings, and synthesizes a research package with full citations.
 
-Returns: summary, key findings, supporting quotes with citations, conflicts detected between sources, and suggested follow-up queries.
+ASYNC: This tool returns immediately with a job_id. You MUST then poll 'research_status' with that job_id to get results. Research typically takes 2-8 minutes depending on the amount of data. Poll every 15-20 seconds. Do NOT assume it is stuck — check the 'activity' array in the status response to see what the agent is doing.
 
 WHEN TO USE:
 - Questions that span multiple sources ("What do we know about authentication?")
@@ -249,9 +249,23 @@ WHEN TO USE:
 - Building a cited research package for decision-making
 - Open-ended exploration of a topic
 
-COST: This tool makes multiple LLM calls internally (typically 3-8 search + read cycles). For simple lookups, use 'search' instead — it's 10x cheaper and faster.`,
+COST: This tool makes multiple LLM calls internally (typically 10-30 search + read cycles). For simple lookups, use 'search' instead — it's 10x cheaper and faster.`,
         inputSchema: zodToJsonSchema(ResearchSchema),
     },
+    // Research status (polling for async results)
+    {
+        name: 'research_status',
+        description: `Check the status of a running research job. Returns the full research package when complete.
+
+Call this after 'research' returns a job_id. Research typically takes 2-8 minutes. Poll every 15-20 seconds. The response includes an 'activity' array showing exactly what the research agent is doing (searches, sources being read, reasoning). As long as 'total_steps' is increasing or 'elapsed_seconds' is under 8 minutes, the research is progressing normally — do NOT abandon it.`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                job_id: { type: 'string', description: 'The job_id returned by the research tool' },
+            },
+            required: ['job_id'],
+        },
+    },
     // Ingest tool
     {
         name: 'ingest',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getlore/cli",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Research knowledge repository with semantic search, citations, and project lineage tracking",
   "type": "module",
   "main": "dist/index.js",