@getlore/cli 0.2.0

package/dist/sync/process.js

@@ -0,0 +1,285 @@
+/**
+ * Lore - Claude Processing (Phase 2)
+ *
+ * Uses Claude to extract metadata from new files:
+ * - title: Descriptive title
+ * - summary: 2-4 sentence summary with key takeaways
+ * - date: ISO date if present
+ * - participants: List of names if present
+ * - content_type: interview|meeting|conversation|document|note|analysis
+ *
+ * Only called for NEW files (not already in Supabase).
+ */
+import { mkdir, writeFile, copyFile } from 'fs/promises';
+import path from 'path';
+import Anthropic from '@anthropic-ai/sdk';
+import { processFile } from './processors.js';
+import { generateEmbedding, createSearchableText } from '../core/embedder.js';
+import { addSource } from '../core/vector-store.js';
+import { gitCommitAndPush } from '../core/git.js';
+import { getExtensionRegistry } from '../extensions/registry.js';
+// ============================================================================
+// Claude Client
+// ============================================================================
+let anthropic = null;
+function getAnthropic() {
+    if (!anthropic) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            throw new Error('ANTHROPIC_API_KEY environment variable is required');
+        }
+        anthropic = new Anthropic({ apiKey });
+    }
+    return anthropic;
+}
+// ============================================================================
+// Metadata Extraction
+// ============================================================================
+const EXTRACTION_PROMPT = `Analyze this document and extract metadata. Return ONLY valid JSON with these fields:
+
+{
+  "title": "A descriptive title (create one if not obvious)",
+  "summary": "2-4 sentences capturing key takeaways, findings, or purpose",
+  "date": "ISO date string (YYYY-MM-DD) if mentioned, otherwise null",
+  "participants": ["list", "of", "names"] if this is a meeting/interview, otherwise [],
+  "content_type": "one of: interview|meeting|conversation|document|note|analysis"
+}
+
+Content type guidelines:
+- interview: User research, customer interview, 1:1 feedback session
+- meeting: Team meeting, standup, planning session
+- conversation: AI chat (Claude, ChatGPT), chat logs
+- document: Spec, design doc, report, article
+- note: Personal notes, memo, quick thoughts
+- analysis: Competitor analysis, market research, data analysis
+
+Be specific in the summary. Include concrete details, names, numbers when present.`;
+export async function extractMetadata(content, filePath, options = {}) {
+    const { model = 'claude-sonnet-4-20250514', image } = options;
+    const client = getAnthropic();
+    // Build message content based on whether we have an image or text
+    let messageContent;
+    if (image) {
+        // Image analysis with Claude Vision
+        messageContent = [
+            {
+                type: 'image',
+                source: {
+                    type: 'base64',
+                    media_type: image.mediaType,
+                    data: image.base64,
+                },
+            },
+            {
+                type: 'text',
+                text: `${EXTRACTION_PROMPT}\n\nFile: ${path.basename(filePath)}\n\nAnalyze this image and extract metadata. Describe what's in the image in detail in the summary.`,
+            },
+        ];
+    }
+    else {
+        // Text-based analysis
+        const maxChars = 50000;
+        const truncatedContent = content.length > maxChars
+            ? content.substring(0, maxChars) + '\n\n[Content truncated...]'
+            : content;
+        messageContent = `${EXTRACTION_PROMPT}\n\nFile: ${path.basename(filePath)}\n\n---\n\n${truncatedContent}`;
+    }
+    const response = await client.messages.create({
+        model,
+        max_tokens: 1000,
+        messages: [
+            {
+                role: 'user',
+                content: messageContent,
+            },
+        ],
+    });
+    // Extract text from response
+    const responseText = response.content
+        .filter((block) => block.type === 'text')
+        .map(block => block.text)
+        .join('');
+    // Parse JSON from response
+    try {
+        // Find JSON in response (handle markdown code blocks)
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            throw new Error('No JSON found in response');
+        }
+        const parsed = JSON.parse(jsonMatch[0]);
+        // Validate and normalize
+        return {
+            title: parsed.title || path.basename(filePath),
+            summary: parsed.summary || 'No summary available',
+            date: parsed.date || null,
+            participants: Array.isArray(parsed.participants) ? parsed.participants : [],
+            content_type: validateContentType(parsed.content_type),
+        };
+    }
+    catch (error) {
+        console.error('[process] Error parsing Claude response:', error);
+        console.error('[process] Raw response:', responseText);
+        // Return fallback metadata
+        return {
+            title: path.basename(filePath),
+            summary: content.substring(0, 200) + '...',
+            date: null,
+            participants: [],
+            content_type: 'document',
+        };
+    }
+}
+function validateContentType(type) {
+    const validTypes = [
+        'interview', 'meeting', 'conversation', 'document', 'note', 'analysis',
+    ];
+    if (validTypes.includes(type)) {
+        return type;
+    }
+    return 'document';
+}
+// ============================================================================
+// Source Storage
+// ============================================================================
+function generateSourceId() {
+    // Generate a UUID for compatibility with Supabase schema
+    return crypto.randomUUID();
+}
+async function storeSourceToDisk(sourceId, file, metadata, processedContent, dataDir) {
+    const sourcesDir = path.join(dataDir, 'sources');
+    const sourceDir = path.join(sourcesDir, sourceId);
+    // Create source directory
+    await mkdir(sourceDir, { recursive: true });
+    // Copy original file
+    const originalExt = path.extname(file.absolutePath);
+    await copyFile(file.absolutePath, path.join(sourceDir, `original${originalExt}`));
+    // Save processed content
+    await writeFile(path.join(sourceDir, 'content.md'), processedContent);
+    // Save metadata
+    const sourceMetadata = {
+        id: sourceId,
+        title: metadata.title,
+        source_type: 'document', // Universal type for sync-ingested sources
+        content_type: metadata.content_type,
+        created_at: metadata.date || new Date().toISOString(),
+        imported_at: new Date().toISOString(),
+        projects: [file.project],
+        tags: [],
+        source_path: file.absolutePath,
+        content_hash: file.contentHash,
+        sync_source: file.sourceName,
+        original_file: file.relativePath,
+    };
+    await writeFile(path.join(sourceDir, 'metadata.json'), JSON.stringify(sourceMetadata, null, 2));
+    // Save insights (summary + themes placeholder)
+    await writeFile(path.join(sourceDir, 'insights.json'), JSON.stringify({ summary: metadata.summary, themes: [], quotes: [] }, null, 2));
+}
+async function indexSource(sourceId, file, metadata, dbPath) {
+    // Generate embedding
+    const searchableText = createSearchableText({
+        type: 'summary',
+        text: metadata.summary,
+        project: file.project,
+    });
+    const vector = await generateEmbedding(searchableText);
+    // Create source record
+    const sourceRecord = {
+        id: sourceId,
+        title: metadata.title,
+        source_type: 'document',
+        content_type: metadata.content_type,
+        projects: JSON.stringify([file.project]),
+        tags: JSON.stringify([]),
+        created_at: metadata.date || new Date().toISOString(),
+        summary: metadata.summary,
+        themes_json: JSON.stringify([]),
+        quotes_json: JSON.stringify([]),
+        has_full_content: true,
+        vector: [],
+    };
+    // Add to vector store with content_hash and source_path
+    await addSource(dbPath, sourceRecord, vector, {
+        content_hash: file.contentHash,
+        source_path: file.absolutePath,
+    });
+}
+// ============================================================================
+// Main Processing Function
+// ============================================================================
+export async function processFiles(files, dataDir, options = {}) {
+    const { onProgress, model = 'claude-sonnet-4-20250514', concurrency = 2, gitPush = true, hookContext, } = options;
+    const dbPath = path.join(dataDir, 'lore.lance');
+    const result = {
+        processed: [],
+        errors: [],
+    };
+    const extensionRegistry = hookContext
+        ? await getExtensionRegistry({ logger: (message) => console.error(message) })
+        : null;
+    // Process files with controlled concurrency
+    for (let i = 0; i < files.length; i += concurrency) {
+        const batch = files.slice(i, i + concurrency);
+        const batchResults = await Promise.allSettled(batch.map(async (file) => {
+            // 1. Read and preprocess file
+            const processed = await processFile(file.absolutePath);
+            // 2. Extract metadata with Claude (handles both text and images)
+            const metadata = await extractMetadata(processed.text, file.absolutePath, { model, image: processed.image });
+            // For images, use the summary as the text content
+            const contentText = processed.image
+                ? `# ${metadata.title}\n\n${metadata.summary}`
+                : processed.text;
+            // 3. Use existing ID for edits, generate new ID for new files
+            const sourceId = file.existingId || generateSourceId();
+            // 4. Index in Supabase FIRST (may fail on duplicate content_hash)
+            await indexSource(sourceId, file, metadata, dbPath);
+            // 5. Store source to disk ONLY if Supabase succeeded
+            await storeSourceToDisk(sourceId, file, metadata, contentText, dataDir);
+            if (extensionRegistry && hookContext) {
+                await extensionRegistry.runHook('onSourceCreated', {
+                    id: sourceId,
+                    title: metadata.title,
+                    source_type: 'document',
+                    content_type: metadata.content_type,
+                    created_at: metadata.date || new Date().toISOString(),
+                    imported_at: new Date().toISOString(),
+                    projects: [file.project],
+                    tags: [],
+                    source_path: file.absolutePath,
+                    content_hash: file.contentHash,
+                    sync_source: file.sourceName,
+                    original_file: file.relativePath,
+                }, {
+                    mode: hookContext.mode,
+                    dataDir,
+                    dbPath,
+                });
+            }
+            return { file, metadata, sourceId };
+        }));
+        // Collect results
+        for (let j = 0; j < batchResults.length; j++) {
+            const batchResult = batchResults[j];
+            const file = batch[j];
+            if (batchResult.status === 'fulfilled') {
+                result.processed.push(batchResult.value);
+                onProgress?.(result.processed.length + result.errors.length, files.length, batchResult.value.metadata.title);
+            }
+            else {
+                result.errors.push({
+                    file,
+                    error: batchResult.reason?.message || String(batchResult.reason),
+                });
+                onProgress?.(result.processed.length + result.errors.length, files.length, `Error: ${file.relativePath}`);
+            }
+        }
+        // Small delay between batches to avoid rate limits
+        if (i + concurrency < files.length) {
+            await new Promise(resolve => setTimeout(resolve, 500));
+        }
+    }
+    // Git commit and push if enabled
+    if (gitPush && result.processed.length > 0) {
+        await gitCommitAndPush(dataDir, `Sync: Added ${result.processed.length} source(s) from universal sync`);
+    }
+    return result;
+}

package/dist/sync/processors.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Lore - Format Preprocessors
+ *
+ * Converts various file formats to plain text for Claude analysis.
+ * All processing is IN MEMORY ONLY - original files are never modified.
+ */
+export type ImageMediaType = 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp';
+export interface ProcessedContent {
+    text: string;
+    format: string;
+    metadata?: {
+        title?: string;
+        date?: string;
+        participants?: string[];
+    };
+    image?: {
+        base64: string;
+        mediaType: ImageMediaType;
+    };
+}
+export declare function processFile(filePath: string): Promise<ProcessedContent>;
+export declare function preprocessFiles(filePaths: string[], options?: {
+    onProgress?: (completed: number, total: number) => void;
+}): Promise<Map<string, ProcessedContent>>;

package/dist/sync/processors.js

@@ -0,0 +1,351 @@
+/**
+ * Lore - Format Preprocessors
+ *
+ * Converts various file formats to plain text for Claude analysis.
+ * All processing is IN MEMORY ONLY - original files are never modified.
+ */
+import { readFile } from 'fs/promises';
+import path from 'path';
+let pdfParser = null;
+async function getPdfParser() {
+    if (!pdfParser) {
+        try {
+            const { PDFParse } = await import('pdf-parse');
+            // Wrap the class in a function
+            pdfParser = async (buffer) => {
+                const parser = new PDFParse({ data: buffer });
+                await parser.load();
+                const text = await parser.getText();
+                return { text: text || '' };
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    return pdfParser;
+}
+// ============================================================================
+// Markdown Processing
+// ============================================================================
+function processMarkdown(content) {
+    // Markdown is already text, just return as-is
+    // Extract title from first H1 if present
+    const titleMatch = content.match(/^#\s+(.+)$/m);
+    return {
+        text: content,
+        format: 'markdown',
+        metadata: titleMatch ? { title: titleMatch[1] } : undefined,
+    };
+}
+function extractTextContent(content) {
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .filter(c => c.type === 'text' && c.text)
+            .map(c => c.text)
+            .join('\n');
+    }
+    return '';
+}
+function processJSONL(content) {
+    const lines = content.split('\n').filter(line => line.trim());
+    const messages = [];
+    for (const line of lines) {
+        try {
+            const obj = JSON.parse(line);
+            // Handle various JSONL formats
+            if (obj.message?.content) {
+                const role = obj.message.role || obj.type || 'unknown';
+                const text = extractTextContent(obj.message.content);
+                if (text) {
+                    messages.push(`[${role.toUpperCase()}]: ${text}`);
+                }
+            }
+            else if (obj.content) {
+                const role = obj.role || obj.type || 'unknown';
+                const text = extractTextContent(obj.content);
+                if (text) {
+                    messages.push(`[${role.toUpperCase()}]: ${text}`);
+                }
+            }
+        }
+        catch {
+            // Skip invalid JSON lines
+        }
+    }
+    return {
+        text: messages.join('\n\n'),
+        format: 'jsonl',
+    };
+}
+function proseMirrorToText(node) {
+    if (!node || typeof node !== 'object')
+        return '';
+    const n = node;
+    if (n.type === 'text')
+        return n.text || '';
+    if (Array.isArray(n.content)) {
+        return n.content.map(proseMirrorToText).join('');
+    }
+    return '';
+}
+function processJSON(content, filePath) {
+    try {
+        const data = JSON.parse(content);
+        // Check if it's a Granola document
+        if (data.notes || data.transcript) {
+            const doc = data;
+            const parts = [];
+            // Extract notes
+            if (doc.notes?.content) {
+                const notesText = proseMirrorToText(doc.notes);
+                if (notesText) {
+                    parts.push('## Notes\n' + notesText);
+                }
+            }
+            // Extract transcript
+            if (doc.transcript?.utterances) {
+                const transcriptText = doc.transcript.utterances
+                    .map(u => {
+                    const speaker = u.source === 'microphone' ? '[ME]' : '[PARTICIPANT]';
+                    const mins = Math.floor(u.start / 60);
+                    const secs = Math.floor(u.start % 60);
+                    const timestamp = `[${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}]`;
+                    return `${timestamp} ${speaker}: ${u.text}`;
+                })
+                    .join('\n\n');
+                if (transcriptText) {
+                    parts.push('## Transcript\n' + transcriptText);
+                }
+            }
+            return {
+                text: parts.join('\n\n'),
+                format: 'json-granola',
+                metadata: {
+                    title: doc.title,
+                    date: doc.created_at,
+                },
+            };
+        }
+        // Generic JSON - just stringify nicely
+        return {
+            text: JSON.stringify(data, null, 2),
+            format: 'json',
+        };
+    }
+    catch {
+        return {
+            text: content,
+            format: 'json-invalid',
+        };
+    }
+}
+// ============================================================================
+// Plain Text Processing
+// ============================================================================
+function processPlainText(content) {
+    return {
+        text: content,
+        format: 'text',
+    };
+}
+// ============================================================================
+// PDF Processing
+// ============================================================================
+async function processPdf(filePath) {
+    const parser = await getPdfParser();
+    if (!parser) {
+        return {
+            text: '[PDF processing not available - install pdf-parse]',
+            format: 'pdf-unsupported',
+        };
+    }
+    const buffer = await readFile(filePath);
+    const data = await parser(buffer);
+    return {
+        text: data.text,
+        format: 'pdf',
+        metadata: {
+            title: path.basename(filePath, '.pdf'),
+        },
+    };
+}
+// ============================================================================
+// Image Processing (for Claude Vision)
+// ============================================================================
+function getImageMediaType(ext) {
+    const types = {
+        '.jpg': 'image/jpeg',
+        '.jpeg': 'image/jpeg',
+        '.png': 'image/png',
+        '.gif': 'image/gif',
+        '.webp': 'image/webp',
+    };
+    return types[ext.toLowerCase()] || null;
+}
+async function processImage(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const mediaType = getImageMediaType(ext);
+    if (!mediaType) {
+        return {
+            text: '[Unsupported image format]',
+            format: 'image-unsupported',
+        };
+    }
+    const buffer = await readFile(filePath);
+    const base64 = buffer.toString('base64');
+    return {
+        text: '', // Will be filled by Claude vision
+        format: 'image',
+        image: {
+            base64,
+            mediaType,
+        },
+    };
+}
+// ============================================================================
+// CSV Processing
+// ============================================================================
+function processCsv(content, filePath) {
+    const lines = content.split('\n').filter(line => line.trim());
+    if (lines.length === 0) {
+        return { text: content, format: 'csv' };
+    }
+    // Parse header and data
+    const header = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
+    const rows = lines.slice(1);
+    // Convert to readable format
+    const formatted = rows.map((row, idx) => {
+        const values = row.split(',').map(v => v.trim().replace(/^"|"$/g, ''));
+        const pairs = header.map((h, i) => `${h}: ${values[i] || ''}`);
+        return `Row ${idx + 1}:\n  ${pairs.join('\n  ')}`;
+    }).join('\n\n');
+    return {
+        text: `CSV Data (${rows.length} rows, ${header.length} columns)\n\nColumns: ${header.join(', ')}\n\n${formatted}`,
+        format: 'csv',
+        metadata: {
+            title: path.basename(filePath, '.csv'),
+        },
+    };
+}
+// ============================================================================
+// HTML Processing
+// ============================================================================
+function processHtml(content) {
+    // Simple HTML to text conversion
+    let text = content
+        // Remove scripts and styles
+        .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+        // Convert common elements
+        .replace(/<br\s*\/?>/gi, '\n')
+        .replace(/<\/p>/gi, '\n\n')
+        .replace(/<\/div>/gi, '\n')
+        .replace(/<\/h[1-6]>/gi, '\n\n')
+        .replace(/<li>/gi, '• ')
+        .replace(/<\/li>/gi, '\n')
+        // Remove remaining tags
+        .replace(/<[^>]+>/g, '')
+        // Decode common entities
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        // Clean up whitespace
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    // Extract title from <title> tag
+    const titleMatch = content.match(/<title[^>]*>([^<]+)<\/title>/i);
+    return {
+        text,
+        format: 'html',
+        metadata: titleMatch ? { title: titleMatch[1].trim() } : undefined,
+    };
+}
+// ============================================================================
+// XML Processing
+// ============================================================================
+function processXml(content) {
+    // Extract text content from XML, preserving structure
+    const text = content
+        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1') // Unwrap CDATA
+        .replace(/<[^>]+>/g, ' ') // Remove tags
+        .replace(/\s+/g, ' ') // Normalize whitespace
+        .trim();
+    return {
+        text: `XML Document:\n\n${text}`,
+        format: 'xml',
+    };
+}
+// ============================================================================
+// Main Processing Function
+// ============================================================================
+export async function processFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    // Handle binary formats first (before trying to read as utf-8)
+    if (ext === '.pdf') {
+        return processPdf(filePath);
+    }
+    const imageMediaType = getImageMediaType(ext);
+    if (imageMediaType) {
+        return processImage(filePath);
+    }
+    // Text-based formats
+    const content = await readFile(filePath, 'utf-8');
+    switch (ext) {
+        case '.md':
+        case '.markdown':
+            return processMarkdown(content);
+        case '.jsonl':
+            return processJSONL(content);
+        case '.json':
+            return processJSON(content, filePath);
+        case '.txt':
+            return processPlainText(content);
+        case '.csv':
+            return processCsv(content, filePath);
+        case '.html':
+        case '.htm':
+            return processHtml(content);
+        case '.xml':
+        case '.xhtml':
+            return processXml(content);
+        default:
+            // Try to detect format from content
+            if (content.trim().startsWith('{') || content.trim().startsWith('[')) {
+                return processJSON(content, filePath);
+            }
+            if (content.includes('{"')) {
+                // Might be JSONL
+                return processJSONL(content);
+            }
+            return processPlainText(content);
+    }
+}
+// ============================================================================
+// Batch Processing
+// ============================================================================
+export async function preprocessFiles(filePaths, options = {}) {
+    const { onProgress } = options;
+    const results = new Map();
+    for (let i = 0; i < filePaths.length; i++) {
+        try {
+            const processed = await processFile(filePaths[i]);
+            results.set(filePaths[i], processed);
+        }
+        catch (error) {
+            // Store error as metadata
+            results.set(filePaths[i], {
+                text: '',
+                format: 'error',
+                metadata: { title: `Error: ${error}` },
+            });
+        }
+        onProgress?.(i + 1, filePaths.length);
+    }
+    return results;
+}

package/dist/tui/browse-handlers-ask.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Ask handlers for the Lore Document Browser TUI
+ *
+ * Handles AI-powered queries with streaming responses.
+ * Supports slash commands and multi-turn conversations.
+ *
+ * Slash commands:
+ *   /project <name> or /p <name> - Set project filter
+ *   /type <type> or /t <type>    - Set content type filter
+ *   /clear                       - Clear all filters
+ *   /new                         - Start new conversation
+ *   /help or /?                  - Show available commands
+ */
+import type { BrowserState, UIComponents } from './browse-types.js';
+/**
+ * Enter ask mode - show input for query
+ */
+export declare function enterAskMode(state: BrowserState, ui: UIComponents): void;
+/**
+ * Exit ask mode - return to list
+ */
+export declare function exitAskMode(state: BrowserState, ui: UIComponents): void;
+/**
+ * Prompt for next input after a response
+ */
+export declare function promptForFollowUp(state: BrowserState, ui: UIComponents): void;
+/**
+ * Execute the ask query with streaming
+ */
+export declare function executeAsk(state: BrowserState, ui: UIComponents, dbPath: string, query: string): Promise<void>;