@getlore/cli 0.2.0

package/dist/core/data-repo.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Lore - Data Repository Helpers
+ *
+ * Shared logic for initializing and managing the lore data directory.
+ * Used by both `lore setup` and `lore init`.
+ */
+/**
+ * Initialize a lore data repository at the given path.
+ * Creates directory structure, .gitignore, README, and git init.
+ * Idempotent — safe to call on an existing directory.
+ */
+export declare function initDataRepo(dirPath: string): Promise<void>;
+export declare const WELCOME_DOC_CONTENT = "# Getting Started with Lore\n\nWelcome to Lore \u2014 your research knowledge repository.\n\n## What is Lore?\n\nLore preserves your original sources (meeting notes, interviews, documents) and makes them searchable with full citations. Unlike a memory system, Lore keeps the original context so you can always trace back to the source.\n\n## Quick Start\n\n- **Search**: Run `lore search \"your query\"` to find relevant documents\n- **Sync**: Run `lore sync` to discover and index new files from your sync sources\n- **Browse**: Run `lore browse` to explore your knowledge base in the terminal\n- **Research**: Use the `research` MCP tool for deep, multi-step research with citations\n\n## Adding Documents\n\n1. **Sync sources**: Run `lore sync add` to watch a directory for new files\n2. **Direct ingest**: Use the `ingest` MCP tool to add documents from any agent\n3. **Manual sync**: Run `lore sync` after adding files to your data directory\n\n## Background Daemon\n\nRun `lore sync start` to launch a background daemon that watches for new files and auto-indexes them. Check status with `lore sync status`.\n\n## Tips\n\n- Lore extracts metadata using AI at sync time, so your documents are enriched automatically\n- Use projects to organize related documents together\n- The research tool can cross-reference multiple sources and synthesize findings with citations\n";
+/**
+ * Check if GitHub CLI is available and authenticated.
+ */
+export declare function isGhAvailable(): Promise<boolean>;
+/**
+ * Create a private GitHub repo from the given local directory.
+ * Returns the repo URL on success, null on failure.
+ */
+export declare function createGithubRepo(dirPath: string, name: string): Promise<string | null>;
+/**
+ * Get the git remote origin URL for a directory.
+ * Returns null if no remote is configured.
+ */
+export declare function getGitRemoteUrl(dirPath: string): string | null;
+/**
+ * Check if a directory is a git repository.
+ */
+export declare function isGitRepo(dirPath: string): boolean;

package/dist/core/data-repo.js

@@ -0,0 +1,146 @@
+/**
+ * Lore - Data Repository Helpers
+ *
+ * Shared logic for initializing and managing the lore data directory.
+ * Used by both `lore setup` and `lore init`.
+ */
+import { mkdir, writeFile } from 'fs/promises';
+import { existsSync } from 'fs';
+import { execSync } from 'child_process';
+import path from 'path';
+// ============================================================================
+// Data Repo Init
+// ============================================================================
+/**
+ * Initialize a lore data repository at the given path.
+ * Creates directory structure, .gitignore, README, and git init.
+ * Idempotent — safe to call on an existing directory.
+ */
+export async function initDataRepo(dirPath) {
+    await mkdir(dirPath, { recursive: true });
+    await mkdir(path.join(dirPath, 'sources'), { recursive: true });
+    await mkdir(path.join(dirPath, 'retained'), { recursive: true });
+    // Create .gitignore if missing
+    const gitignorePath = path.join(dirPath, '.gitignore');
+    if (!existsSync(gitignorePath)) {
+        await writeFile(gitignorePath, `.env\n.env.local\n`);
+    }
+    // Create README if missing
+    const readmePath = path.join(dirPath, 'README.md');
+    if (!existsSync(readmePath)) {
+        await writeFile(readmePath, `# Lore Data Repository
+
+Your personal knowledge repository for Lore.
+
+## Structure
+
+- \`sources/\` - Ingested documents
+- \`retained/\` - Explicitly saved insights
+
+Vector embeddings are stored in Supabase (cloud) for multi-machine access.
+`);
+    }
+    // Git init if not already a repo
+    if (!existsSync(path.join(dirPath, '.git'))) {
+        try {
+            execSync('git init', { cwd: dirPath, stdio: 'pipe' });
+            execSync('git add .', { cwd: dirPath, stdio: 'pipe' });
+            execSync('git commit -m "Initial lore data repository"', {
+                cwd: dirPath,
+                stdio: 'pipe',
+            });
+        }
+        catch {
+            // git not installed or commit failed — non-fatal
+        }
+    }
+}
+// ============================================================================
+// Welcome Document (seeded during setup)
+// ============================================================================
+export const WELCOME_DOC_CONTENT = `# Getting Started with Lore
+
+Welcome to Lore — your research knowledge repository.
+
+## What is Lore?
+
+Lore preserves your original sources (meeting notes, interviews, documents) and makes them searchable with full citations. Unlike a memory system, Lore keeps the original context so you can always trace back to the source.
+
+## Quick Start
+
+- **Search**: Run \`lore search "your query"\` to find relevant documents
+- **Sync**: Run \`lore sync\` to discover and index new files from your sync sources
+- **Browse**: Run \`lore browse\` to explore your knowledge base in the terminal
+- **Research**: Use the \`research\` MCP tool for deep, multi-step research with citations
+
+## Adding Documents
+
+1. **Sync sources**: Run \`lore sync add\` to watch a directory for new files
+2. **Direct ingest**: Use the \`ingest\` MCP tool to add documents from any agent
+3. **Manual sync**: Run \`lore sync\` after adding files to your data directory
+
+## Background Daemon
+
+Run \`lore sync start\` to launch a background daemon that watches for new files and auto-indexes them. Check status with \`lore sync status\`.
+
+## Tips
+
+- Lore extracts metadata using AI at sync time, so your documents are enriched automatically
+- Use projects to organize related documents together
+- The research tool can cross-reference multiple sources and synthesize findings with citations
+`;
+// ============================================================================
+// GitHub CLI helpers
+// ============================================================================
+/**
+ * Check if GitHub CLI is available and authenticated.
+ */
+export async function isGhAvailable() {
+    try {
+        execSync('which gh', { stdio: 'pipe' });
+        execSync('gh auth status', { stdio: 'pipe' });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Create a private GitHub repo from the given local directory.
+ * Returns the repo URL on success, null on failure.
+ */
+export async function createGithubRepo(dirPath, name) {
+    try {
+        execSync(`gh repo create ${name} --private --source=. --push`, {
+            cwd: dirPath,
+            stdio: 'pipe',
+        });
+        return getGitRemoteUrl(dirPath);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Get the git remote origin URL for a directory.
+ * Returns null if no remote is configured.
+ */
+export function getGitRemoteUrl(dirPath) {
+    try {
+        return execSync('git remote get-url origin', {
+            cwd: dirPath,
+            stdio: 'pipe',
+        })
+            .toString()
+            .trim();
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Check if a directory is a git repository.
+ */
+export function isGitRepo(dirPath) {
+    return existsSync(path.join(dirPath, '.git'));
+}

package/dist/core/embedder.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Lore - Embedding Generation
+ *
+ * Handles generating embeddings for semantic search using OpenAI's API.
+ * Adapted from granola-extractor with retry logic and batching.
+ */
+export declare function getEmbeddingDimensions(): number;
+export declare function generateEmbedding(text: string, model?: string): Promise<number[]>;
+export declare function generateEmbeddings(texts: string[], model?: string, options?: {
+    batchSize?: number;
+    onProgress?: (completed: number, total: number) => void;
+}): Promise<number[][]>;
+/**
+ * Create searchable text with context prefix for better embedding quality
+ */
+export declare function createSearchableText(content: {
+    type: 'summary' | 'theme' | 'quote' | 'decision' | 'requirement' | 'note';
+    text: string;
+    theme_name?: string;
+    context?: string;
+    project?: string;
+}): string;

package/dist/core/embedder.js

@@ -0,0 +1,104 @@
+/**
+ * Lore - Embedding Generation
+ *
+ * Handles generating embeddings for semantic search using OpenAI's API.
+ * Adapted from granola-extractor with retry logic and batching.
+ */
+import OpenAI from 'openai';
+let openaiClient = null;
+function getOpenAI() {
+    if (!openaiClient) {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            throw new Error('OPENAI_API_KEY is not configured. Run "lore setup" to set your API keys.');
+        }
+        openaiClient = new OpenAI({ apiKey });
+    }
+    return openaiClient;
+}
+const DEFAULT_MODEL = 'text-embedding-3-small';
+const EMBEDDING_DIMENSIONS = 1536;
+export function getEmbeddingDimensions() {
+    return EMBEDDING_DIMENSIONS;
+}
+export async function generateEmbedding(text, model = DEFAULT_MODEL) {
+    const openai = getOpenAI();
+    // Truncate very long texts (embedding model has token limits)
+    const truncatedText = text.substring(0, 8000);
+    // Retry logic for transient errors
+    const maxRetries = 3;
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const response = await openai.embeddings.create({
+                model,
+                input: truncatedText,
+            });
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            lastError = error;
+            const isRetryable = lastError.message?.includes('Connection') ||
+                lastError.message?.includes('timeout') ||
+                lastError.message?.includes('ECONNRESET') ||
+                lastError.message?.includes('rate limit');
+            if (isRetryable && attempt < maxRetries) {
+                const delay = Math.pow(2, attempt) * 1000;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+                continue;
+            }
+            throw error;
+        }
+    }
+    throw lastError;
+}
+export async function generateEmbeddings(texts, model = DEFAULT_MODEL, options = {}) {
+    const openai = getOpenAI();
+    const { batchSize = 100, onProgress } = options;
+    const results = [];
+    // Process in batches
+    for (let i = 0; i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        // Truncate each text
+        const truncatedBatch = batch.map((t) => t.substring(0, 8000));
+        const response = await openai.embeddings.create({
+            model,
+            input: truncatedBatch,
+        });
+        // Sort by index to maintain order
+        const sortedData = response.data.sort((a, b) => a.index - b.index);
+        for (const item of sortedData) {
+            results.push(item.embedding);
+        }
+        onProgress?.(Math.min(i + batchSize, texts.length), texts.length);
+        // Small delay between batches to avoid rate limits
+        if (i + batchSize < texts.length) {
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+    }
+    return results;
+}
+/**
+ * Create searchable text with context prefix for better embedding quality
+ */
+export function createSearchableText(content) {
+    const projectPrefix = content.project ? `[Project: ${content.project}] ` : '';
+    switch (content.type) {
+        case 'summary':
+            return `${projectPrefix}Summary: ${content.text}`;
+        case 'theme':
+            return `${projectPrefix}Theme ${content.theme_name}: ${content.text}`;
+        case 'quote':
+            return content.context
+                ? `${projectPrefix}Quote about ${content.context}: "${content.text}"`
+                : `${projectPrefix}Quote: "${content.text}"`;
+        case 'decision':
+            return `${projectPrefix}Decision: ${content.text}`;
+        case 'requirement':
+            return `${projectPrefix}Requirement: ${content.text}`;
+        case 'note':
+            return `${projectPrefix}Note: ${content.text}`;
+        default:
+            return content.text;
+    }
+}

package/dist/core/git.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Git utilities for Lore data synchronization
+ */
+export interface GitResult {
+    success: boolean;
+    message?: string;
+    error?: string;
+}
+/**
+ * Check if a directory is a git repository
+ */
+export declare function isGitRepo(dir: string): Promise<boolean>;
+/**
+ * Check if the repo has a remote configured
+ */
+export declare function hasRemote(dir: string): Promise<boolean>;
+/**
+ * Check if there are uncommitted changes
+ */
+export declare function hasChanges(dir: string): Promise<boolean>;
+/**
+ * Git pull with rebase
+ */
+export declare function gitPull(dir: string): Promise<GitResult>;
+/**
+ * Git add, commit, and push
+ */
+export declare function gitCommitAndPush(dir: string, message: string): Promise<GitResult>;
+/**
+ * Delete a file and commit the removal to its git repo (if git-tracked).
+ * No-op if the file doesn't exist.
+ */
+export declare function deleteFileAndCommit(filePath: string, commitMessage: string): Promise<void>;
+/**
+ * Sync: pull, then push any local changes
+ */
+export declare function gitSync(dir: string): Promise<GitResult>;

package/dist/core/git.js

@@ -0,0 +1,140 @@
+/**
+ * Git utilities for Lore data synchronization
+ */
+import { exec } from 'child_process';
+import { promisify } from 'util';
+const execAsync = promisify(exec);
+/**
+ * Check if a directory is a git repository
+ */
+export async function isGitRepo(dir) {
+    try {
+        await execAsync('git rev-parse --git-dir', { cwd: dir });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Check if the repo has a remote configured
+ */
+export async function hasRemote(dir) {
+    try {
+        const { stdout } = await execAsync('git remote', { cwd: dir });
+        return stdout.trim().length > 0;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Check if there are uncommitted changes
+ */
+export async function hasChanges(dir) {
+    try {
+        const { stdout } = await execAsync('git status --porcelain', { cwd: dir });
+        return stdout.trim().length > 0;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Git pull with rebase
+ */
+export async function gitPull(dir) {
+    try {
+        if (!(await isGitRepo(dir))) {
+            return { success: false, error: 'Not a git repository' };
+        }
+        if (!(await hasRemote(dir))) {
+            return { success: false, error: 'No remote configured' };
+        }
+        // Stash any local changes
+        await execAsync('git stash', { cwd: dir }).catch(() => { });
+        // Pull with rebase
+        const { stdout } = await execAsync('git pull --rebase', { cwd: dir });
+        const pulled = !stdout.includes('Already up to date');
+        return {
+            success: true,
+            message: pulled ? 'Pulled new changes' : 'Already up to date'
+        };
+    }
+    catch (error) {
+        return { success: false, error: String(error) };
+    }
+}
+/**
+ * Git add, commit, and push
+ */
+export async function gitCommitAndPush(dir, message) {
+    try {
+        if (!(await isGitRepo(dir))) {
+            return { success: false, error: 'Not a git repository' };
+        }
+        // Check for changes
+        if (!(await hasChanges(dir))) {
+            return { success: true, message: 'No changes to commit' };
+        }
+        // Stage all changes
+        await execAsync('git add -A', { cwd: dir });
+        // Commit
+        await execAsync(`git commit -m "${message.replace(/"/g, '\\"')}"`, { cwd: dir });
+        // Push if remote exists
+        if (await hasRemote(dir)) {
+            await execAsync('git push', { cwd: dir });
+            return { success: true, message: 'Committed and pushed' };
+        }
+        else {
+            return { success: true, message: 'Committed (no remote to push)' };
+        }
+    }
+    catch (error) {
+        return { success: false, error: String(error) };
+    }
+}
+/**
+ * Delete a file and commit the removal to its git repo (if git-tracked).
+ * No-op if the file doesn't exist.
+ */
+export async function deleteFileAndCommit(filePath, commitMessage) {
+    const { existsSync } = await import('fs');
+    if (!existsSync(filePath))
+        return;
+    const { rm } = await import('fs/promises');
+    const dir = (await import('path')).dirname(filePath);
+    await rm(filePath);
+    if (await isGitRepo(dir)) {
+        await gitCommitAndPush(dir, commitMessage);
+    }
+}
+/**
+ * Sync: pull, then push any local changes
+ */
+export async function gitSync(dir) {
+    try {
+        if (!(await isGitRepo(dir))) {
+            return { success: false, error: 'Not a git repository' };
+        }
+        if (!(await hasRemote(dir))) {
+            return { success: false, error: 'No remote configured' };
+        }
+        // Pull first
+        const pullResult = await gitPull(dir);
+        if (!pullResult.success) {
+            return pullResult;
+        }
+        // Push any local changes
+        if (await hasChanges(dir)) {
+            await execAsync('git add -A', { cwd: dir });
+            await execAsync('git commit -m "Auto-sync from Lore"', { cwd: dir });
+            await execAsync('git push', { cwd: dir });
+            return { success: true, message: 'Pulled and pushed changes' };
+        }
+        return { success: true, message: pullResult.message };
+    }
+    catch (error) {
+        return { success: false, error: String(error) };
+    }
+}

package/dist/core/index.d.ts

@@ -0,0 +1,4 @@
+export * from './types.js';
+export * from './embedder.js';
+export * from './vector-store.js';
+export * from './insight-extractor.js';

package/dist/core/index.js

@@ -0,0 +1,5 @@
+// Core exports
+export * from './types.js';
+export * from './embedder.js';
+export * from './vector-store.js';
+export * from './insight-extractor.js';

package/dist/core/insight-extractor.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Lore - Insight Extraction
+ *
+ * Generates summaries for source content. Keeps it simple - the agent
+ * does the real analysis at query time with full context.
+ */
+import type { Quote, Theme } from './types.js';
+export interface ExtractedInsights {
+    summary: string;
+    themes: Theme[];
+    quotes: Quote[];
+}
+export declare function extractInsights(content: string, title: string, sourceId: string, options?: {
+    contentType?: string;
+    model?: string;
+}): Promise<ExtractedInsights>;
+export declare function extractInsightsBatch(documents: Array<{
+    id: string;
+    title: string;
+    content: string;
+    contentType?: string;
+}>, options?: {
+    model?: string;
+    concurrency?: number;
+    onProgress?: (completed: number, total: number, title: string) => void;
+}): Promise<Map<string, ExtractedInsights>>;

package/dist/core/insight-extractor.js

@@ -0,0 +1,114 @@
+/**
+ * Lore - Insight Extraction
+ *
+ * Generates summaries for source content. Keeps it simple - the agent
+ * does the real analysis at query time with full context.
+ */
+import OpenAI from 'openai';
+let openaiClient = null;
+function getOpenAI() {
+    if (!openaiClient) {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            throw new Error('OPENAI_API_KEY environment variable is required');
+        }
+        openaiClient = new OpenAI({ apiKey });
+    }
+    return openaiClient;
+}
+export async function extractInsights(content, title, sourceId, options = {}) {
+    const { contentType = 'document', model = 'gpt-4o-mini' } = options;
+    const openai = getOpenAI();
+    // Simple, content-type aware summary prompt
+    const contextHint = {
+        interview: 'This is a user interview or research call.',
+        meeting: 'This is a meeting transcript.',
+        conversation: 'This is an AI conversation (Claude, ChatGPT, etc.).',
+        analysis: 'This is a research or competitor analysis document.',
+        survey: 'This is survey results or user feedback data.',
+        document: 'This is a document or notes.',
+        note: 'This is a note or memo.',
+    }[contentType] || 'This is a document.';
+    const systemPrompt = `You generate concise, information-dense summaries.
+
+${contextHint}
+
+Write a 2-4 sentence summary that captures:
+- What this content is about
+- The key takeaways or findings
+- Any important decisions, insights, or action items
+
+Be specific and factual. Include names, numbers, and concrete details when present.
+Return only the summary text, no JSON or formatting.`;
+    const userPrompt = `Title: ${title}
+
+${content.substring(0, 30000)}`;
+    // Retry logic for transient errors
+    const maxRetries = 3;
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const response = await openai.chat.completions.create({
+                model,
+                messages: [
+                    { role: 'system', content: systemPrompt },
+                    { role: 'user', content: userPrompt },
+                ],
+                temperature: 0.3,
+                max_tokens: 500,
+            });
+            const summary = response.choices[0]?.message?.content?.trim();
+            if (!summary) {
+                throw new Error('No content in OpenAI response');
+            }
+            return {
+                summary,
+                themes: [], // Agent extracts at query time
+                quotes: [], // Agent extracts at query time
+            };
+        }
+        catch (error) {
+            lastError = error;
+            const isRetryable = lastError.message?.includes('Connection') ||
+                lastError.message?.includes('timeout') ||
+                lastError.message?.includes('ECONNRESET') ||
+                lastError.message?.includes('rate limit');
+            if (isRetryable && attempt < maxRetries) {
+                const delay = Math.pow(2, attempt) * 1000;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+                continue;
+            }
+            break;
+        }
+    }
+    // All retries failed - use first 200 chars as fallback
+    console.error(`Error generating summary for "${title}":`, lastError);
+    return {
+        summary: content.substring(0, 200).trim() + '...',
+        themes: [],
+        quotes: [],
+    };
+}
+export async function extractInsightsBatch(documents, options = {}) {
+    const { model = 'gpt-4o-mini', concurrency = 3, onProgress } = options;
+    const results = new Map();
+    for (let i = 0; i < documents.length; i += concurrency) {
+        const batch = documents.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (doc) => {
+            const insights = await extractInsights(doc.content, doc.title, doc.id, {
+                contentType: doc.contentType,
+                model,
+            });
+            onProgress?.(i + batch.indexOf(doc) + 1, documents.length, doc.title);
+            return { id: doc.id, insights };
+        }));
+        for (const { id, insights } of batchResults) {
+            results.set(id, insights);
+        }
+        // Delay between batches
+        if (i + concurrency < documents.length) {
+            await new Promise((resolve) => setTimeout(resolve, 500));
+        }
+    }
+    return results;
+}

package/dist/core/local-search.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Local Search - Regex/pattern search in local source files
+ *
+ * Provides grep-like functionality for searching source content
+ * stored in the data directory. Tries ripgrep first for speed,
+ * falls back to native JavaScript regex.
+ */
+export interface LocalSearchMatch {
+    line_number: number;
+    line_content: string;
+    match_start: number;
+    match_end: number;
+}
+export interface LocalSearchResult {
+    source_id: string;
+    file_path: string;
+    matches: LocalSearchMatch[];
+}
+export interface LocalSearchOptions {
+    /** Maximum results per file */
+    maxMatchesPerFile?: number;
+    /** Maximum total results */
+    maxTotalResults?: number;
+    /** Case-insensitive search */
+    ignoreCase?: boolean;
+    /** Lines of context before match */
+    contextBefore?: number;
+    /** Lines of context after match */
+    contextAfter?: number;
+}
+/**
+ * Search local source files using regex pattern
+ *
+ * @param dataDir - The lore data directory
+ * @param pattern - Regex pattern to search for
+ * @param options - Search options
+ * @returns Array of search results with matches
+ */
+export declare function searchLocalFiles(dataDir: string, pattern: string, options?: LocalSearchOptions): Promise<LocalSearchResult[]>;
+/**
+ * Get a snippet of text around a match for display
+ */
+export declare function getMatchSnippet(lineContent: string, matchStart: number, matchEnd: number, maxLength?: number): string;