@getlore/cli 0.2.0

package/dist/core/local-search.js

@@ -0,0 +1,221 @@
+/**
+ * Local Search - Regex/pattern search in local source files
+ *
+ * Provides grep-like functionality for searching source content
+ * stored in the data directory. Tries ripgrep first for speed,
+ * falls back to native JavaScript regex.
+ */
+import { spawn } from 'child_process';
+import { readdir, readFile } from 'fs/promises';
+import path from 'path';
+import { existsSync } from 'fs';
+/**
+ * Check if ripgrep is available
+ */
+async function hasRipgrep() {
+    return new Promise((resolve) => {
+        const proc = spawn('rg', ['--version']);
+        proc.on('error', () => resolve(false));
+        proc.on('close', (code) => resolve(code === 0));
+    });
+}
+/**
+ * Search using ripgrep (fast)
+ */
+async function searchWithRipgrep(sourcesDir, pattern, options) {
+    const { maxMatchesPerFile = 10, maxTotalResults = 100, ignoreCase = false, contextBefore = 0, contextAfter = 0, } = options;
+    return new Promise((resolve, reject) => {
+        const args = [
+            '--json',
+            '--max-count', String(maxMatchesPerFile),
+            '--glob', 'content.md',
+        ];
+        if (ignoreCase) {
+            args.push('--ignore-case');
+        }
+        if (contextBefore > 0) {
+            args.push('-B', String(contextBefore));
+        }
+        if (contextAfter > 0) {
+            args.push('-A', String(contextAfter));
+        }
+        args.push(pattern, sourcesDir);
+        const proc = spawn('rg', args);
+        let stdout = '';
+        let stderr = '';
+        proc.stdout.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr.on('data', (data) => {
+            stderr += data.toString();
+        });
+        proc.on('error', (err) => {
+            reject(err);
+        });
+        proc.on('close', (code) => {
+            // ripgrep returns 1 if no matches, 0 if matches, 2 if error
+            if (code === 2) {
+                reject(new Error(`ripgrep error: ${stderr}`));
+                return;
+            }
+            const results = [];
+            const resultMap = new Map();
+            // Parse JSON lines output
+            const lines = stdout.split('\n').filter((l) => l.trim());
+            for (const line of lines) {
+                try {
+                    const entry = JSON.parse(line);
+                    if (entry.type === 'match') {
+                        const filePath = entry.data.path.text;
+                        // Extract source_id from path (e.g., /path/sources/{source_id}/content.md)
+                        const parts = filePath.split(path.sep);
+                        const contentIdx = parts.findIndex((p) => p === 'content.md');
+                        const sourceId = contentIdx > 0 ? parts[contentIdx - 1] : path.basename(path.dirname(filePath));
+                        let result = resultMap.get(sourceId);
+                        if (!result) {
+                            result = {
+                                source_id: sourceId,
+                                file_path: filePath,
+                                matches: [],
+                            };
+                            resultMap.set(sourceId, result);
+                        }
+                        // Process submatches
+                        for (const submatch of entry.data.submatches || []) {
+                            result.matches.push({
+                                line_number: entry.data.line_number,
+                                line_content: entry.data.lines.text.replace(/\n$/, ''),
+                                match_start: submatch.start,
+                                match_end: submatch.end,
+                            });
+                        }
+                    }
+                }
+                catch {
+                    // Skip invalid JSON lines
+                }
+            }
+            // Convert map to array and limit total results
+            for (const result of resultMap.values()) {
+                if (results.length >= maxTotalResults)
+                    break;
+                results.push(result);
+            }
+            resolve(results);
+        });
+    });
+}
+/**
+ * Search using native JavaScript regex (fallback)
+ */
+async function searchWithNative(sourcesDir, pattern, options) {
+    const { maxMatchesPerFile = 10, maxTotalResults = 100, ignoreCase = false, } = options;
+    const results = [];
+    // Create regex
+    let regex;
+    try {
+        regex = new RegExp(pattern, ignoreCase ? 'gi' : 'g');
+    }
+    catch (err) {
+        throw new Error(`Invalid regex pattern: ${pattern}`);
+    }
+    // Read all source directories
+    if (!existsSync(sourcesDir)) {
+        return [];
+    }
+    const entries = await readdir(sourcesDir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (results.length >= maxTotalResults)
+            break;
+        if (!entry.isDirectory())
+            continue;
+        const contentPath = path.join(sourcesDir, entry.name, 'content.md');
+        if (!existsSync(contentPath))
+            continue;
+        try {
+            const content = await readFile(contentPath, 'utf-8');
+            const lines = content.split('\n');
+            const matches = [];
+            for (let i = 0; i < lines.length && matches.length < maxMatchesPerFile; i++) {
+                const line = lines[i];
+                regex.lastIndex = 0; // Reset regex state
+                let match;
+                while ((match = regex.exec(line)) !== null && matches.length < maxMatchesPerFile) {
+                    matches.push({
+                        line_number: i + 1,
+                        line_content: line,
+                        match_start: match.index,
+                        match_end: match.index + match[0].length,
+                    });
+                    // Prevent infinite loop on zero-length matches
+                    if (match[0].length === 0) {
+                        regex.lastIndex++;
+                    }
+                }
+            }
+            if (matches.length > 0) {
+                results.push({
+                    source_id: entry.name,
+                    file_path: contentPath,
+                    matches,
+                });
+            }
+        }
+        catch {
+            // Skip files that can't be read
+        }
+    }
+    return results;
+}
+/**
+ * Search local source files using regex pattern
+ *
+ * @param dataDir - The lore data directory
+ * @param pattern - Regex pattern to search for
+ * @param options - Search options
+ * @returns Array of search results with matches
+ */
+export async function searchLocalFiles(dataDir, pattern, options = {}) {
+    const sourcesDir = path.join(dataDir, 'sources');
+    if (!existsSync(sourcesDir)) {
+        return [];
+    }
+    // Try ripgrep first, fall back to native
+    const useRipgrep = await hasRipgrep();
+    if (useRipgrep) {
+        try {
+            return await searchWithRipgrep(sourcesDir, pattern, options);
+        }
+        catch {
+            // Fall back to native on error
+            return await searchWithNative(sourcesDir, pattern, options);
+        }
+    }
+    return await searchWithNative(sourcesDir, pattern, options);
+}
+/**
+ * Get a snippet of text around a match for display
+ */
+export function getMatchSnippet(lineContent, matchStart, matchEnd, maxLength = 100) {
+    const matchText = lineContent.slice(matchStart, matchEnd);
+    // Calculate window around match
+    const halfWindow = Math.floor((maxLength - matchText.length) / 2);
+    let start = Math.max(0, matchStart - halfWindow);
+    let end = Math.min(lineContent.length, matchEnd + halfWindow);
+    // Adjust if we're near edges
+    if (start === 0) {
+        end = Math.min(lineContent.length, maxLength);
+    }
+    else if (end === lineContent.length) {
+        start = Math.max(0, lineContent.length - maxLength);
+    }
+    let snippet = lineContent.slice(start, end);
+    // Add ellipses
+    if (start > 0) {
+        snippet = '...' + snippet;
+    }
+    if (end < lineContent.length) {
+        snippet = snippet + '...';
+    }
+    return snippet;
+}

package/dist/core/themes.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Lore - Theme Definitions
+ *
+ * Pre-defined themes for insight extraction from sources.
+ * Adapted from granola-extractor.
+ */
+export interface ThemeDefinition {
+    id: string;
+    name: string;
+    prompt: string;
+}
+export declare const THEMES: ThemeDefinition[];
+export declare function getThemeById(id: string): ThemeDefinition | undefined;
+export declare function getThemeNames(): string[];
+export declare function getThemePromptList(): string;

package/dist/core/themes.js

@@ -0,0 +1,77 @@
+/**
+ * Lore - Theme Definitions
+ *
+ * Pre-defined themes for insight extraction from sources.
+ * Adapted from granola-extractor.
+ */
+export const THEMES = [
+    {
+        id: 'pain-points',
+        name: 'Pain Points',
+        prompt: 'Problems, frustrations, difficulties, blockers, challenges, risks identified',
+    },
+    {
+        id: 'feature-requests',
+        name: 'Feature Requests',
+        prompt: 'Desired features, wishlist items, suggestions, improvements, capabilities needed',
+    },
+    {
+        id: 'positive-feedback',
+        name: 'Positive Feedback',
+        prompt: 'Strengths, what works well, advantages, praise, value delivered',
+    },
+    {
+        id: 'pricing',
+        name: 'Pricing & Value',
+        prompt: 'Cost, pricing, value perception, ROI, budget, willingness to pay, financial considerations',
+    },
+    {
+        id: 'competition',
+        name: 'Competition',
+        prompt: 'Competitors, alternatives, market comparison, competitive advantages/disadvantages',
+    },
+    {
+        id: 'workflow',
+        name: 'Workflow & Process',
+        prompt: 'How things work, processes, workarounds, current state, tools and methods used',
+    },
+    {
+        id: 'decisions',
+        name: 'Decisions',
+        prompt: 'Key decisions, conclusions, action items, next steps, agreements, choices made',
+    },
+    {
+        id: 'questions',
+        name: 'Open Questions',
+        prompt: 'Unanswered questions, uncertainties, things needing clarification, follow-ups',
+    },
+    {
+        id: 'requirements',
+        name: 'Requirements',
+        prompt: 'Must-haves, specifications, constraints, acceptance criteria, needs, dependencies',
+    },
+    {
+        id: 'insights',
+        name: 'Key Insights',
+        prompt: 'Important learnings, discoveries, observations, patterns, strategic insights',
+    },
+    {
+        id: 'data-points',
+        name: 'Data & Metrics',
+        prompt: 'Statistics, numbers, measurements, KPIs, benchmarks, quantitative findings',
+    },
+    {
+        id: 'opportunities',
+        name: 'Opportunities',
+        prompt: 'Market opportunities, growth areas, untapped potential, strategic openings',
+    },
+];
+export function getThemeById(id) {
+    return THEMES.find((t) => t.id === id);
+}
+export function getThemeNames() {
+    return THEMES.map((t) => t.id);
+}
+export function getThemePromptList() {
+    return THEMES.map((t) => `- ${t.id}: ${t.prompt}`).join('\n');
+}

package/dist/core/types.d.ts

@@ -0,0 +1,177 @@
+/**
+ * Lore - Core Types
+ *
+ * A three-layer knowledge architecture:
+ * 1. Source Documents - Immutable original content (interviews, transcripts, docs)
+ * 2. Extracted Insights - Quotes, themes, decisions with links back to sources
+ * 3. Working Memory - Synthesized context for agent consumption
+ */
+export type SourceType = string;
+export type SearchMode = 'semantic' | 'keyword' | 'hybrid' | 'regex';
+export type ContentType = 'interview' | 'meeting' | 'conversation' | 'document' | 'note' | 'analysis' | 'survey' | 'research' | 'decision' | 'insight' | 'requirement';
+export interface SourceDocument {
+    id: string;
+    source_type: SourceType;
+    source_id: string;
+    source_path?: string;
+    title: string;
+    content: string;
+    content_type: ContentType;
+    created_at: string;
+    imported_at: string;
+    participants?: string[];
+    projects: string[];
+    tags: string[];
+}
+export interface Citation {
+    source_id: string;
+    location?: string;
+    context?: string;
+}
+export interface Quote {
+    id: string;
+    text: string;
+    speaker?: 'user' | 'participant' | 'ai' | 'unknown';
+    speaker_name?: string;
+    timestamp?: string;
+    theme?: string;
+    citation: Citation;
+}
+export interface Theme {
+    name: string;
+    evidence: Quote[];
+    summary?: string;
+}
+export type ThemeName = 'pain-points' | 'feature-requests' | 'positive-feedback' | 'pricing' | 'competition' | 'workflow' | 'decisions' | 'requirements' | 'insights';
+export interface Decision {
+    id: string;
+    decision: string;
+    rationale: string;
+    alternatives_considered?: string[];
+    made_at: string;
+    citation: Citation;
+    project_id: string;
+}
+export interface Requirement {
+    id: string;
+    description: string;
+    priority: 'must' | 'should' | 'could' | 'wont';
+    source_quotes: Quote[];
+    project_id: string;
+}
+export type ProjectStatus = 'active' | 'paused' | 'completed' | 'archived';
+export interface Project {
+    id: string;
+    name: string;
+    description: string;
+    status: ProjectStatus;
+    created_at: string;
+    updated_at: string;
+    parent_id?: string;
+    source_count?: number;
+    quote_count?: number;
+    decision_count?: number;
+}
+export type LineageEventType = 'created' | 'decision' | 'pivot' | 'milestone' | 'insight' | 'delegation';
+export interface LineageEvent {
+    id: string;
+    project_id: string;
+    event_type: LineageEventType;
+    title: string;
+    description: string;
+    timestamp: string;
+    source_ids: string[];
+}
+export interface SourceRecord {
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string;
+    tags: string;
+    created_at: string;
+    summary: string;
+    themes_json: string;
+    quotes_json: string;
+    has_full_content: boolean;
+    vector: number[];
+    source_url?: string;
+    source_name?: string;
+}
+export interface ChunkRecord {
+    id: string;
+    source_id: string;
+    content: string;
+    type: 'quote' | 'theme' | 'summary' | 'decision' | 'requirement' | 'note' | 'insight';
+    theme_name?: string;
+    speaker?: string;
+    timestamp?: string;
+    vector: number[];
+}
+export interface SearchResult {
+    source_id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    summary: string;
+    relevance_score: number;
+    matching_quotes: Quote[];
+    matching_themes: string[];
+}
+/**
+ * Extended search result with ranking information from hybrid search
+ */
+export interface SearchResultWithRanks extends SearchResult {
+    /** Rank in semantic (vector) search results, null if not in semantic results */
+    semantic_rank?: number;
+    /** Rank in keyword (full-text) search results, null if not in keyword results */
+    keyword_rank?: number;
+}
+export interface ResearchPackage {
+    query: string;
+    project?: string;
+    generated_at: string;
+    summary: string;
+    key_findings: string[];
+    conflicts_resolved?: string[];
+    supporting_quotes: Quote[];
+    related_decisions: Decision[];
+    sources_consulted: Array<{
+        id: string;
+        title: string;
+        source_type: SourceType;
+        relevance: number;
+    }>;
+    gaps_identified?: string[];
+    suggested_queries?: string[];
+}
+export interface SearchArgs {
+    query: string;
+    project?: string;
+    source_type?: SourceType;
+    content_type?: ContentType;
+    limit?: number;
+    mode?: SearchMode;
+}
+export interface RetainArgs {
+    content: string;
+    project: string;
+    type: 'insight' | 'decision' | 'requirement' | 'note';
+    source_context?: string;
+    tags?: string[];
+}
+export interface ResearchArgs {
+    task: string;
+    project?: string;
+    depth?: 'quick' | 'thorough' | 'exhaustive';
+    include_sources?: boolean;
+}
+export interface IngestArgs {
+    path: string;
+    source_type: SourceType;
+    project?: string;
+    tags?: string[];
+    source_url?: string;
+    source_name?: string;
+}

package/dist/core/types.js

@@ -0,0 +1,9 @@
+/**
+ * Lore - Core Types
+ *
+ * A three-layer knowledge architecture:
+ * 1. Source Documents - Immutable original content (interviews, transcripts, docs)
+ * 2. Extracted Insights - Quotes, themes, decisions with links back to sources
+ * 3. Working Memory - Synthesized context for agent consumption
+ */
+export {};

package/dist/core/user-settings.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Lore - User Settings (Supabase key-value store)
+ *
+ * Generic per-user settings stored in Supabase for cross-machine discovery.
+ * Uses the user_settings table with RLS for data isolation.
+ */
+export declare const SETTING_DATA_REPO_URL = "data_repo_url";
+/**
+ * Get a user setting by key. Returns null if not found.
+ */
+export declare function getUserSetting(key: string): Promise<string | null>;
+/**
+ * Set a user setting (upsert). Creates or updates the value for the given key.
+ */
+export declare function setUserSetting(key: string, value: string): Promise<void>;

package/dist/core/user-settings.js

@@ -0,0 +1,42 @@
+/**
+ * Lore - User Settings (Supabase key-value store)
+ *
+ * Generic per-user settings stored in Supabase for cross-machine discovery.
+ * Uses the user_settings table with RLS for data isolation.
+ */
+import { getSupabase } from './vector-store.js';
+// ============================================================================
+// Well-known setting keys
+// ============================================================================
+export const SETTING_DATA_REPO_URL = 'data_repo_url';
+// ============================================================================
+// CRUD
+// ============================================================================
+/**
+ * Get a user setting by key. Returns null if not found.
+ */
+export async function getUserSetting(key) {
+    const client = await getSupabase();
+    const { data, error } = await client
+        .from('user_settings')
+        .select('value')
+        .eq('key', key)
+        .single();
+    if (error || !data) {
+        return null;
+    }
+    return data.value;
+}
+/**
+ * Set a user setting (upsert). Creates or updates the value for the given key.
+ */
+export async function setUserSetting(key, value) {
+    const client = await getSupabase();
+    const { error } = await client
+        .from('user_settings')
+        .upsert({ key, value, updated_at: new Date().toISOString() }, { onConflict: 'user_id,key' });
+    if (error) {
+        console.error(`[setUserSetting] Error setting '${key}':`, error);
+        throw error;
+    }
+}

package/dist/core/vector-store-lance.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Lore - Vector Store
+ *
+ * LanceDB-based vector storage for semantic search across sources and chunks.
+ * Adapted from granola-extractor with expanded schema for projects and citations.
+ */
+import * as lancedb from '@lancedb/lancedb';
+import type { SourceRecord, ChunkRecord, Quote, Theme, SourceType, ContentType } from './types.js';
+export declare function getDatabase(dbPath: string): Promise<lancedb.Connection>;
+/**
+ * Reset the database connection to force a fresh read
+ */
+export declare function resetDatabaseConnection(): void;
+export declare function closeDatabase(): Promise<void>;
+export declare function indexExists(dbPath: string): Promise<boolean>;
+export declare function initializeTables(dbPath: string): Promise<void>;
+/**
+ * Add a single source to the existing index (for retain operations)
+ */
+export declare function addSource(dbPath: string, source: SourceRecord, vector: number[]): Promise<void>;
+/**
+ * Add chunks to the existing index
+ */
+export declare function addChunks(dbPath: string, chunks: ChunkRecord[]): Promise<void>;
+export declare function storeSources(dbPath: string, sources: Array<{
+    source: SourceRecord;
+    vector: number[];
+}>): Promise<void>;
+export declare function storeChunks(dbPath: string, chunks: ChunkRecord[]): Promise<void>;
+export declare function searchSources(dbPath: string, queryVector: number[], options?: {
+    limit?: number;
+    project?: string;
+    source_type?: SourceType;
+    content_type?: ContentType;
+    recency_boost?: number;
+}): Promise<Array<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    tags: string[];
+    created_at: string;
+    summary: string;
+    themes: Theme[];
+    quotes: Quote[];
+    score: number;
+}>>;
+export declare function searchChunks(dbPath: string, queryVector: number[], options?: {
+    limit?: number;
+    type?: ChunkRecord['type'];
+    theme_name?: string;
+    source_id?: string;
+}): Promise<Array<{
+    id: string;
+    source_id: string;
+    content: string;
+    type: string;
+    theme_name: string;
+    speaker: string;
+    timestamp: string;
+    score: number;
+}>>;
+export declare function getAllSources(dbPath: string, options?: {
+    project?: string;
+    source_type?: SourceType;
+    limit?: number;
+}): Promise<Array<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    created_at: string;
+    summary: string;
+}>>;
+export declare function getSourceById(dbPath: string, sourceId: string): Promise<{
+    id: string;
+    title: string;
+    source_type: SourceType;
+    content_type: ContentType;
+    projects: string[];
+    tags: string[];
+    created_at: string;
+    summary: string;
+    themes: Theme[];
+    quotes: Quote[];
+} | null>;
+export declare function getThemeStats(dbPath: string, project?: string): Promise<Map<string, {
+    source_count: number;
+    quote_count: number;
+}>>;
+export declare function getProjectStats(dbPath: string): Promise<Array<{
+    project: string;
+    source_count: number;
+    quote_count: number;
+    latest_activity: string;
+}>>;