@getlore/cli 0.2.0

package/dist/sync/config.js

@@ -0,0 +1,140 @@
+/**
+ * Lore - Sync Configuration
+ *
+ * Loads and validates sync-sources.json from ~/.config/lore/sync-sources.json
+ * This config is machine-specific (NOT in lore-data).
+ */
+import { readFile, writeFile, mkdir } from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+import os from 'os';
+// ============================================================================
+// Config Paths
+// ============================================================================
+const CONFIG_DIR = path.join(os.homedir(), '.config', 'lore');
+const CONFIG_FILE = path.join(CONFIG_DIR, 'sync-sources.json');
+export function getConfigPath() {
+    return CONFIG_FILE;
+}
+// ============================================================================
+// Path Expansion
+// ============================================================================
+export function expandPath(p) {
+    if (p.startsWith('~')) {
+        return path.join(os.homedir(), p.slice(1));
+    }
+    return p;
+}
+// ============================================================================
+// Default Config
+// ============================================================================
+function getDefaultConfig() {
+    return {
+        version: 1,
+        sources: [
+            {
+                name: 'Example Source',
+                path: '~/Documents/notes',
+                glob: '**/*.md',
+                project: 'notes',
+                enabled: false,
+            },
+        ],
+    };
+}
+// ============================================================================
+// Config Loading
+// ============================================================================
+export async function loadSyncConfig() {
+    if (!existsSync(CONFIG_FILE)) {
+        return getDefaultConfig();
+    }
+    try {
+        const content = await readFile(CONFIG_FILE, 'utf-8');
+        const config = JSON.parse(content);
+        // Validate version
+        if (config.version !== 1) {
+            console.warn(`[sync-config] Unknown config version: ${config.version}, expected 1`);
+        }
+        // Validate sources
+        if (!Array.isArray(config.sources)) {
+            throw new Error('Invalid config: sources must be an array');
+        }
+        for (const source of config.sources) {
+            if (!source.name || typeof source.name !== 'string') {
+                throw new Error(`Invalid source: missing or invalid 'name'`);
+            }
+            if (!source.path || typeof source.path !== 'string') {
+                throw new Error(`Invalid source "${source.name}": missing or invalid 'path'`);
+            }
+            if (!source.glob || typeof source.glob !== 'string') {
+                throw new Error(`Invalid source "${source.name}": missing or invalid 'glob'`);
+            }
+            if (!source.project || typeof source.project !== 'string') {
+                throw new Error(`Invalid source "${source.name}": missing or invalid 'project'`);
+            }
+            if (typeof source.enabled !== 'boolean') {
+                source.enabled = true; // Default to enabled
+            }
+        }
+        return config;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return getDefaultConfig();
+        }
+        throw error;
+    }
+}
+export async function saveSyncConfig(config) {
+    await mkdir(CONFIG_DIR, { recursive: true });
+    await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2));
+}
+export async function initializeSyncConfig() {
+    if (existsSync(CONFIG_FILE)) {
+        return loadSyncConfig();
+    }
+    const config = getDefaultConfig();
+    await saveSyncConfig(config);
+    return config;
+}
+// ============================================================================
+// Config Manipulation
+// ============================================================================
+export async function addSyncSource(source) {
+    const config = await loadSyncConfig();
+    // Check for duplicate names
+    const existingIndex = config.sources.findIndex(s => s.name === source.name);
+    if (existingIndex !== -1) {
+        throw new Error(`Source with name "${source.name}" already exists`);
+    }
+    config.sources.push(source);
+    await saveSyncConfig(config);
+    return config;
+}
+export async function updateSyncSource(name, updates) {
+    const config = await loadSyncConfig();
+    const sourceIndex = config.sources.findIndex(s => s.name === name);
+    if (sourceIndex === -1) {
+        throw new Error(`Source "${name}" not found`);
+    }
+    config.sources[sourceIndex] = {
+        ...config.sources[sourceIndex],
+        ...updates,
+    };
+    await saveSyncConfig(config);
+    return config;
+}
+export async function removeSyncSource(name) {
+    const config = await loadSyncConfig();
+    const sourceIndex = config.sources.findIndex(s => s.name === name);
+    if (sourceIndex === -1) {
+        throw new Error(`Source "${name}" not found`);
+    }
+    config.sources.splice(sourceIndex, 1);
+    await saveSyncConfig(config);
+    return config;
+}
+export function getEnabledSources(config) {
+    return config.sources.filter(s => s.enabled);
+}

package/dist/sync/discover.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Lore - File Discovery (Phase 1)
+ *
+ * Discovers files from configured sources, computes content hashes,
+ * and checks against Supabase for deduplication.
+ *
+ * NO LLM calls in this phase - it's essentially free to run.
+ */
+import type { SyncSource } from './config.js';
+export interface DiscoveredFile {
+    absolutePath: string;
+    relativePath: string;
+    contentHash: string;
+    size: number;
+    modifiedAt: Date;
+    sourceName: string;
+    project: string;
+    existingId?: string;
+}
+export interface DiscoveryResult {
+    source: SyncSource;
+    totalFiles: number;
+    newFiles: DiscoveredFile[];
+    editedFiles: DiscoveredFile[];
+    existingFiles: number;
+    errors: string[];
+}
+export declare function computeFileHash(filePath: string): Promise<string>;
+/**
+ * Simple glob pattern matching for common patterns:
+ * - "**\/*.md" - any .md file in any subdirectory
+ * - "*.md" - .md files in root only
+ * - "**\/*.{md,pdf}" - multiple extensions
+ */
+export declare function matchesGlob(relativePath: string, glob: string): boolean;
+export declare function checkExistingHashes(hashes: string[]): Promise<Set<string>>;
+export declare function discoverSource(source: SyncSource, options?: {
+    onProgress?: (found: number, checked: number) => void;
+}): Promise<DiscoveryResult>;
+export declare function discoverAllSources(sources: SyncSource[], options?: {
+    onSourceStart?: (source: SyncSource) => void;
+    onSourceComplete?: (result: DiscoveryResult) => void;
+}): Promise<DiscoveryResult[]>;
+export declare function summarizeDiscovery(results: DiscoveryResult[]): {
+    totalSources: number;
+    totalFiles: number;
+    newFiles: number;
+    editedFiles: number;
+    existingFiles: number;
+    errors: number;
+};

package/dist/sync/discover.js

@@ -0,0 +1,190 @@
+/**
+ * Lore - File Discovery (Phase 1)
+ *
+ * Discovers files from configured sources, computes content hashes,
+ * and checks against Supabase for deduplication.
+ *
+ * NO LLM calls in this phase - it's essentially free to run.
+ */
+import { readFile, readdir, stat } from 'fs/promises';
+import { createHash } from 'crypto';
+import path from 'path';
+import { existsSync } from 'fs';
+import { expandPath } from './config.js';
+import { getSourcePathMappings, getExistingContentHashes } from '../core/vector-store.js';
+// ============================================================================
+// Hash Computation
+// ============================================================================
+export async function computeFileHash(filePath) {
+    const content = await readFile(filePath);
+    return createHash('sha256').update(content).digest('hex');
+}
+// ============================================================================
+// Glob Matching
+// ============================================================================
+/**
+ * Simple glob pattern matching for common patterns:
+ * - "**\/*.md" - any .md file in any subdirectory
+ * - "*.md" - .md files in root only
+ * - "**\/*.{md,pdf}" - multiple extensions
+ */
+export function matchesGlob(relativePath, glob) {
+    // Handle {ext1,ext2} extension lists
+    const extensionMatch = glob.match(/\.\{([^}]+)\}$/);
+    if (extensionMatch) {
+        const extensions = extensionMatch[1].split(',');
+        const baseGlob = glob.replace(/\.\{[^}]+\}$/, '');
+        return extensions.some(ext => matchesGlob(relativePath, `${baseGlob}.${ext}`));
+    }
+    // Convert glob to regex
+    // Important: Replace glob wildcards BEFORE inserting regex patterns with special chars
+    let pattern = glob
+        .replace(/\?/g, '.') // ? matches single char (do first!)
+        .replace(/\*\*\//g, '{{DOUBLE_STAR_SLASH}}') // Placeholder for **/
+        .replace(/\*\*/g, '{{DOUBLE_STAR}}') // Placeholder for ** (without trailing /)
+        .replace(/\*/g, '[^/]*') // * matches anything except /
+        .replace(/{{DOUBLE_STAR_SLASH}}/g, '(.*\\/)?') // **/ matches any path including empty
+        .replace(/{{DOUBLE_STAR}}/g, '.*'); // ** matches anything including /
+    // Anchor the pattern
+    const regex = new RegExp(`^${pattern}$`);
+    return regex.test(relativePath);
+}
+// ============================================================================
+// File Discovery
+// ============================================================================
+async function discoverFilesRecursive(dir, baseDir, glob, results) {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        const relativePath = path.relative(baseDir, fullPath);
+        if (entry.isDirectory()) {
+            // Skip hidden directories and common non-content dirs
+            if (entry.name.startsWith('.') ||
+                entry.name === 'node_modules' ||
+                entry.name === '__pycache__') {
+                continue;
+            }
+            await discoverFilesRecursive(fullPath, baseDir, glob, results);
+        }
+        else if (entry.isFile()) {
+            // Check if file matches glob pattern
+            if (matchesGlob(relativePath, glob)) {
+                results.push({ path: fullPath, relativePath });
+            }
+        }
+    }
+}
+// ============================================================================
+// Supabase Deduplication (delegates to vector-store)
+// ============================================================================
+export async function checkExistingHashes(hashes) {
+    return getExistingContentHashes('', hashes);
+}
+// ============================================================================
+// Main Discovery Function
+// ============================================================================
+export async function discoverSource(source, options = {}) {
+    const { onProgress } = options;
+    const expandedPath = expandPath(source.path);
+    const result = {
+        source,
+        totalFiles: 0,
+        newFiles: [],
+        editedFiles: [],
+        existingFiles: 0,
+        errors: [],
+    };
+    // Check if source directory exists
+    if (!existsSync(expandedPath)) {
+        result.errors.push(`Directory not found: ${expandedPath}`);
+        return result;
+    }
+    // Discover all matching files
+    const matchingFiles = [];
+    try {
+        await discoverFilesRecursive(expandedPath, expandedPath, source.glob, matchingFiles);
+    }
+    catch (error) {
+        result.errors.push(`Error scanning directory: ${error}`);
+        return result;
+    }
+    result.totalFiles = matchingFiles.length;
+    if (matchingFiles.length === 0) {
+        return result;
+    }
+    // Compute hashes for all files
+    const filesWithHashes = [];
+    for (let i = 0; i < matchingFiles.length; i++) {
+        const file = matchingFiles[i];
+        try {
+            const fileStat = await stat(file.path);
+            const contentHash = await computeFileHash(file.path);
+            filesWithHashes.push({
+                absolutePath: file.path,
+                relativePath: file.relativePath,
+                contentHash,
+                size: fileStat.size,
+                modifiedAt: fileStat.mtime,
+                sourceName: source.name,
+                project: source.project,
+            });
+            onProgress?.(filesWithHashes.length, matchingFiles.length);
+        }
+        catch (error) {
+            result.errors.push(`Error processing ${file.path}: ${error}`);
+        }
+    }
+    // Check which hashes already exist in Supabase (unchanged files)
+    const allHashes = filesWithHashes.map(f => f.contentHash);
+    const existingHashes = await checkExistingHashes(allHashes);
+    // Check which paths already exist in Supabase (for edit detection)
+    const allPaths = filesWithHashes.map(f => f.absolutePath);
+    const pathMappings = await getSourcePathMappings('', allPaths);
+    // Categorize files: existing (unchanged), edited, or new
+    for (const file of filesWithHashes) {
+        if (existingHashes.has(file.contentHash)) {
+            // Content hash matches - file is unchanged
+            result.existingFiles++;
+        }
+        else {
+            // Content is different - check if path exists (edit) or not (new)
+            const existingSource = pathMappings.get(file.absolutePath);
+            if (existingSource) {
+                // Same path, different hash = edit
+                file.existingId = existingSource.id;
+                result.editedFiles.push(file);
+            }
+            else {
+                // New path = new file
+                result.newFiles.push(file);
+            }
+        }
+    }
+    return result;
+}
+export async function discoverAllSources(sources, options = {}) {
+    const { onSourceStart, onSourceComplete } = options;
+    const results = [];
+    for (const source of sources) {
+        if (!source.enabled)
+            continue;
+        onSourceStart?.(source);
+        const result = await discoverSource(source);
+        onSourceComplete?.(result);
+        results.push(result);
+    }
+    return results;
+}
+// ============================================================================
+// Summary Statistics
+// ============================================================================
+export function summarizeDiscovery(results) {
+    return {
+        totalSources: results.length,
+        totalFiles: results.reduce((sum, r) => sum + r.totalFiles, 0),
+        newFiles: results.reduce((sum, r) => sum + r.newFiles.length, 0),
+        editedFiles: results.reduce((sum, r) => sum + r.editedFiles.length, 0),
+        existingFiles: results.reduce((sum, r) => sum + r.existingFiles, 0),
+        errors: results.reduce((sum, r) => sum + r.errors.length, 0),
+    };
+}

package/dist/sync/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Lore - Universal Sync Module
+ *
+ * Two-phase sync system:
+ * 1. Discovery: Find files, compute hashes, check for duplicates
+ * 2. Processing: Claude extracts metadata, generates embeddings, stores
+ */
+export * from './config.js';
+export * from './discover.js';
+export * from './processors.js';
+export * from './process.js';

package/dist/sync/index.js

@@ -0,0 +1,11 @@
+/**
+ * Lore - Universal Sync Module
+ *
+ * Two-phase sync system:
+ * 1. Discovery: Find files, compute hashes, check for duplicates
+ * 2. Processing: Claude extracts metadata, generates embeddings, stores
+ */
+export * from './config.js';
+export * from './discover.js';
+export * from './processors.js';
+export * from './process.js';

package/dist/sync/process.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Lore - Claude Processing (Phase 2)
+ *
+ * Uses Claude to extract metadata from new files:
+ * - title: Descriptive title
+ * - summary: 2-4 sentence summary with key takeaways
+ * - date: ISO date if present
+ * - participants: List of names if present
+ * - content_type: interview|meeting|conversation|document|note|analysis
+ *
+ * Only called for NEW files (not already in Supabase).
+ */
+import type { DiscoveredFile } from './discover.js';
+import type { ContentType } from '../core/types.js';
+import { type ImageMediaType } from './processors.js';
+export interface ExtractedMetadata {
+    title: string;
+    summary: string;
+    date: string | null;
+    participants: string[];
+    content_type: ContentType;
+}
+export interface ProcessedFile {
+    file: DiscoveredFile;
+    metadata: ExtractedMetadata;
+    sourceId: string;
+}
+export interface ProcessResult {
+    processed: ProcessedFile[];
+    errors: Array<{
+        file: DiscoveredFile;
+        error: string;
+    }>;
+}
+export declare function extractMetadata(content: string, filePath: string, options?: {
+    model?: string;
+    image?: {
+        base64: string;
+        mediaType: ImageMediaType;
+    };
+}): Promise<ExtractedMetadata>;
+export declare function processFiles(files: DiscoveredFile[], dataDir: string, options?: {
+    onProgress?: (completed: number, total: number, title: string) => void;
+    model?: string;
+    concurrency?: number;
+    gitPush?: boolean;
+    hookContext?: {
+        mode: 'mcp' | 'cli';
+    };
+}): Promise<ProcessResult>;