@realtimex/realtimex-alchemy 1.0.0

package/dist/api/services/AlchemistService.js

@@ -0,0 +1,241 @@
+import OpenAI from 'openai';
+import { ProcessingEventService } from './ProcessingEventService.js';
+import { RouterService } from './RouterService.js';
+export class AlchemistService {
+    processingEvents;
+    router;
+    constructor() {
+        this.processingEvents = ProcessingEventService.getInstance();
+        this.router = new RouterService();
+    }
+    async process(entries, settings, supabase, userId, syncStartTime) {
+        if (!entries || entries.length === 0)
+            return;
+        // Track stats for completion event
+        const stats = {
+            total: entries.length,
+            signals: 0,
+            skipped: 0,
+            errors: 0
+        };
+        // Initialize OpenAI client with user settings
+        const config = {
+            baseUrl: settings.llm_base_url || 'http://localhost:11434',
+            model: settings.llm_model_name || 'llama3',
+            apiKey: settings.llm_api_key || 'ollama'
+        };
+        console.log('[AlchemistService] LLM Config:', {
+            baseUrl: config.baseUrl,
+            model: config.model,
+            hasApiKey: !!config.apiKey
+        });
+        const client = new OpenAI({
+            baseURL: config.baseUrl.endsWith('/v1') ? config.baseUrl : `${config.baseUrl}/v1`,
+            apiKey: config.apiKey,
+            dangerouslyAllowBrowser: true
+        });
+        for (const entry of entries) {
+            // Emit: Reading
+            await this.processingEvents.log({
+                eventType: 'analysis',
+                agentState: 'Reading',
+                message: `Reading content from: ${entry.url}`,
+                level: 'info',
+                userId
+            }, supabase);
+            try {
+                // 2. Extract Content via RouterService (Tier 1 → Tier 2 fallback)
+                let content = '';
+                try {
+                    const markdown = await this.router.extractContent(entry.url);
+                    if (markdown && markdown.length > 100) {
+                        // Truncate to avoid token limits (keep ~8000 chars)
+                        const truncated = markdown.length > 8000 ? markdown.substring(0, 8000) + '...' : markdown;
+                        content = `Page Title: ${entry.title}\nContent: ${truncated}`;
+                    }
+                    else {
+                        content = `Page Title: ${entry.title} (Content unavailable or too short)`;
+                    }
+                }
+                catch (scrapeError) {
+                    console.warn(`Extraction failed for ${entry.url}:`, scrapeError.message);
+                    content = `Page Title: ${entry.title}. URL: ${entry.url} (Extraction failed)`;
+                }
+                // Emit: Analyzing
+                const startAnalysis = Date.now();
+                await this.processingEvents.log({
+                    eventType: 'analysis',
+                    agentState: 'Thinking',
+                    message: `Analyzing relevance of: ${entry.title}`,
+                    level: 'info',
+                    userId
+                }, supabase);
+                // 3. LLM Analysis
+                const response = await this.analyzeContent(client, content, entry.url, config.model);
+                const duration = Date.now() - startAnalysis;
+                if (response.relevant) {
+                    // Emit: Signal Found
+                    await this.processingEvents.log({
+                        eventType: 'action',
+                        agentState: 'Signal',
+                        message: `Found signal: ${response.summary} (${response.score}%)`,
+                        level: 'info',
+                        metadata: response,
+                        durationMs: duration,
+                        userId
+                    }, supabase);
+                    // 4. Save Signal
+                    console.log('[AlchemistService] Saving signal to database...');
+                    const { error: insertError } = await supabase.from('signals').insert([{
+                            user_id: userId,
+                            url: entry.url,
+                            title: entry.title,
+                            score: response.score,
+                            summary: response.summary,
+                            category: response.category,
+                            entities: response.entities,
+                            tags: response.tags,
+                            content: content
+                        }]);
+                    if (insertError) {
+                        console.error('[AlchemistService] Insert error:', insertError);
+                        stats.errors++;
+                    }
+                    else {
+                        console.log('[AlchemistService] Signal saved successfully');
+                        stats.signals++;
+                    }
+                }
+                else {
+                    // Emit: Skipped
+                    await this.processingEvents.log({
+                        eventType: 'info',
+                        agentState: 'Skipped',
+                        message: `Irrelevant content (${response.score}%): ${entry.title}`,
+                        level: 'debug',
+                        durationMs: duration,
+                        userId
+                    }, supabase);
+                    stats.skipped++;
+                }
+            }
+            catch (error) {
+                console.error(`Error analyzing ${entry.url}:`, error);
+                await this.processingEvents.log({
+                    eventType: 'error',
+                    agentState: 'Error',
+                    message: `Analysis failed for ${entry.title}: ${error.message}`,
+                    level: 'error',
+                    userId
+                }, supabase);
+                stats.errors++;
+            }
+        }
+        // Emit: Sync Completed
+        const totalDuration = syncStartTime ? Date.now() - syncStartTime : 0;
+        await this.processingEvents.log({
+            eventType: 'info',
+            agentState: 'Completed',
+            message: `Sync completed: ${stats.signals} signals found, ${stats.skipped} skipped, ${stats.errors} errors`,
+            level: 'info',
+            durationMs: totalDuration,
+            metadata: {
+                is_completion: true,
+                total_urls: stats.total,
+                signals_found: stats.signals,
+                skipped: stats.skipped,
+                errors: stats.errors,
+                duration_seconds: Math.round(totalDuration / 1000)
+            },
+            userId
+        }, supabase);
+    }
+    async analyzeContent(client, content, url, model) {
+        const prompt = `
+        Act as "The Alchemist", a high-level intelligence analyst.
+        Analyze the following article value.
+        
+        Input:
+        URL: ${url}
+        Content: ${content}
+        
+        CRITICAL SCORING: 
+        High Score (80-100): Original research, concrete data points, contrarian insights, deep technical details, official documentation.
+        Medium Score (50-79): Decent summaries, useful aggregate news, tutorials, reference material, software documentation.
+        Low Score (0-49): Marketing fluff, SEO clickbait, generic listicles, navigation menus only, login pages, or site footers.
+        
+        Return STRICT JSON:
+        {
+            "score": number (0-100),
+            "category": string (one of: AI & ML, Business, Politics, Technology, Finance, Crypto, Science, Other),
+            "summary": string (1-sentence concise gist),
+            "entities": string[],
+            "tags": string[] (3-5 relevant topic tags for categorization),
+            "relevant": boolean (true if score > 50)
+        }
+        `;
+        const completion = await client.chat.completions.create({
+            model: model,
+            messages: [
+                { role: 'system', content: 'You are a precise analyzer. Return ONLY valid JSON, no other text.' },
+                { role: 'user', content: prompt }
+            ]
+            // Note: response_format removed for compatibility with LM Studio and other local LLMs
+        });
+        const raw = completion.choices[0].message.content || '{}';
+        return this.parseRobustJSON(raw);
+    }
+    // For manual signal test from UI
+    async analyzeSignal(text, config) {
+        const client = new OpenAI({
+            baseURL: config.baseUrl.endsWith('/v1') ? config.baseUrl : `${config.baseUrl}/v1`,
+            apiKey: config.apiKey || 'ollama',
+            dangerouslyAllowBrowser: true
+        });
+        return this.analyzeContent(client, text, 'Manual Text Input', config.model);
+    }
+    async testConnection(config) {
+        try {
+            const client = new OpenAI({
+                apiKey: config.apiKey || 'ollama',
+                baseURL: config.baseUrl.endsWith('/v1') ? config.baseUrl : `${config.baseUrl}/v1`
+            });
+            const completion = await client.chat.completions.create({
+                messages: [{ role: 'user', content: 'Say "OK"' }],
+                model: config.model,
+                max_tokens: 5
+            });
+            return {
+                success: true,
+                message: `Connection successful!`,
+                model: config.model
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                message: error.message || 'Connection failed'
+            };
+        }
+    }
+    parseRobustJSON(input) {
+        try {
+            const jsonMatch = input.match(/\{[\s\S]*\}/);
+            const jsonStr = jsonMatch ? jsonMatch[0] : input;
+            const cleaned = jsonStr
+                .replace(/<\|[\s\S]*?\|>/g, '')
+                .replace(/```json/g, '')
+                .replace(/```/g, '')
+                .trim();
+            const result = JSON.parse(cleaned);
+            // Ensure tags exists for backward compatibility if LLM fails to return it
+            if (!result.tags)
+                result.tags = result.entities || [];
+            return result;
+        }
+        catch (e) {
+            console.error("JSON Parse Error:", e, input);
+            return { score: 0, summary: 'Failed to parse', category: 'Error', entities: [], tags: [], relevant: false };
+        }
+    }
+}

package/dist/api/services/EventService.js

@@ -0,0 +1,53 @@
+import { v4 as uuidv4 } from 'uuid';
+export class EventService {
+    static instance;
+    clients = [];
+    eventHistory = [];
+    maxHistory = 50;
+    constructor() { }
+    static getInstance() {
+        if (!EventService.instance) {
+            EventService.instance = new EventService();
+        }
+        return EventService.instance;
+    }
+    addClient(res) {
+        this.clients.push(res);
+        // Send connection success
+        this.sendEvent(res, {
+            type: 'system',
+            message: 'Alchemy Engine Connected',
+            timestamp: new Date().toISOString()
+        });
+        // Send history
+        if (this.eventHistory.length > 0) {
+            this.sendEvent(res, {
+                type: 'history',
+                message: 'Event History',
+                data: this.eventHistory
+            });
+        }
+    }
+    removeClient(res) {
+        this.clients = this.clients.filter(client => client !== res);
+    }
+    emit(event) {
+        const fullEvent = {
+            id: uuidv4(),
+            timestamp: new Date().toISOString(),
+            ...event
+        };
+        // Add to history
+        this.eventHistory.unshift(fullEvent);
+        if (this.eventHistory.length > this.maxHistory) {
+            this.eventHistory.pop();
+        }
+        // Broadcast to all clients
+        this.clients.forEach(client => {
+            this.sendEvent(client, fullEvent);
+        });
+    }
+    sendEvent(res, event) {
+        res.write(`data: ${JSON.stringify(event)}\n\n`);
+    }
+}

package/dist/api/services/LibrarianService.js

@@ -0,0 +1,72 @@
+import { SupabaseService } from './SupabaseService.js';
+export class LibrarianService {
+    async getSignals(supabase) {
+        // Use provided client or try to get service role client (which may fail if no env)
+        const client = supabase || (SupabaseService.isConfigured() ? SupabaseService.getServiceRoleClient() : null);
+        if (!client) {
+            console.warn('[Librarian] Supabase not configured, returning empty signals');
+            return [];
+        }
+        const { data, error } = await client
+            .from('signals')
+            .select('*')
+            .order('created_at', { ascending: false });
+        if (error) {
+            console.error('[Librarian] Error fetching signals:', error);
+            return [];
+        }
+        return (data || []).map((s) => ({
+            id: s.id,
+            url: s.url,
+            title: s.title,
+            score: s.score,
+            summary: s.summary,
+            date: s.created_at,
+            category: s.category,
+            entities: s.entities,
+            content: s.content
+        }));
+    }
+    async saveSignal(metadata, content, supabase) {
+        const client = supabase || (SupabaseService.isConfigured() ? SupabaseService.getServiceRoleClient() : null);
+        if (!client) {
+            console.warn('[Librarian] Supabase not configured, signal not saved to cloud');
+            return;
+        }
+        // For saving, we need the user_id.
+        // If the client is scoped, we can get it from auth.
+        const { data: { user } } = await client.auth.getUser();
+        // If no user (e.g. using service role without user context), fallback or skip
+        // But for Zero-Env, we expect a scoped client.
+        const { error } = await client
+            .from('signals')
+            .upsert({
+            url: metadata.url,
+            title: metadata.title,
+            score: metadata.score,
+            summary: metadata.summary,
+            category: metadata.category,
+            entities: metadata.entities || [],
+            content: content,
+            user_id: user?.id,
+            updated_at: new Date().toISOString()
+        }, {
+            onConflict: 'user_id, url' // Column names separated by comma and space
+        });
+        if (error) {
+            console.error('[Librarian] Error saving signal:', error);
+        }
+        else {
+            console.log('[Librarian] Signal saved/updated successfully for URL:', metadata.url);
+        }
+    }
+    async getSystemUserId() {
+        // Legacy fallback
+        if (SupabaseService.isConfigured()) {
+            const supabase = SupabaseService.getServiceRoleClient();
+            const { data } = await supabase.from('profiles').select('id').limit(1).maybeSingle();
+            return data?.id || null;
+        }
+        return null;
+    }
+}

package/dist/api/services/MinerService.js

@@ -0,0 +1,314 @@
+import fs from 'fs/promises';
+import path from 'path';
+import sqlite3 from 'better-sqlite3';
+import { v4 as uuidv4 } from 'uuid';
+import { CONFIG } from '../config/index.js';
+import { ProcessingEventService } from './ProcessingEventService.js';
+import { UrlNormalizer } from '../utils/UrlNormalizer.js';
+export class MinerService {
+    processingEvents = ProcessingEventService.getInstance();
+    blacklist = [
+        'google.com/search',
+        'localhost:',
+        '127.0.0.1',
+        'facebook.com',
+        'twitter.com',
+        'instagram.com',
+        'linkedin.com/feed',
+    ];
+    async mineHistory(settings, supabase) {
+        // Extract enabled sources from settings
+        console.log('[MinerService] settings.custom_browser_paths:', JSON.stringify(settings.custom_browser_paths, null, 2));
+        const enabledSources = (settings.custom_browser_paths || []).filter((s) => {
+            console.log(`[MinerService] Checking source: ${s.path}, enabled: ${s.enabled} (${typeof s.enabled})`);
+            return s.enabled === true || s.enabled === 'true';
+        });
+        console.log('[MinerService] enabledSources:', enabledSources.length);
+        if (enabledSources.length === 0) {
+            this.processingEvents.log({
+                eventType: 'warning',
+                agentState: 'Idle',
+                message: 'No active browser sources configured.'
+            }, supabase);
+            return [];
+        }
+        const maxUrlsPerSync = settings.max_urls_per_sync || CONFIG.MAX_HISTORY_ITEMS;
+        const syncMode = settings.sync_mode || 'incremental';
+        const syncFromDate = settings.sync_from_date;
+        let allEntries = [];
+        // Verify user content
+        const { data: { user } } = await supabase.auth.getUser();
+        const userId = user?.id;
+        for (const source of enabledSources) {
+            const sourceStart = Date.now();
+            await this.processingEvents.log({
+                eventType: 'info',
+                agentState: 'Mining',
+                message: `Mining source: ${source.label} (${source.browser})...`,
+                level: 'info',
+                userId
+            }, supabase);
+            try {
+                const entries = await this.mineSource(source, supabase, userId, maxUrlsPerSync, settings);
+                allEntries = [...allEntries, ...entries];
+                const duration = Date.now() - sourceStart;
+                await this.processingEvents.log({
+                    eventType: 'info',
+                    agentState: 'Mining',
+                    message: `Found ${entries.length} URLs from ${source.label}`,
+                    details: { source: source.label, count: entries.length },
+                    level: 'info',
+                    durationMs: duration,
+                    userId
+                }, supabase);
+            }
+            catch (error) {
+                console.error(`Error mining ${source.label}:`, error);
+                const duration = Date.now() - sourceStart;
+                await this.processingEvents.log({
+                    eventType: 'error',
+                    agentState: 'Mining',
+                    message: `Failed to mine ${source.label}: ${error.message}`,
+                    details: { source: source.label, error: error.message },
+                    level: 'error',
+                    durationMs: duration,
+                    userId
+                }, supabase);
+            }
+        }
+        // Cross-source deduplication: if same URL appears in multiple browsers, keep only one
+        const seenUrls = new Set();
+        const uniqueEntries = [];
+        let crossSourceDupes = 0;
+        for (const entry of allEntries) {
+            if (!seenUrls.has(entry.url)) {
+                seenUrls.add(entry.url);
+                uniqueEntries.push(entry);
+            }
+            else {
+                crossSourceDupes++;
+            }
+        }
+        if (crossSourceDupes > 0) {
+            console.log(`[MinerService] Cross-source dedup: removed ${crossSourceDupes} duplicates`);
+            await this.processingEvents.log({
+                eventType: 'info',
+                agentState: 'Mining',
+                message: `Deduplication: ${crossSourceDupes} cross-browser duplicates removed`,
+                level: 'debug',
+                userId
+            }, supabase);
+        }
+        // Auto-clear sync_start_date after successful sync (like email-automator)
+        // This ensures next sync automatically uses incremental mode
+        if (userId && settings.sync_start_date) {
+            console.log('[MinerService] Auto-clearing sync_start_date after successful sync');
+            await supabase
+                .from('alchemy_settings')
+                .update({ sync_start_date: null })
+                .eq('user_id', userId);
+        }
+        return uniqueEntries;
+    }
+    async mineSource(source, supabase, userId, maxItems, settings) {
+        const historyPath = source.path;
+        if (!historyPath)
+            return [];
+        // Determine starting timestamp based on simplified sync logic
+        // Priority: sync_start_date > last_sync_checkpoint > checkpoint from history_checkpoints
+        let startTime;
+        if (settings?.sync_start_date) {
+            // User has set a manual sync start date - use it
+            startTime = new Date(settings.sync_start_date).getTime();
+            console.log(`[MinerService] Using sync_start_date: ${new Date(startTime).toISOString()}`);
+        }
+        else if (settings?.last_sync_checkpoint) {
+            // Use the last checkpoint from settings
+            startTime = new Date(settings.last_sync_checkpoint).getTime();
+            console.log(`[MinerService] Using last_sync_checkpoint: ${new Date(startTime).toISOString()}`);
+        }
+        else {
+            // Fall back to browser-specific checkpoint
+            startTime = await this.getCheckpoint(source.path, supabase);
+            console.log(`[MinerService] Using browser checkpoint: ${new Date(startTime).toISOString()}`);
+        }
+        try {
+            await fs.access(historyPath);
+            // Bypass SQLite lock by copying
+            const tempId = uuidv4();
+            const tempPath = path.join(CONFIG.DATA_DIR, `history_${tempId}_temp.db`);
+            await fs.mkdir(CONFIG.DATA_DIR, { recursive: true });
+            await fs.copyFile(historyPath, tempPath);
+            const db = new sqlite3(tempPath, { readonly: true });
+            // Adjust query based on browser type
+            let query = '';
+            let queryParamTime = 0;
+            // Normalize checkpoint time to browser-specific format for querying
+            queryParamTime = this.fromUnixMs(startTime, source.browser);
+            console.log(`[MinerService] Browser: ${source.browser}`);
+            console.log(`[MinerService] Start Time (Unix Ms): ${startTime}`);
+            console.log(`[MinerService] Query Param Time (Browser Format): ${queryParamTime}`);
+            if (source.browser === 'firefox') {
+                query = `
+                    SELECT url, title, visit_count, last_visit_date as last_visit_time
+                    FROM moz_places
+                    WHERE last_visit_date > ? AND url LIKE 'http%'
+                    ORDER BY last_visit_date DESC
+                    LIMIT ?
+                `;
+            }
+            else {
+                // Chrome, Edge, Brave, Arc, Safari (usually)
+                if (source.browser === 'safari') {
+                    // Safari uses Core Data timestamp (seconds since 2001-01-01)
+                    // Not fully implemented yet, but keeping placeholder
+                    query = `
+                        SELECT url, title, visit_count, last_visit_time 
+                        FROM history_items 
+                        WHERE last_visit_time > ? 
+                        ORDER BY last_visit_time DESC 
+                        LIMIT ?
+                    `;
+                }
+                else {
+                    query = `
+                        SELECT url, title, visit_count, last_visit_time 
+                        FROM urls 
+                        WHERE last_visit_time > ?
+                        ORDER BY last_visit_time DESC 
+                        LIMIT ?
+                    `;
+                }
+            }
+            let rows = [];
+            const limit = maxItems || CONFIG.MAX_HISTORY_ITEMS;
+            try {
+                rows = db.prepare(query).all(queryParamTime, limit);
+            }
+            catch (sqlErr) {
+                console.warn(`SQL Error for ${source.label}:`, sqlErr);
+                // Fallback or skip
+            }
+            db.close();
+            await fs.unlink(tempPath);
+            // Track seen normalized URLs for deduplication within this batch
+            const seenUrls = new Set();
+            let skippedDuplicates = 0;
+            let skippedNonContent = 0;
+            let skippedBlacklist = 0;
+            const entries = rows
+                .filter(row => {
+                if (!row.url)
+                    return false;
+                // 1. Blacklist check (domain-level)
+                if (this.blacklist.some(b => row.url.includes(b))) {
+                    skippedBlacklist++;
+                    return false;
+                }
+                // 2. Non-content URL check (login pages, APIs, assets, etc.)
+                if (UrlNormalizer.isLikelyNonContent(row.url)) {
+                    skippedNonContent++;
+                    return false;
+                }
+                // 3. Normalize and deduplicate
+                const normalizedUrl = UrlNormalizer.normalize(row.url);
+                if (seenUrls.has(normalizedUrl)) {
+                    skippedDuplicates++;
+                    return false;
+                }
+                seenUrls.add(normalizedUrl);
+                return true;
+            })
+                .map(row => ({
+                id: uuidv4(),
+                url: UrlNormalizer.normalize(row.url), // Store normalized URL
+                title: row.title || 'Untitled',
+                visit_count: row.visit_count || 1,
+                // Normalize back to Unix Ms for internal storage/usage
+                last_visit_time: this.toUnixMs(row.last_visit_time, source.browser),
+                browser: source.browser
+            }));
+            // Log filtering stats
+            if (skippedDuplicates > 0 || skippedNonContent > 0 || skippedBlacklist > 0) {
+                console.log(`[MinerService] URL Filtering: ${skippedDuplicates} duplicates, ${skippedNonContent} non-content, ${skippedBlacklist} blacklisted`);
+            }
+            if (entries.length > 0) {
+                const newestTime = Math.max(...entries.map(e => e.last_visit_time));
+                await this.saveCheckpoint(source.path, newestTime, supabase, userId);
+                // Also update last_sync_checkpoint in settings for global tracking
+                if (userId && settings) {
+                    await supabase
+                        .from('alchemy_settings')
+                        .update({ last_sync_checkpoint: new Date(newestTime).toISOString() })
+                        .eq('user_id', userId);
+                }
+            }
+            return entries;
+        }
+        catch (error) {
+            throw error;
+        }
+    }
+    toUnixMs(timestamp, browser) {
+        if (!timestamp)
+            return Date.now();
+        if (browser === 'firefox') {
+            // Firefox: Microseconds -> Milliseconds
+            return Math.floor(timestamp / 1000);
+        }
+        else if (browser === 'safari') {
+            // Safari: Seconds since 2001-01-01 -> Unix Ms
+            // 978307200 is seconds between 1970 and 2001
+            return Math.floor((timestamp + 978307200) * 1000);
+        }
+        else {
+            // Chrome/Webkit: Microseconds since 1601-01-01
+            // Difference between 1601 and 1970 in ms: 11644473600000
+            return Math.floor((timestamp / 1000) - 11644473600000);
+        }
+    }
+    fromUnixMs(unixMs, browser) {
+        if (!unixMs)
+            return 0; // Default to beginning of time
+        if (browser === 'firefox') {
+            return unixMs * 1000;
+        }
+        else if (browser === 'safari') {
+            return (unixMs / 1000) - 978307200;
+        }
+        else {
+            // Chrome/Webkit
+            return (unixMs + 11644473600000) * 1000;
+        }
+    }
+    async getCheckpoint(browser, supabase) {
+        const { data } = await supabase
+            .from('history_checkpoints')
+            .select('last_visit_time')
+            .eq('browser', browser)
+            .limit(1)
+            .maybeSingle();
+        let checkpoint = data?.last_visit_time || 0;
+        // Sanity Check: If checkpoint is from the "future" (likely old raw Chrome timestamp)
+        // Chrome timestamps (microseconds) are ~10^16
+        // Unix Ms timestamps are ~10^12
+        // If checkpoint > 3000000000000 (Year 2065), it's definitely invalid/raw format.
+        if (checkpoint > 3000000000000) {
+            console.warn(`[MinerService] Checkpoint ${checkpoint} looks invalid (too large/raw format). Resetting to 0.`);
+            return 0;
+        }
+        return checkpoint;
+    }
+    async saveCheckpoint(browser, time, supabase, userId) {
+        if (!userId)
+            return;
+        await supabase
+            .from('history_checkpoints')
+            .upsert({
+            user_id: userId,
+            browser,
+            last_visit_time: time,
+            updated_at: new Date().toISOString()
+        }, { onConflict: 'user_id,browser' });
+    }
+}