@realtimex/realtimex-alchemy 1.0.0

package/dist/api/services/ProcessingEventService.js

@@ -0,0 +1,75 @@
+import { SupabaseService } from './SupabaseService.js';
+import { EventService } from './EventService.js';
+export class ProcessingEventService {
+    static instance;
+    sseEvents = EventService.getInstance();
+    static getInstance() {
+        if (!this.instance) {
+            this.instance = new ProcessingEventService();
+        }
+        return this.instance;
+    }
+    async log(event, supabase) {
+        // 1. Log to SSE for instant feedback in existing terminal
+        this.sseEvents.emit({
+            type: event.eventType,
+            message: event.message,
+            data: event.details
+        });
+        // Mirror to console for developer visibility
+        const icon = event.eventType === 'error' ? '❌' : event.eventType === 'action' ? '⚡' : 'ℹ️';
+        console.log(`[${new Date().toLocaleTimeString()}] ${icon} [${event.agentState}] ${event.message}`);
+        // 2. Persist to Supabase for the new Advanced Terminal
+        const client = supabase || (SupabaseService.isConfigured() ? SupabaseService.getServiceRoleClient() : null);
+        if (client) {
+            try {
+                const userId = event.userId || await this.getFallbackUserId(client);
+                if (userId) {
+                    await client.from('processing_events').insert([{
+                            user_id: userId,
+                            event_type: event.eventType,
+                            agent_state: event.agentState,
+                            message: event.message,
+                            details: event.details || {},
+                            level: event.level || 'info',
+                            duration_ms: event.durationMs || null,
+                            metadata: event.metadata || {},
+                            created_at: new Date().toISOString()
+                        }]);
+                }
+            }
+            catch (error) {
+                console.error('[ProcessingEventService] Failed to persist event:', error);
+            }
+        }
+        else {
+            // Supabase not configured and no client provided - skip persistence
+            // console.warn('[ProcessingEventService] Skipping persistence (no config)');
+        }
+    }
+    async getFallbackUserId(client) {
+        // Safe check for auth
+        if (!client || !client.auth)
+            return null;
+        // Try to get user from client auth context first
+        try {
+            const { data: { user } } = await client.auth.getUser();
+            if (user)
+                return user.id;
+        }
+        catch (e) {
+            // Ignore auth errors during fallback lookup
+        }
+        // Fallback to searching DB if we have service role (unlikely in Zero-Env)
+        if (SupabaseService.isConfigured()) {
+            try {
+                const { data } = await client.from('profiles').select('id').limit(1).maybeSingle();
+                return data?.id || null;
+            }
+            catch (e) {
+                return null;
+            }
+        }
+        return null;
+    }
+}

package/dist/api/services/RouterService.js

@@ -0,0 +1,40 @@
+import axios from 'axios';
+import puppeteer from 'puppeteer';
+import TurndownService from 'turndown';
+import { EventService } from './EventService.js';
+export class RouterService {
+    turndown = new TurndownService();
+    events = EventService.getInstance();
+    async extractContent(url) {
+        this.events.emit({ type: 'router', message: `Attempting Tier 1 Extraction (Axios): ${url.substring(0, 30)}...` });
+        try {
+            // Tier 1: Fast Fetch
+            const response = await axios.get(url, { timeout: 5000 });
+            const html = response.data;
+            const markdown = this.turndown.turndown(html);
+            if (markdown.length > 500) {
+                this.events.emit({ type: 'router', message: `Tier 1 Success (${markdown.length} chars)` });
+                return markdown;
+            }
+        }
+        catch (e) {
+            this.events.emit({ type: 'router', message: `Tier 1 Failed, Falling back to Tier 2...` });
+        }
+        // Tier 2: Puppeteer
+        this.events.emit({ type: 'router', message: `Attempting Tier 2 Extraction (Puppeteer)...` });
+        try {
+            const browser = await puppeteer.launch({ headless: true });
+            const page = await browser.newPage();
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+            const content = await page.content();
+            await browser.close();
+            const markdown = this.turndown.turndown(content);
+            this.events.emit({ type: 'router', message: `Tier 2 Success (${markdown.length} chars)` });
+            return markdown;
+        }
+        catch (e) {
+            this.events.emit({ type: 'router', message: `Tier 2 Failed: ${e.message}` });
+            throw new Error(`Failed to extract content from ${url}`);
+        }
+    }
+}

package/dist/api/services/SupabaseService.js

@@ -0,0 +1,49 @@
+import { createClient } from '@supabase/supabase-js';
+export class SupabaseService {
+    static instance = null;
+    static serviceRoleInstance = null;
+    static getClient() {
+        if (!process.env.SUPABASE_URL || !process.env.SUPABASE_ANON_KEY) {
+            throw new Error('Supabase URL and Anon Key must be configured');
+        }
+        if (!this.instance) {
+            this.instance = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_ANON_KEY, {
+                auth: { persistSession: false }
+            });
+        }
+        return this.instance;
+    }
+    static getServiceRoleClient() {
+        if (!process.env.SUPABASE_URL || !process.env.SUPABASE_SERVICE_ROLE_KEY) {
+            throw new Error('Missing Supabase Config. Please add SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY to your .env file.');
+        }
+        if (!this.serviceRoleInstance) {
+            this.serviceRoleInstance = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY, {
+                auth: {
+                    autoRefreshToken: false,
+                    persistSession: false
+                }
+            });
+        }
+        return this.serviceRoleInstance;
+    }
+    static createClient(url, key, accessToken) {
+        const options = {
+            auth: {
+                persistSession: false,
+                autoRefreshToken: false,
+            }
+        };
+        if (accessToken) {
+            options.global = {
+                headers: {
+                    Authorization: `Bearer ${accessToken}`
+                }
+            };
+        }
+        return createClient(url, key, options);
+    }
+    static isConfigured() {
+        return !!(process.env.SUPABASE_URL && process.env.SUPABASE_ANON_KEY);
+    }
+}

package/dist/api/utils/BrowserPathDetector.js

@@ -0,0 +1,206 @@
+import * as os from 'os';
+import * as path from 'path';
+import * as fs from 'fs';
+import Database from 'better-sqlite3';
+import { v4 as uuidv4 } from 'uuid';
+export class BrowserPathDetector {
+    platform;
+    homeDir;
+    constructor() {
+        this.platform = os.platform();
+        this.homeDir = os.homedir();
+    }
+    /**
+     * Get default browser history paths based on platform
+     */
+    getDefaultPaths() {
+        if (this.platform === 'darwin') {
+            // macOS
+            return {
+                chrome: [
+                    path.join(this.homeDir, 'Library/Application Support/Google/Chrome/Default/History'),
+                    path.join(this.homeDir, 'Library/Application Support/Google/Chrome/Profile 1/History'),
+                ],
+                firefox: [
+                    // Firefox uses profiles with random names
+                    path.join(this.homeDir, 'Library/Application Support/Firefox/Profiles'),
+                ],
+                safari: [
+                    path.join(this.homeDir, 'Library/Safari/History.db'),
+                ],
+                edge: [
+                    path.join(this.homeDir, 'Library/Application Support/Microsoft Edge/Default/History'),
+                ],
+                brave: [
+                    path.join(this.homeDir, 'Library/Application Support/BraveSoftware/Brave-Browser/Default/History'),
+                ],
+                arc: [
+                    path.join(this.homeDir, 'Library/Application Support/Arc/User Data/Default/History'),
+                ],
+                custom: [],
+            };
+        }
+        else if (this.platform === 'win32') {
+            // Windows
+            const appData = process.env.APPDATA || '';
+            const localAppData = process.env.LOCALAPPDATA || '';
+            return {
+                chrome: [
+                    path.join(localAppData, 'Google\\Chrome\\User Data\\Default\\History'),
+                ],
+                firefox: [
+                    path.join(appData, 'Mozilla\\Firefox\\Profiles'),
+                ],
+                safari: [],
+                edge: [
+                    path.join(localAppData, 'Microsoft\\Edge\\User Data\\Default\\History'),
+                ],
+                brave: [
+                    path.join(localAppData, 'BraveSoftware\\Brave-Browser\\User Data\\Default\\History'),
+                ],
+                arc: [],
+                custom: [],
+            };
+        }
+        else {
+            // Linux
+            return {
+                chrome: [
+                    path.join(this.homeDir, '.config/google-chrome/Default/History'),
+                ],
+                firefox: [
+                    path.join(this.homeDir, '.mozilla/firefox'),
+                ],
+                safari: [],
+                edge: [
+                    path.join(this.homeDir, '.config/microsoft-edge/Default/History'),
+                ],
+                brave: [
+                    path.join(this.homeDir, '.config/BraveSoftware/Brave-Browser/Default/History'),
+                ],
+                arc: [],
+                custom: [],
+            };
+        }
+    }
+    /**
+     * Find Firefox profile directories and locate places.sqlite
+     */
+    findFirefoxHistory(profilesDir) {
+        try {
+            if (!fs.existsSync(profilesDir))
+                return null;
+            const profiles = fs.readdirSync(profilesDir);
+            for (const profile of profiles) {
+                if (profile.endsWith('.default') || profile.includes('default-release')) {
+                    const historyPath = path.join(profilesDir, profile, 'places.sqlite');
+                    if (fs.existsSync(historyPath)) {
+                        return historyPath;
+                    }
+                }
+            }
+        }
+        catch (err) {
+            console.error('Error finding Firefox history:', err);
+        }
+        return null;
+    }
+    /**
+     * Validate if a file is a valid SQLite database
+     * Uses copy-to-temp to avoid "database is locked" errors when browser is open
+     */
+    validateSQLitePath(filePath) {
+        let tempPath = null;
+        try {
+            if (!fs.existsSync(filePath)) {
+                return { valid: false, error: 'File does not exist' };
+            }
+            const stats = fs.statSync(filePath);
+            if (!stats.isFile()) {
+                return { valid: false, error: 'Path is not a file' };
+            }
+            // Copy to temp file to avoid database lock issues
+            const tempDir = os.tmpdir();
+            tempPath = path.join(tempDir, `history_validate_${uuidv4()}.db`);
+            fs.copyFileSync(filePath, tempPath);
+            // Try to open the temp copy as SQLite database
+            const db = new Database(tempPath, { readonly: true });
+            // Check if it has expected tables (moz_places for Firefox, urls for Chrome/others)
+            const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all();
+            const tableNames = tables.map((t) => t.name);
+            db.close();
+            const hasHistoryTables = tableNames.includes('urls') ||
+                tableNames.includes('moz_places') ||
+                tableNames.includes('history_items');
+            if (!hasHistoryTables) {
+                return { valid: false, error: 'Not a browser history database' };
+            }
+            return { valid: true };
+        }
+        catch (err) {
+            return { valid: false, error: err.message || 'Invalid SQLite database' };
+        }
+        finally {
+            // Clean up temp file
+            if (tempPath && fs.existsSync(tempPath)) {
+                try {
+                    fs.unlinkSync(tempPath);
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
+            }
+        }
+    }
+    /**
+     * Detect all available browser history paths
+     */
+    detectAll() {
+        const defaultPaths = this.getDefaultPaths();
+        const results = {};
+        for (const [browser, paths] of Object.entries(defaultPaths)) {
+            if (browser === 'custom')
+                continue;
+            const browserType = browser;
+            let foundPath = null;
+            if (browserType === 'firefox' && paths.length > 0) {
+                // Special handling for Firefox
+                foundPath = this.findFirefoxHistory(paths[0]);
+            }
+            else {
+                // Check each potential path
+                for (const p of paths) {
+                    if (fs.existsSync(p)) {
+                        foundPath = p;
+                        break;
+                    }
+                }
+            }
+            if (foundPath) {
+                const validation = this.validateSQLitePath(foundPath);
+                results[browserType] = {
+                    browser: browserType,
+                    path: foundPath,
+                    found: true,
+                    valid: validation.valid,
+                    error: validation.error,
+                };
+            }
+            else {
+                results[browserType] = {
+                    browser: browserType,
+                    path: '',
+                    found: false,
+                };
+            }
+        }
+        return results;
+    }
+    /**
+     * Detect a specific browser's history path
+     */
+    detect(browser) {
+        const all = this.detectAll();
+        return all[browser] || { browser, path: '', found: false };
+    }
+}

package/dist/api/utils/UrlNormalizer.js

@@ -0,0 +1,176 @@
+/**
+ * URL Normalizer - Strips tracking parameters and normalizes URLs for deduplication
+ */
+// Common tracking parameters to strip
+const TRACKING_PARAMS = new Set([
+    // Google Analytics / Ads
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'gclid', 'gclsrc', 'dclid',
+    // Facebook
+    'fbclid', 'fb_action_ids', 'fb_action_types', 'fb_source', 'fb_ref',
+    // Twitter/X
+    'twclid', 's', 't', // Twitter share params
+    // Microsoft/Bing
+    'msclkid',
+    // Email marketing
+    'mc_cid', 'mc_eid', // Mailchimp
+    'oly_enc_id', 'oly_anon_id', // Omeda
+    '_hsenc', '_hsmi', 'hsCtaTracking', // HubSpot
+    // Analytics & tracking
+    '_ga', '_gl', // Google Analytics
+    'ref', 'ref_src', 'ref_url', // Referrer tracking
+    'source', 'src',
+    'campaign', 'medium',
+    // Social sharing
+    'share', 'shared', 'via',
+    // Session/user tracking
+    'sessionid', 'session_id', 'sid',
+    'userid', 'user_id', 'uid',
+    'token', 'auth',
+    // Misc tracking
+    'pk_campaign', 'pk_kwd', 'pk_source', // Piwik/Matomo
+    'zanpid', // Zanox
+    'irclickid', // Impact Radius
+    'affiliate', 'affiliate_id', 'aff_id',
+    'clickid', 'click_id',
+    'trk', 'tracking', 'track',
+]);
+// Parameters that might be tracking but context-dependent (be more careful)
+const SUSPICIOUS_PARAMS = new Set([
+    'id', 'ref', 'source', 'from', 'origin',
+]);
+export class UrlNormalizer {
+    /**
+     * Normalize a URL by stripping tracking parameters and standardizing format
+     */
+    static normalize(url) {
+        try {
+            const parsed = new URL(url);
+            // 1. Lowercase the hostname
+            parsed.hostname = parsed.hostname.toLowerCase();
+            // 2. Remove default ports
+            if (parsed.port === '80' || parsed.port === '443') {
+                parsed.port = '';
+            }
+            // 3. Remove trailing slash from path (except root)
+            if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+                parsed.pathname = parsed.pathname.slice(0, -1);
+            }
+            // 4. Remove fragment/hash
+            parsed.hash = '';
+            // 5. Strip tracking parameters
+            const cleanParams = new URLSearchParams();
+            parsed.searchParams.forEach((value, key) => {
+                const lowerKey = key.toLowerCase();
+                if (!TRACKING_PARAMS.has(lowerKey)) {
+                    cleanParams.set(key, value);
+                }
+            });
+            // 6. Sort remaining params for consistent ordering
+            const sortedParams = new URLSearchParams();
+            const keys = Array.from(cleanParams.keys()).sort();
+            for (const key of keys) {
+                sortedParams.set(key, cleanParams.get(key));
+            }
+            parsed.search = sortedParams.toString() ? `?${sortedParams.toString()}` : '';
+            return parsed.toString();
+        }
+        catch {
+            // If URL parsing fails, return original
+            return url;
+        }
+    }
+    /**
+     * Extract the canonical URL (domain + path, no params)
+     * Useful for aggressive deduplication
+     */
+    static getCanonical(url) {
+        try {
+            const parsed = new URL(url);
+            parsed.hostname = parsed.hostname.toLowerCase();
+            parsed.search = '';
+            parsed.hash = '';
+            // Remove trailing slash
+            if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+                parsed.pathname = parsed.pathname.slice(0, -1);
+            }
+            return `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
+        }
+        catch {
+            return url;
+        }
+    }
+    /**
+     * Get a fingerprint for deduplication (hash of normalized URL)
+     */
+    static getFingerprint(url) {
+        const normalized = this.normalize(url);
+        // Simple hash for dedup - not cryptographic, just for comparison
+        let hash = 0;
+        for (let i = 0; i < normalized.length; i++) {
+            const char = normalized.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32bit integer
+        }
+        return hash.toString(36);
+    }
+    /**
+     * Check if two URLs are effectively the same (after normalization)
+     */
+    static areSameUrl(url1, url2) {
+        return this.normalize(url1) === this.normalize(url2);
+    }
+    /**
+     * Extract domain from URL
+     */
+    static getDomain(url) {
+        try {
+            const parsed = new URL(url);
+            return parsed.hostname.toLowerCase();
+        }
+        catch {
+            return '';
+        }
+    }
+    /**
+     * Check if URL matches common non-content patterns
+     */
+    static isLikelyNonContent(url) {
+        try {
+            const parsed = new URL(url);
+            const path = parsed.pathname.toLowerCase();
+            // Common non-content paths
+            const nonContentPaths = [
+                '/login', '/signin', '/sign-in', '/signup', '/sign-up', '/register',
+                '/logout', '/signout', '/sign-out',
+                '/auth', '/oauth', '/callback', '/sso',
+                '/cart', '/checkout', '/basket', '/bag',
+                '/account', '/profile', '/settings', '/preferences',
+                '/admin', '/dashboard', '/manage',
+                '/api/', '/graphql', '/_next/', '/__',
+                '/search', '/results',
+            ];
+            for (const nonContent of nonContentPaths) {
+                if (path.startsWith(nonContent) || path.includes(nonContent)) {
+                    return true;
+                }
+            }
+            // Non-content file extensions
+            const nonContentExtensions = [
+                '.json', '.xml', '.js', '.css', '.map',
+                '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.webp',
+                '.woff', '.woff2', '.ttf', '.eot',
+                '.pdf', '.zip', '.gz',
+            ];
+            for (const ext of nonContentExtensions) {
+                if (path.endsWith(ext)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }
+}

package/dist/api/utils/contentCleaner.js

@@ -0,0 +1,114 @@
+export class ContentCleaner {
+    /**
+     * Cleans email body by removing noise, quoted replies, and footers.
+     * optimized for LLM processing.
+     */
+    static cleanEmailBody(text) {
+        if (!text)
+            return "";
+        const originalText = text;
+        // 1. Detect if content is actually HTML
+        const isHtml = /<[a-z][\s\S]*>/i.test(text);
+        if (isHtml) {
+            // Lightweight HTML -> Markdown Conversion
+            // Structure: <br>, <p> -> Newlines
+            text = text.replace(/<br\s*\/?>/gi, '\n');
+            text = text.replace(/<\/p>/gi, '\n\n');
+            text = text.replace(/<p.*?>/gi, '');
+            // Structure: Headers <h1>-<h6> -> # Title
+            text = text.replace(/<h[1-6].*?>(.*?)<\/h[1-6]>/gsi, (match, p1) => `\n# ${p1}\n`);
+            // Structure: Lists <li> -> - Item
+            text = text.replace(/<li.*?>(.*?)<\/li>/gsi, (match, p1) => `\n- ${p1}`);
+            text = text.replace(/<ul.*?>/gi, '');
+            text = text.replace(/<\/ul>/gi, '\n');
+            // Links: <a href=\"...\">text</a> -> [text](href)
+            text = text.replace(/<a\s+(?:[^>]*?\s+)?href=\"([^\"]*)\"[^>]*>(.*?)<\/a>/gsi, (match, href, content) => `[${content}](${href})`);
+            // Images: <img src=\"...\" alt=\"...\"> -> ![alt](src)
+            text = text.replace(/<img\s+(?:[^>]*?\s+)?src=\"([^\"]*)\"(?:[^>]*?\s+)?alt=\"([^\"]*)\"[^>]*>/gsi, (match, src, alt) => `![${alt}](${src})`);
+            // Style/Script removal (strictly remove content)
+            text = text.replace(/<script.*?>.*?<\/script>/gsi, '');
+            text = text.replace(/<style.*?>.*?<\/style>/gsi, '');
+            // Final Strip of remaining tags
+            text = text.replace(/<[^>]+>/g, ' ');
+            // Entity decoding (Basic)
+            text = text.replace(/&nbsp;/gi, ' ');
+            text = text.replace(/&amp;/gi, '&');
+            text = text.replace(/&lt;/gi, '<');
+            text = text.replace(/&gt;/gi, '>');
+            text = text.replace(/&quot;/gi, '"');
+            text = text.replace(/&#39;/gi, "'");
+        }
+        const lines = text.split('\n');
+        const cleanedLines = [];
+        // Patterns that usually mark the START of a reply chain or a generic footer
+        const truncationPatterns = [
+            /^On .* wrote:$/i,
+            /^From: .* <.*>$/i,
+            /^-----Original Message-----$/i,
+            /^________________________________$/i,
+            /^Sent from my iPhone$/i,
+            /^Sent from my Android$/i,
+            /^Get Outlook for/i,
+            /^--$/ // Standard signature separator
+        ];
+        // Patterns for lines that should be stripped but NOT truncate the whole email
+        const noisePatterns = [
+            /view in browser/i,
+            /click here to view/i,
+            /legal notice/i,
+            /all rights reserved/i,
+            /privacy policy/i,
+            /terms of service/i,
+            /unsubscribe/i
+        ];
+        for (let line of lines) {
+            let lineStripped = line.trim();
+            if (!lineStripped) {
+                cleanedLines.push("");
+                continue;
+            }
+            // 2. Quoted text removal (lines starting with >)
+            if (lineStripped.startsWith('>')) {
+                continue;
+            }
+            // 3. Truncation check: If we hit a reply header, we stop entirely
+            let shouldTruncate = false;
+            for (const pattern of truncationPatterns) {
+                if (pattern.test(lineStripped)) {
+                    shouldTruncate = true;
+                    break;
+                }
+            }
+            if (shouldTruncate)
+                break;
+            // 4. Noise check: Strip boilerplate lines
+            let isNoise = false;
+            if (lineStripped.length < 100) {
+                for (const pattern of noisePatterns) {
+                    if (pattern.test(lineStripped)) {
+                        isNoise = true;
+                        break;
+                    }
+                }
+            }
+            if (isNoise)
+                continue;
+            cleanedLines.push(line);
+        }
+        // Reassemble
+        text = cleanedLines.join('\n');
+        // Collapse whitespace
+        text = text.replace(/\n{3,}/g, '\n\n');
+        text = text.replace(/[ \t]{2,}/g, ' ');
+        // Safety Fallback: If cleaning stripped too much, return original text truncated
+        if (text.trim().length < 20 && originalText.trim().length > 20) {
+            return originalText.substring(0, 3000).trim();
+        }
+        // Sanitize LLM Special Tokens
+        text = text.replace(/<\|/g, '< |');
+        text = text.replace(/\|>/g, '| >');
+        text = text.replace(/\[INST\]/gi, '[ INST ]');
+        text = text.replace(/\[\/INST\]/gi, '[ /INST ]');
+        return text.trim();
+    }
+}