@realtimex/email-automator 2.1.0

package/api/src/services/scheduler.ts

@@ -0,0 +1,255 @@
+import { SupabaseClient } from '@supabase/supabase-js';
+import { config } from '../config/index.js';
+import { createLogger } from '../utils/logger.js';
+import { EmailProcessorService } from './processor.js';
+import { getServerSupabase } from './supabase.js';
+
+const logger = createLogger('Scheduler');
+
+interface ScheduledJob {
+    id: string;
+    name: string;
+    interval: number;
+    lastRun: Date | null;
+    isRunning: boolean;
+    timer: NodeJS.Timeout | null;
+}
+
+class SyncScheduler {
+    private jobs: Map<string, ScheduledJob> = new Map();
+    private supabase: SupabaseClient | null = null;
+
+    constructor() {
+        this.supabase = getServerSupabase();
+    }
+
+    async start(): Promise<void> {
+        if (!this.supabase) {
+            logger.warn('Supabase not configured, scheduler disabled');
+            return;
+        }
+
+        logger.info('Starting sync scheduler');
+
+        // Schedule periodic sync for all active accounts
+        this.scheduleGlobalSync();
+
+        // Schedule cleanup job
+        this.scheduleCleanup();
+    }
+
+    stop(): void {
+        logger.info('Stopping sync scheduler');
+        for (const job of this.jobs.values()) {
+            if (job.timer) {
+                clearInterval(job.timer);
+            }
+        }
+        this.jobs.clear();
+    }
+
+    private scheduleGlobalSync(): void {
+        const jobId = 'global-sync';
+        const interval = config.processing.syncIntervalMs;
+
+        const job: ScheduledJob = {
+            id: jobId,
+            name: 'Global Email Sync',
+            interval,
+            lastRun: null,
+            isRunning: false,
+            timer: null,
+        };
+
+        job.timer = setInterval(async () => {
+            if (job.isRunning) {
+                logger.debug('Global sync already running, skipping');
+                return;
+            }
+
+            job.isRunning = true;
+            try {
+                await this.runGlobalSync();
+                job.lastRun = new Date();
+            } catch (error) {
+                logger.error('Global sync failed', error);
+            } finally {
+                job.isRunning = false;
+            }
+        }, interval);
+
+        this.jobs.set(jobId, job);
+        logger.info(`Scheduled global sync every ${interval / 1000}s`);
+    }
+
+    private async runGlobalSync(): Promise<void> {
+        if (!this.supabase) return;
+
+        // Get all active accounts with their user settings
+        const { data: accounts, error } = await this.supabase
+            .from('email_accounts')
+            .select(`
+                id,
+                user_id,
+                provider,
+                is_active
+            `)
+            .eq('is_active', true);
+
+        if (error) {
+            logger.error('Failed to fetch accounts for sync', error);
+            return;
+        }
+
+        if (!accounts || accounts.length === 0) {
+            logger.debug('No active accounts to sync');
+            return;
+        }
+
+        logger.info(`Running global sync for ${accounts.length} accounts`);
+
+        // Group by user to check their sync interval settings
+        const userAccounts = new Map<string, typeof accounts>();
+        for (const account of accounts) {
+            const existing = userAccounts.get(account.user_id) || [];
+            existing.push(account);
+            userAccounts.set(account.user_id, existing);
+        }
+
+        // Process each user's accounts
+        for (const [userId, userAccountList] of userAccounts) {
+            // Check user's sync interval preference
+            const { data: settings } = await this.supabase
+                .from('user_settings')
+                .select('sync_interval_minutes')
+                .eq('user_id', userId)
+                .single();
+
+            const syncIntervalMs = (settings?.sync_interval_minutes || 5) * 60 * 1000;
+
+            // Check last sync time
+            const { data: lastLog } = await this.supabase
+                .from('processing_logs')
+                .select('started_at')
+                .eq('user_id', userId)
+                .eq('status', 'success')
+                .order('started_at', { ascending: false })
+                .limit(1)
+                .single();
+
+            if (lastLog) {
+                const lastSyncTime = new Date(lastLog.started_at).getTime();
+                const now = Date.now();
+                if (now - lastSyncTime < syncIntervalMs) {
+                    logger.debug(`Skipping sync for user ${userId}, last sync was recent`);
+                    continue;
+                }
+            }
+
+            // Sync each account
+            const processor = new EmailProcessorService(this.supabase);
+            for (const account of userAccountList) {
+                try {
+                    await processor.syncAccount(account.id, userId);
+                } catch (error) {
+                    logger.error('Account sync failed', error, { accountId: account.id });
+                }
+            }
+        }
+    }
+
+    private scheduleCleanup(): void {
+        const jobId = 'cleanup';
+        const interval = 24 * 60 * 60 * 1000; // Daily
+
+        const job: ScheduledJob = {
+            id: jobId,
+            name: 'Data Cleanup',
+            interval,
+            lastRun: null,
+            isRunning: false,
+            timer: null,
+        };
+
+        job.timer = setInterval(async () => {
+            if (job.isRunning) return;
+
+            job.isRunning = true;
+            try {
+                await this.runCleanup();
+                job.lastRun = new Date();
+            } catch (error) {
+                logger.error('Cleanup failed', error);
+            } finally {
+                job.isRunning = false;
+            }
+        }, interval);
+
+        this.jobs.set(jobId, job);
+        logger.info('Scheduled daily cleanup');
+    }
+
+    private async runCleanup(): Promise<void> {
+        if (!this.supabase) return;
+
+        logger.info('Running cleanup job');
+
+        // Delete old processing logs (older than 30 days)
+        const thirtyDaysAgo = new Date();
+        thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+        const { error: logsError } = await this.supabase
+            .from('processing_logs')
+            .delete()
+            .lt('started_at', thirtyDaysAgo.toISOString());
+
+        if (logsError) {
+            logger.error('Failed to cleanup old logs', logsError);
+        }
+
+        // Delete emails that were trashed more than 7 days ago
+        const sevenDaysAgo = new Date();
+        sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+
+        const { error: emailsError } = await this.supabase
+            .from('emails')
+            .delete()
+            .eq('action_taken', 'delete')
+            .lt('created_at', sevenDaysAgo.toISOString());
+
+        if (emailsError) {
+            logger.error('Failed to cleanup old emails', emailsError);
+        }
+
+        logger.info('Cleanup completed');
+    }
+
+    getJobStatus(): Array<{ id: string; name: string; lastRun: Date | null; isRunning: boolean }> {
+        return Array.from(this.jobs.values()).map(job => ({
+            id: job.id,
+            name: job.name,
+            lastRun: job.lastRun,
+            isRunning: job.isRunning,
+        }));
+    }
+}
+
+// Singleton
+let schedulerInstance: SyncScheduler | null = null;
+
+export function getScheduler(): SyncScheduler {
+    if (!schedulerInstance) {
+        schedulerInstance = new SyncScheduler();
+    }
+    return schedulerInstance;
+}
+
+export function startScheduler(): void {
+    getScheduler().start();
+}
+
+export function stopScheduler(): void {
+    if (schedulerInstance) {
+        schedulerInstance.stop();
+    }
+}

package/api/src/services/supabase.ts

@@ -0,0 +1,144 @@
+import { createClient, SupabaseClient } from '@supabase/supabase-js';
+import { config } from '../config/index.js';
+import { createLogger } from '../utils/logger.js';
+
+const logger = createLogger('SupabaseService');
+
+let serverClient: SupabaseClient | null = null;
+
+export function isValidUrl(url: string): boolean {
+    try {
+        return url.startsWith('http://') || url.startsWith('https://');
+    } catch {
+        return false;
+    }
+}
+
+export function getServerSupabase(): SupabaseClient | null {
+    if (serverClient) return serverClient;
+
+    const url = config.supabase.url;
+    const key = config.supabase.anonKey;
+
+    if (!url || !key || !isValidUrl(url)) {
+        logger.warn('Supabase not configured or invalid URL - skipping client initialization', {
+            url: url || 'missing'
+        });
+        return null;
+    }
+
+    try {
+        serverClient = createClient(url, key, {
+            auth: {
+                autoRefreshToken: false,
+                persistSession: false,
+            },
+        });
+
+        logger.info('Server Supabase client initialized');
+        return serverClient;
+    } catch (error) {
+        logger.error('Failed to initialize Supabase client', error);
+        return null;
+    }
+}
+
+export function getServiceRoleSupabase(): SupabaseClient | null {
+    const url = config.supabase.url;
+    const key = config.supabase.serviceRoleKey;
+
+    if (!url || !key || !isValidUrl(url)) {
+        logger.warn('Service role Supabase not configured or invalid URL');
+        return null;
+    }
+
+    try {
+        return createClient(url, key, {
+            auth: {
+                autoRefreshToken: false,
+                persistSession: false,
+            },
+        });
+    } catch (error) {
+        logger.error('Failed to initialize Service Role Supabase client', error);
+        return null;
+    }
+}
+
+// Database types (expand as needed)
+export interface EmailAccount {
+    id: string;
+    user_id: string;
+    provider: 'gmail' | 'outlook';
+    email_address: string;
+    access_token: string | null;
+    refresh_token: string | null;
+    token_expires_at: string | null;
+    scopes: string[];
+    is_active: boolean;
+    last_sync_checkpoint?: string | null;
+    sync_start_date?: string | null;
+    sync_max_emails_per_run?: number;
+    last_sync_at?: string | null;
+    last_sync_status?: 'idle' | 'syncing' | 'success' | 'error';
+    last_sync_error?: string | null;
+    created_at: string;
+    updated_at: string;
+}
+
+export interface Email {
+    id: string;
+    account_id: string;
+    external_id: string;
+    subject: string | null;
+    sender: string | null;
+    recipient: string | null;
+    date: string | null;
+    body_snippet: string | null;
+    category: string | null;
+    is_useless: boolean;
+    ai_analysis: Record<string, unknown> | null;
+    suggested_action: string | null; // Deprecated
+    suggested_actions?: string[];
+    action_taken: string | null; // Deprecated
+    actions_taken?: string[];
+    created_at: string;
+}
+
+export interface Rule {
+    id: string;
+    user_id: string;
+    name: string;
+    condition: Record<string, unknown>;
+    action: 'delete' | 'archive' | 'draft' | 'star' | 'read';
+    instructions?: string;
+    attachments?: any[];
+    is_enabled: boolean;
+    created_at: string;
+}
+
+export interface ProcessingLog {
+    id: string;
+    user_id: string;
+    account_id: string | null;
+    status: 'running' | 'success' | 'failed';
+    started_at: string;
+    completed_at: string | null;
+    emails_processed: number;
+    emails_deleted: number;
+    emails_drafted: number;
+    error_message: string | null;
+}
+
+export interface UserSettings {
+    id: string;
+    user_id: string;
+    llm_model: string | null;
+    llm_base_url: string | null;
+    llm_api_key: string | null;
+    auto_trash_spam: boolean;
+    smart_drafts: boolean;
+    sync_interval_minutes: number;
+    created_at: string;
+    updated_at: string;
+}

package/api/src/utils/contentCleaner.ts

@@ -0,0 +1,114 @@
+export class ContentCleaner {
+    /**
+     * Cleans email body by removing noise, quoted replies, and footers.
+     * Ported from Python ContentCleaner.
+     */
+    static cleanEmailBody(text: string): string {
+        if (!text) return "";
+
+        // 0. Lightweight HTML -> Markdown Conversion
+        
+        // Structure: <br>, <p> -> Newlines
+        text = text.replace(/<br\s*\/?\?>/gi, '\n');
+        text = text.replace(/<\/p>/gi, '\n\n');
+        text = text.replace(/<p.*?>/gi, ''); // Open p tags just gone
+        
+        // Structure: Headers <h1>-<h6> -> # Title
+        text = text.replace(/<h[1-6].*?>(.*?)<\/h[1-6]>/gsi, (match, p1) => `\n# ${p1}\n`);
+        
+        // Structure: Lists <li> -> - Item
+        text = text.replace(/<li.*?>(.*?)<\/li>/gsi, (match, p1) => `\n- ${p1}`);
+        text = text.replace(/<ul.*?>/gi, '');
+        text = text.replace(/<\/ul>/gi, '\n');
+        
+        // Links: <a href=\"...\">text</a> -> [text](href)
+        text = text.replace(/<a\s+(?:[^>]*?\s+)?href=\"([^\"]*)\"[^>]*>(.*?)<\/a>/gsi, (match, href, content) => `[${content}](${href})`);
+        
+        // Images: <img src=\"...\" alt=\"...\"> -> ![alt](src)
+        text = text.replace(/<img\s+(?:[^>]*?\s+)?src=\"([^\"]*)\"(?:[^>]*?\s+)?alt=\"([^\"]*)\"[^>]*>/gsi, (match, src, alt) => `![${alt}](${src})`);
+
+        // Style/Script removal (strictly remove content)
+        text = text.replace(/<script.*?>.*?<\/script>/gsi, '');
+        text = text.replace(/<style.*?>.*?<\/style>/gsi, '');
+        
+        // Final Strip of remaining tags
+        text = text.replace(/<[^>]+>/g, ' ');
+        
+        // Entity decoding (Basic)
+        text = text.replace(/&nbsp;/gi, ' ');
+        text = text.replace(/&amp;/gi, '&');
+        text = text.replace(/&lt;/gi, '<');
+        text = text.replace(/&gt;/gi, '>');
+        text = text.replace(/&quot;/gi, '"');
+        text = text.replace(/&#39;/gi, "'");
+
+        const lines = text.split('\n');
+        const cleanedLines: string[] = [];
+        
+        // Heuristics for reply headers
+        const replyHeaderPatterns = [
+            /^On .* wrote:$/i,
+            /^From: .*$/i,
+            /^Sent: .*$/i,
+            /^To: .*$/i,
+            /^Subject: .*$/i
+        ];
+
+        // Heuristics for footers
+        const footerPatterns = [
+            /unsubscribe/i,
+            /privacy policy/i,
+            /terms of service/i,
+            /view in browser/i,
+            /copyright \d{4}/i
+        ];
+
+        for (let line of lines) {
+            let lineStripped = line.trim();
+            
+            // 2. Quoted text removal (lines starting with >)
+            if (lineStripped.startsWith('>')) {
+                continue;
+            }
+                
+            // 3. Check for specific reply separators
+            // If we hit a reply header, we truncate the rest (Aggressive strategy per Python code)
+            if (/^On .* wrote:$/i.test(lineStripped)) {
+                break;
+            }
+
+            // 4. Footer removal (simple check on short lines)
+            if (lineStripped.length < 100) {
+                let isFooter = false;
+                for (const pattern of footerPatterns) {
+                    if (pattern.test(lineStripped)) {
+                        isFooter = true;
+                        break;
+                    }
+                }
+                if (isFooter) {
+                    continue;
+                }
+            }
+
+            cleanedLines.push(line);
+        }
+
+        // Reassemble
+        text = cleanedLines.join('\n');
+        
+        // Collapse multiple newlines
+        text = text.replace(/\n{3,}/g, '\n\n');
+
+        // Sanitize LLM Special Tokens (Prevent Prompt Injection/Confusion)
+        // Break sequences like <|channel|>, [INST], <s>
+        text = text.replace(/<\|/g, '< |'); 
+        text = text.replace(/\|>/g, '| >');
+        text = text.replace(/\[INST\]/gi, '[ INST ]');
+        text = text.replace(/\[\/INST\]/gi, '[ /INST ]');
+        text = text.replace(/<s>/gi, '&lt;s&gt;');
+        text = text.replace(/<\/s>/gi, '&lt;/s&gt;');
+        
+        return text.trim();
+    }
+}

package/api/src/utils/crypto.ts

@@ -0,0 +1,80 @@
+import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from 'crypto';
+import { config } from '../config/index.js';
+
+// Edge Functions compatible encryption (matching supabase/functions/_shared/encryption.ts)
+const ALGORITHM = 'aes-256-gcm';
+const IV_LENGTH = 12; // Match Edge Functions
+const KEY_LENGTH = 32;
+
+function getKey(): Buffer {
+    const secret = config.security.encryptionKey || 'dev-key-not-secure';
+    // Match Edge Functions key derivation: pad/slice to 32 chars
+    return Buffer.from(secret.padEnd(32, '0').slice(0, 32), 'utf8');
+}
+
+export function encryptToken(plaintext: string): string {
+    if (!plaintext) return '';
+    
+    const iv = randomBytes(IV_LENGTH);
+    const key = getKey();
+    
+    const cipher = createCipheriv(ALGORITHM, key, iv);
+    const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
+    const tag = cipher.getAuthTag();
+    
+    // Format: base64(iv + ciphertext + tag) - compatible with Edge Functions
+    const combined = Buffer.concat([iv, encrypted, tag]);
+    return combined.toString('base64');
+}
+
+export function decryptToken(encrypted: string): string {
+    if (!encrypted) return '';
+    
+    try {
+        // Try Edge Functions format first: base64(iv + ciphertext + tag)
+        const combined = Buffer.from(encrypted, 'base64');
+        
+        if (combined.length < IV_LENGTH + 16) {
+            // Too short, might be plaintext
+            return encrypted;
+        }
+        
+        const iv = combined.subarray(0, IV_LENGTH);
+        const tag = combined.subarray(combined.length - 16);
+        const data = combined.subarray(IV_LENGTH, combined.length - 16);
+        const key = getKey();
+        
+        const decipher = createDecipheriv(ALGORITHM, key, iv);
+        decipher.setAuthTag(tag);
+        
+        return Buffer.concat([decipher.update(data), decipher.final()]).toString('utf8');
+    } catch (error) {
+        // Try legacy format: salt:iv:tag:encrypted
+        try {
+            const parts = encrypted.split(':');
+            if (parts.length === 4) {
+                const [saltB64, ivB64, tagB64, dataB64] = parts;
+                const salt = Buffer.from(saltB64, 'base64');
+                const iv = Buffer.from(ivB64, 'base64');
+                const tag = Buffer.from(tagB64, 'base64');
+                const data = Buffer.from(dataB64, 'base64');
+                const secret = config.security.encryptionKey || 'dev-key-not-secure';
+                const key = scryptSync(secret, salt, KEY_LENGTH);
+                
+                const decipher = createDecipheriv(ALGORITHM, key, iv);
+                decipher.setAuthTag(tag);
+                
+                return Buffer.concat([decipher.update(data), decipher.final()]).toString('utf8');
+            }
+        } catch {
+            // Fall through to plaintext
+        }
+        
+        // If all decryption fails, assume plaintext (for migration)
+        return encrypted;
+    }
+}
+
+export function generateSecureToken(length: number = 32): string {
+    return randomBytes(length).toString('hex');
+}