@realtimex/email-automator 2.2.1 → 2.3.1

package/dist/api/src/services/processor.js

@@ -0,0 +1,503 @@
+import { createLogger } from '../utils/logger.js';
+import { config } from '../config/index.js';
+import { getGmailService } from './gmail.js';
+import { getMicrosoftService } from './microsoft.js';
+import { getIntelligenceService } from './intelligence.js';
+import { EventLogger } from './eventLogger.js';
+const logger = createLogger('Processor');
+export class EmailProcessorService {
+    supabase;
+    gmailService = getGmailService();
+    microsoftService = getMicrosoftService();
+    constructor(supabase) {
+        this.supabase = supabase;
+    }
+    async syncAccount(accountId, userId) {
+        const result = { processed: 0, deleted: 0, drafted: 0, errors: 0 };
+        // Create processing log
+        const { data: log } = await this.supabase
+            .from('processing_logs')
+            .insert({
+            user_id: userId,
+            account_id: accountId,
+            status: 'running',
+        })
+            .select()
+            .single();
+        try {
+            // Fetch account
+            const { data: account, error: accError } = await this.supabase
+                .from('email_accounts')
+                .select('*')
+                .eq('id', accountId)
+                .eq('user_id', userId)
+                .single();
+            if (accError || !account) {
+                throw new Error('Account not found or access denied');
+            }
+            // Refresh token if needed
+            let refreshedAccount = account;
+            if (account.provider === 'gmail') {
+                refreshedAccount = await this.gmailService.refreshTokenIfNeeded(this.supabase, account);
+            }
+            // Update status to syncing
+            await this.supabase
+                .from('email_accounts')
+                .update({
+                last_sync_status: 'syncing',
+                last_sync_at: new Date().toISOString()
+            })
+                .eq('id', accountId);
+            // Fetch user's rules
+            const { data: rules } = await this.supabase
+                .from('rules')
+                .select('*')
+                .eq('user_id', userId)
+                .eq('is_enabled', true);
+            // Fetch user settings for AI preferences
+            const { data: settings } = await this.supabase
+                .from('user_settings')
+                .select('*')
+                .eq('user_id', userId)
+                .single();
+            const eventLogger = log ? new EventLogger(this.supabase, log.id) : null;
+            if (eventLogger)
+                await eventLogger.info('Running', 'Starting sync process');
+            // Process based on provider
+            if (refreshedAccount.provider === 'gmail') {
+                await this.processGmailAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+            }
+            else if (refreshedAccount.provider === 'outlook') {
+                await this.processOutlookAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+            }
+            // After processing new emails, run retention rules for this account
+            await this.runRetentionRules(refreshedAccount, rules || [], settings, result, eventLogger);
+            // Update log and account on success
+            if (log) {
+                await this.supabase
+                    .from('processing_logs')
+                    .update({
+                    status: 'success',
+                    completed_at: new Date().toISOString(),
+                    emails_processed: result.processed,
+                    emails_deleted: result.deleted,
+                    emails_drafted: result.drafted,
+                })
+                    .eq('id', log.id);
+            }
+            await this.supabase
+                .from('email_accounts')
+                .update({
+                last_sync_status: 'success',
+                last_sync_error: null
+            })
+                .eq('id', accountId);
+            logger.info('Sync completed', { accountId, ...result });
+        }
+        catch (error) {
+            logger.error('Sync failed', error, { accountId });
+            const errMsg = error instanceof Error ? error.message : 'Unknown error';
+            if (log) {
+                await this.supabase
+                    .from('processing_logs')
+                    .update({
+                    status: 'failed',
+                    completed_at: new Date().toISOString(),
+                    error_message: errMsg,
+                })
+                    .eq('id', log.id);
+            }
+            await this.supabase
+                .from('email_accounts')
+                .update({
+                last_sync_status: 'error',
+                last_sync_error: errMsg
+            })
+                .eq('id', accountId);
+            // If it's a fatal setup error (e.g. Account not found), throw it
+            if (errMsg.includes('Account not found') || errMsg.includes('access denied')) {
+                throw error;
+            }
+            // Otherwise, increment error count and return partial results
+            result.errors++;
+        }
+        return result;
+    }
+    async processGmailAccount(account, rules, settings, result, eventLogger) {
+        const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
+        // Construct query: Use checkpoint if it's newer than the user-defined start date
+        let effectiveStartMs = 0;
+        if (account.sync_start_date) {
+            effectiveStartMs = new Date(account.sync_start_date).getTime();
+        }
+        if (account.last_sync_checkpoint) {
+            const checkpointMs = parseInt(account.last_sync_checkpoint);
+            if (checkpointMs > effectiveStartMs) {
+                effectiveStartMs = checkpointMs;
+            }
+        }
+        let query = '';
+        if (effectiveStartMs > 0) {
+            const startSeconds = Math.floor(effectiveStartMs / 1000);
+            query = `after:${startSeconds}`;
+        }
+        if (eventLogger)
+            await eventLogger.info('Fetching', 'Fetching emails from Gmail', { query, batchSize });
+        const { messages } = await this.gmailService.fetchMessages(account, {
+            maxResults: batchSize,
+            query: query || undefined,
+        });
+        if (eventLogger)
+            await eventLogger.info('Fetching', `Fetched ${messages.length} emails`);
+        // We process in ASCENDING order (oldest to newest) to move checkpoint forward correctly
+        const sortedMessages = [...messages].reverse();
+        let maxInternalDate = account.last_sync_checkpoint ? parseInt(account.last_sync_checkpoint) : 0;
+        for (const message of sortedMessages) {
+            try {
+                await this.processMessage(account, message, rules, settings, result, eventLogger);
+                // Checkpoint tracking
+                const msgDate = new Date(message.date).getTime();
+                if (msgDate > maxInternalDate) {
+                    maxInternalDate = msgDate;
+                    await this.supabase
+                        .from('email_accounts')
+                        .update({ last_sync_checkpoint: maxInternalDate.toString() })
+                        .eq('id', account.id);
+                }
+            }
+            catch (error) {
+                logger.error('Failed to process Gmail message', error, { messageId: message.id });
+                if (eventLogger)
+                    await eventLogger.error('Error', error);
+                result.errors++;
+            }
+        }
+    }
+    async processOutlookAccount(account, rules, settings, result, eventLogger) {
+        const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
+        // Construct filter: Use checkpoint if it's newer than the user-defined start date
+        let effectiveStartIso = '';
+        if (account.sync_start_date) {
+            effectiveStartIso = new Date(account.sync_start_date).toISOString();
+        }
+        if (account.last_sync_checkpoint) {
+            if (!effectiveStartIso || account.last_sync_checkpoint > effectiveStartIso) {
+                effectiveStartIso = account.last_sync_checkpoint;
+            }
+        }
+        let filter = '';
+        if (effectiveStartIso) {
+            filter = `receivedDateTime gt ${effectiveStartIso}`;
+        }
+        if (eventLogger)
+            await eventLogger.info('Fetching', 'Fetching emails from Outlook', { filter, batchSize });
+        const { messages } = await this.microsoftService.fetchMessages(account, {
+            top: batchSize,
+            filter: filter || undefined,
+        });
+        if (eventLogger)
+            await eventLogger.info('Fetching', `Fetched ${messages.length} emails`);
+        const sortedMessages = [...messages].reverse();
+        let latestCheckpoint = account.last_sync_checkpoint || '';
+        for (const message of sortedMessages) {
+            try {
+                await this.processMessage(account, message, rules, settings, result, eventLogger);
+                if (!latestCheckpoint || message.date > latestCheckpoint) {
+                    latestCheckpoint = message.date;
+                    await this.supabase
+                        .from('email_accounts')
+                        .update({ last_sync_checkpoint: latestCheckpoint })
+                        .eq('id', account.id);
+                }
+            }
+            catch (error) {
+                logger.error('Failed to process Outlook message', error, { messageId: message.id });
+                if (eventLogger)
+                    await eventLogger.error('Error', error);
+                result.errors++;
+            }
+        }
+    }
+    async processMessage(account, message, rules, settings, result, eventLogger) {
+        // Check if already processed
+        const { data: existing } = await this.supabase
+            .from('emails')
+            .select('id')
+            .eq('account_id', account.id)
+            .eq('external_id', message.id)
+            .single();
+        if (existing) {
+            logger.debug('Message already processed', { messageId: message.id });
+            if (eventLogger)
+                await eventLogger.info('Skipped', `Already processed: ${message.subject}`);
+            return;
+        }
+        if (eventLogger)
+            await eventLogger.info('Processing', `Processing email: ${message.subject}`);
+        // 1. Create a "Skeleton" record to get an ID for tracing
+        const emailData = {
+            account_id: account.id,
+            external_id: message.id,
+            subject: message.subject,
+            sender: message.sender,
+            recipient: message.recipient,
+            date: message.date ? new Date(message.date).toISOString() : null,
+            body_snippet: message.snippet || message.body.substring(0, 500),
+        };
+        const { data: savedEmail, error: saveError } = await this.supabase
+            .from('emails')
+            .insert(emailData)
+            .select()
+            .single();
+        if (saveError || !savedEmail) {
+            logger.error('Failed to create initial email record', saveError);
+            if (eventLogger)
+                await eventLogger.error('Database Error', saveError);
+            return;
+        }
+        // 2. Analyze with AI (passing the ID so events are linked)
+        const intelligenceService = getIntelligenceService(settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
+            ? {
+                model: settings.llm_model,
+                baseUrl: settings.llm_base_url,
+                apiKey: settings.llm_api_key,
+            }
+            : undefined);
+        const analysis = await intelligenceService.analyzeEmail(message.body, {
+            subject: message.subject,
+            sender: message.sender,
+            date: message.date,
+            metadata: message.headers,
+            userPreferences: {
+                autoTrashSpam: settings?.auto_trash_spam,
+                smartDrafts: settings?.smart_drafts,
+            },
+        }, eventLogger || undefined, savedEmail.id);
+        if (!analysis) {
+            result.errors++;
+            return;
+        }
+        // 3. Update the email record with results
+        await this.supabase
+            .from('emails')
+            .update({
+            category: analysis.category,
+            is_useless: analysis.is_useless,
+            ai_analysis: analysis,
+            suggested_actions: analysis.suggested_actions || [],
+            suggested_action: analysis.suggested_actions?.[0] || 'none',
+        })
+            .eq('id', savedEmail.id);
+        const processedEmail = {
+            ...savedEmail,
+            category: analysis.category,
+            is_useless: analysis.is_useless,
+            suggested_actions: analysis.suggested_actions
+        };
+        result.processed++;
+        // 4. Execute automation rules
+        await this.executeRules(account, processedEmail, analysis, rules, settings, result, eventLogger);
+    }
+    async executeRules(account, email, analysis, rules, settings, result, eventLogger) {
+        // User-defined and System rules (Unified)
+        for (const rule of rules) {
+            if (this.matchesCondition(email, analysis, rule.condition)) {
+                let draftContent = undefined;
+                // If the rule is to draft, and it has specific instructions, generate it now
+                if (rule.action === 'draft' && rule.instructions) {
+                    if (eventLogger)
+                        await eventLogger.info('Thinking', `Generating customized draft based on rule: ${rule.name}`, undefined, email.id);
+                    const intelligenceService = getIntelligenceService(settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
+                        ? {
+                            model: settings.llm_model,
+                            baseUrl: settings.llm_base_url,
+                            apiKey: settings.llm_api_key,
+                        }
+                        : undefined);
+                    const customizedDraft = await intelligenceService.generateDraftReply({
+                        subject: email.subject || '',
+                        sender: email.sender || '',
+                        body: email.body_snippet || '' // Note: body_snippet is used here, might want full body if available
+                    }, rule.instructions);
+                    if (customizedDraft) {
+                        draftContent = customizedDraft;
+                    }
+                }
+                await this.executeAction(account, email, rule.action, draftContent, eventLogger, `Rule: ${rule.name}`, rule.attachments);
+                if (rule.action === 'delete')
+                    result.deleted++;
+                else if (rule.action === 'draft')
+                    result.drafted++;
+            }
+        }
+    }
+    matchesCondition(email, analysis, condition) {
+        if (!analysis)
+            return false;
+        for (const [key, value] of Object.entries(condition)) {
+            const val = value;
+            switch (key) {
+                case 'sender_email':
+                    if (email.sender?.toLowerCase() !== val.toLowerCase())
+                        return false;
+                    break;
+                case 'sender_domain':
+                    // Check if sender ends with domain (e.g. @gmail.com)
+                    const domain = val.startsWith('@') ? val : `@${val}`;
+                    if (!email.sender?.toLowerCase().endsWith(domain.toLowerCase()))
+                        return false;
+                    break;
+                case 'sender_contains':
+                    if (!email.sender?.toLowerCase().includes(val.toLowerCase()))
+                        return false;
+                    break;
+                case 'subject_contains':
+                    if (!email.subject?.toLowerCase().includes(val.toLowerCase()))
+                        return false;
+                    break;
+                case 'body_contains':
+                    if (!email.body_snippet?.toLowerCase().includes(val.toLowerCase()))
+                        return false;
+                    break;
+                case 'older_than_days':
+                    if (!email.date)
+                        return false;
+                    const ageInMs = Date.now() - new Date(email.date).getTime();
+                    const ageInDays = ageInMs / (1000 * 60 * 60 * 24);
+                    if (ageInDays < value)
+                        return false;
+                    break;
+                case 'category':
+                    if (analysis.category !== value)
+                        return false;
+                    break;
+                case 'priority':
+                    if (analysis.priority !== value)
+                        return false;
+                    break;
+                case 'sentiment':
+                    if (analysis.sentiment !== value)
+                        return false;
+                    break;
+                case 'is_useless':
+                    if (analysis.is_useless !== value)
+                        return false;
+                    break;
+                case 'suggested_actions':
+                    // Handle array membership check (e.g. if condition expects "reply" to be in actions)
+                    const requiredActions = Array.isArray(value) ? value : [value];
+                    const hasAllActions = requiredActions.every(req => analysis.suggested_actions?.includes(req));
+                    if (!hasAllActions)
+                        return false;
+                    break;
+                default:
+                    // Fallback for any other keys that might be in analysis
+                    if (analysis[key] !== value)
+                        return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Scans already processed emails and applies rules that have a time-based condition (retention).
+     */
+    async runRetentionRules(account, rules, settings, result, eventLogger) {
+        // Find rules that have an age condition
+        const retentionRules = rules.filter(r => r.condition.older_than_days !== undefined);
+        if (retentionRules.length === 0)
+            return;
+        if (eventLogger)
+            await eventLogger.info('Retention', `Checking retention rules for ${retentionRules.length} policies`);
+        // Fetch emails for this account that have been analyzed but haven't had an action taken yet
+        const { data: processedEmails, error } = await this.supabase
+            .from('emails')
+            .select('*')
+            .eq('account_id', account.id)
+            .is('action_taken', null)
+            .not('ai_analysis', 'is', null)
+            .order('date', { ascending: true });
+        if (error || !processedEmails)
+            return;
+        for (const email of processedEmails) {
+            for (const rule of retentionRules) {
+                if (this.matchesCondition(email, email.ai_analysis, rule.condition)) {
+                    if (eventLogger)
+                        await eventLogger.info('Retention', `Applying retention rule: ${rule.name} to ${email.subject}`);
+                    // We don't support custom drafts in retention yet (usually retention is for delete/archive)
+                    await this.executeAction(account, email, rule.action, undefined, eventLogger, `Retention Rule: ${rule.name}`);
+                    if (rule.action === 'delete')
+                        result.deleted++;
+                    else if (rule.action === 'draft')
+                        result.drafted++;
+                    break; // Only one rule per email
+                }
+            }
+        }
+    }
+    async executeAction(account, email, action, draftContent, eventLogger, reason, attachments) {
+        try {
+            if (eventLogger) {
+                await eventLogger.info('Acting', `Executing action: ${action}`, { reason, hasAttachments: !!attachments?.length }, email.id);
+            }
+            if (account.provider === 'gmail') {
+                if (action === 'delete') {
+                    await this.gmailService.trashMessage(account, email.external_id);
+                }
+                else if (action === 'archive') {
+                    await this.gmailService.archiveMessage(account, email.external_id);
+                }
+                else if (action === 'draft' && draftContent) {
+                    const draftId = await this.gmailService.createDraft(account, email.external_id, draftContent, this.supabase, attachments);
+                    if (eventLogger) {
+                        await eventLogger.info('Drafted', `Draft created successfully. ID: ${draftId}`, { draftId }, email.id);
+                    }
+                }
+                else if (action === 'read') {
+                    await this.gmailService.markAsRead(account, email.external_id);
+                }
+                else if (action === 'star') {
+                    await this.gmailService.starMessage(account, email.external_id);
+                }
+            }
+            else if (account.provider === 'outlook') {
+                if (action === 'delete') {
+                    await this.microsoftService.trashMessage(account, email.external_id);
+                }
+                else if (action === 'archive') {
+                    await this.microsoftService.archiveMessage(account, email.external_id);
+                }
+                else if (action === 'draft' && draftContent) {
+                    await this.microsoftService.createDraft(account, email.external_id, draftContent);
+                }
+                else if (action === 'read') {
+                    await this.microsoftService.markAsRead(account, email.external_id);
+                }
+                else if (action === 'star') {
+                    await this.microsoftService.flagMessage(account, email.external_id);
+                }
+            }
+            // Update email record using atomic array concatenation to prevent race conditions
+            await this.supabase.rpc('append_email_action', {
+                p_email_id: email.id,
+                p_action: action
+            });
+            // Fallback for legacy column
+            await this.supabase
+                .from('emails')
+                .update({ action_taken: action })
+                .eq('id', email.id);
+            logger.debug('Action executed', { emailId: email.id, action });
+            if (eventLogger) {
+                await eventLogger.action('Acted', email.id, action, reason);
+            }
+        }
+        catch (error) {
+            logger.error('Failed to execute action', error, { emailId: email.id, action });
+            if (eventLogger) {
+                const errMsg = error instanceof Error ? error.message : String(error);
+                await eventLogger.error('Action Failed', { error: errMsg, action }, email.id);
+            }
+            // Do NOT throw here - we want to continue with other emails/actions
+        }
+    }
+}