@realtimex/email-automator 2.24.0 → 2.26.0

package/api/server.ts

@@ -14,7 +14,7 @@ import routes from './src/routes/index.js';
 import { logger } from './src/utils/logger.js';
 import { getServerSupabase, getServiceRoleSupabase } from './src/services/supabase.js';
 import { startScheduler, stopScheduler } from './src/services/scheduler.js';
-import { setEncryptionKey } from './src/utils/encryption.js';
+import { setEncryptionKey, getEncryptionKeyHex } from './src/utils/encryption.js';
 
 // Initialize Persistence Encryption
 // NOTE: In RealTimeX Desktop sandbox, all config is stored in Supabase (no .env files)
@@ -83,6 +83,44 @@ async function initializePersistenceEncryption() {
     }
 }
 
+// Periodic sync: Persist in-memory encryption key to users without one
+// This handles the "first user" case and any users created before key was persisted
+async function syncEncryptionKeyToUsers() {
+    try {
+        const supabase = getServiceRoleSupabase();
+        if (!supabase) return;
+
+        const currentKey = getEncryptionKeyHex();
+        if (!currentKey) return; // No key in memory yet
+
+        // Find users without encryption key
+        const { data: usersWithoutKey, error } = await supabase
+            .from('user_settings')
+            .select('user_id')
+            .is('encryption_key', null)
+            .limit(100);
+
+        if (error || !usersWithoutKey || usersWithoutKey.length === 0) {
+            return; // No users to update
+        }
+
+        logger.info(`Found ${usersWithoutKey.length} user(s) without encryption key, persisting...`);
+
+        // Update all users without a key
+        const updates = usersWithoutKey.map(user =>
+            supabase
+                .from('user_settings')
+                .update({ encryption_key: currentKey })
+                .eq('user_id', user.user_id)
+        );
+
+        await Promise.all(updates);
+        logger.info(`✓ Persisted encryption key to ${usersWithoutKey.length} user(s)`);
+    } catch (err) {
+        logger.warn('Error syncing encryption key to users:', { error: err });
+    }
+}
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
@@ -386,6 +424,9 @@ const startServer = async () => {
     // Try to load encryption key before accepting requests
     await initializePersistenceEncryption();
 
+    // Initial sync to persist key to any users without one (handles first user case)
+    setTimeout(() => syncEncryptionKeyToUsers(), 500); // Wait 500ms for DB to be ready
+
     const server = app.listen(config.port, () => {
         const url = `http://localhost:${config.port}`;
         logger.info(`Server running at ${url}`, {
@@ -397,6 +438,19 @@ const startServer = async () => {
         if (getServerSupabase()) {
             startScheduler();
         }
+
+        // Periodic sync: persist encryption key to new users every 30 seconds for first 5 minutes
+        // This ensures first user gets the key even if created after server start
+        let syncCount = 0;
+        const syncInterval = setInterval(() => {
+            syncEncryptionKeyToUsers();
+            syncCount++;
+            if (syncCount >= 10) { // 10 * 30s = 5 minutes
+                clearInterval(syncInterval);
+                // After 5 minutes, sync less frequently (every 5 minutes)
+                setInterval(syncEncryptionKeyToUsers, 5 * 60 * 1000);
+            }
+        }, 30 * 1000); // Every 30 seconds
     });
 
     // Handle server errors

package/api/src/middleware/index.ts

@@ -1,4 +1,4 @@
 export { errorHandler, asyncHandler, AppError, ValidationError, AuthenticationError, AuthorizationError, NotFoundError, RateLimitError } from './errorHandler.js';
 export { authMiddleware, optionalAuth, requireRole } from './auth.js';
-export { rateLimit, authRateLimit, apiRateLimit, syncRateLimit } from './rateLimit.js';
+export { rateLimit, authRateLimit, apiRateLimit, syncRateLimit, connectionRateLimit } from './rateLimit.js';
 export { validateBody, validateQuery, validateParams, schemas } from './validation.js';

package/api/src/middleware/rateLimit.ts

@@ -85,3 +85,9 @@ export const syncRateLimit = rateLimit({
     windowMs: 60 * 1000, // 1 minute
     max: 20, // 20 sync requests per minute
 });
+
+// Connection testing rate limit (lenient for troubleshooting)
+export const connectionRateLimit = rateLimit({
+    windowMs: 15 * 60 * 1000, // 15 minutes
+    max: 30, // 30 connection attempts per 15 minutes
+});

package/api/src/routes/auth.ts

@@ -1,19 +1,20 @@
 import { Router } from 'express';
 import { asyncHandler, ValidationError } from '../middleware/errorHandler.js';
 import { authMiddleware } from '../middleware/auth.js';
-import { authRateLimit } from '../middleware/rateLimit.js';
+import { authRateLimit, connectionRateLimit } from '../middleware/rateLimit.js';
 import { validateBody, schemas } from '../middleware/validation.js';
 import { getGmailService } from '../services/gmail.js';
 import { getMicrosoftService } from '../services/microsoft.js';
 import { getImapService } from '../services/imap-service.js';
 import { createLogger } from '../utils/logger.js';
+import { getEncryptionKeyHex } from '../utils/encryption.js';
 
 const router = Router();
 const logger = createLogger('AuthRoutes');
 
 // IMAP/SMTP Connection
 router.post('/imap/connect',
-    authRateLimit,
+    connectionRateLimit, // More lenient for connection testing (30 attempts / 15 min)
     authMiddleware,
     validateBody(schemas.imapConnect),
     asyncHandler(async (req, res) => {
@@ -23,6 +24,27 @@ router.post('/imap/connect',
             smtpHost, smtpPort, smtpSecure
         } = req.body;
 
+        // CRITICAL FIX: Ensure user has encryption key before IMAP operations
+        // This handles the "first user connects IMAP immediately after signup" case
+        const currentKey = getEncryptionKeyHex();
+        if (currentKey) {
+            const { data: userSettings } = await req.supabase!
+                .from('user_settings')
+                .select('encryption_key')
+                .eq('user_id', req.user!.id)
+                .single();
+
+            if (userSettings && !userSettings.encryption_key) {
+                // User doesn't have encryption key yet - persist it now
+                logger.info('User missing encryption key, persisting immediately', { userId: req.user!.id });
+                await req.supabase!
+                    .from('user_settings')
+                    .update({ encryption_key: currentKey })
+                    .eq('user_id', req.user!.id);
+                logger.info('✓ Encryption key persisted to user');
+            }
+        }
+
         const imapService = getImapService();
         const imapConfig = {
             host: imapHost,

package/api/src/services/processor.ts

@@ -813,10 +813,22 @@ export class EmailProcessorService {
 
             if (eventLogger) await eventLogger.info('Processing', `Background processing: ${email.subject}`, undefined, email.id);
 
+            // Timeline tracking for performance metrics
+            const timeline = {
+                start: Date.now(),
+                parsed: 0,
+                metadata_extracted: 0,
+                llm_analysis: 0,
+                validation: 0,
+                actions: 0,
+                end: 0
+            };
+
             // 2. Read content from disk and parse with mailparser
             if (!email.file_path) throw new Error('No file path found for email');
             const rawMime = await this.storageService.readEmail(email.file_path);
             const parsed = await simpleParser(rawMime);
+            timeline.parsed = Date.now();
 
             // Extract clean content (prioritize text)
             const cleanContent = parsed.text || parsed.textAsHtml || '';
@@ -835,6 +847,40 @@ export class EmailProcessorService {
                 sender_priority: email.sender_priority || undefined,
                 thread_id: email.thread_id || undefined,
             };
+            timeline.metadata_extracted = Date.now();
+
+            // Calculate email age
+            const emailAge = email.date ? Math.floor((Date.now() - new Date(email.date).getTime()) / (1000 * 60 * 60 * 24)) : 0;
+
+            // Check for VIP sender and learned patterns (for metadata event)
+            const senderDomain = email.sender?.split('@')[1];
+            const learnedCategory = senderDomain && settings?.category_patterns?.[senderDomain];
+            const isVIP = email.sender && settings?.vip_senders?.includes(email.sender);
+
+            // Log Email Metadata event for trace UI
+            if (eventLogger) {
+                await eventLogger.info('Email Context',
+                    `Email metadata extracted: ${emailAge} days old, ${metadata.recipient_type || 'TO'} recipient`,
+                    {
+                        email_age_days: emailAge,
+                        email_date: email.date,
+                        recipient_type: metadata.recipient_type || 'to',
+                        is_automated: metadata.is_automated || false,
+                        has_unsubscribe: metadata.has_unsubscribe || false,
+                        is_reply: metadata.is_reply || false,
+                        is_thread: !!metadata.thread_id,
+                        sender_priority: metadata.sender_priority || 'normal',
+                        mailer: metadata.mailer || 'unknown',
+                        vip_sender: isVIP || false,
+                        vip_sender_email: isVIP ? email.sender : null,
+                        learned_category: learnedCategory || null,
+                        learned_domain: learnedCategory ? senderDomain : null,
+                        sender: email.sender,
+                        subject: email.subject
+                    },
+                    email.id
+                );
+            }
 
             // 3. Fetch account for action execution
             const { data: account } = await this.supabase
@@ -971,9 +1017,13 @@ export class EmailProcessorService {
             if (!analysis) {
                 throw new Error('AI analysis returned no result');
             }
+            timeline.llm_analysis = Date.now();
 
             // PHASE 2: Post-LLM Validation - Filter out incorrectly matched rules
             // This catches any LLM hallucinations or fuzzy matches that don't meet actual conditions
+            // Track detailed validation results for trace UI
+            const validationDetails: any[] = [];
+
             if (analysis.matched_rules && analysis.matched_rules.length > 0 && rules) {
                 const emailAge = email.date ? Math.floor((Date.now() - new Date(email.date).getTime()) / (1000 * 60 * 60 * 24)) : 0;
                 const validatedMatches = [];
@@ -1012,6 +1062,17 @@ export class EmailProcessorService {
                             min_confidence: minConfidence,
                             email_id: email.id
                         });
+
+                        // Track validation failure for trace UI
+                        validationDetails.push({
+                            rule_name: rule.name,
+                            rule_id: rule.id,
+                            status: 'FILTERED_CONFIDENCE',
+                            confidence: match.confidence,
+                            min_confidence: minConfidence,
+                            reason: `Confidence ${(match.confidence * 100).toFixed(0)}% below threshold ${(minConfidence * 100).toFixed(0)}%`
+                        });
+
                         if (eventLogger) {
                             await eventLogger.info('Validation',
                                 `Rule "${rule.name}" below confidence threshold (${(match.confidence * 100).toFixed(0)}% < ${(minConfidence * 100).toFixed(0)}%)`,
@@ -1049,6 +1110,18 @@ export class EmailProcessorService {
                                 negative_condition: rule.negative_condition,
                                 email_id: email.id
                             });
+
+                            // Track validation failure for trace UI
+                            validationDetails.push({
+                                rule_name: rule.name,
+                                rule_id: rule.id,
+                                status: 'FILTERED_NEGATIVE_CONDITION',
+                                confidence: match.confidence,
+                                min_confidence: minConfidence,
+                                negative_condition: rule.negative_condition,
+                                reason: 'Excluded by negative condition'
+                            });
+
                             if (eventLogger) {
                                 await eventLogger.info('Validation',
                                     `Rule "${rule.name}" excluded by negative condition`,
@@ -1063,8 +1136,28 @@ export class EmailProcessorService {
                     }
 
                     if (isValid && !isExcluded) {
+                        // Track successful validation for trace UI
+                        validationDetails.push({
+                            rule_name: rule.name,
+                            rule_id: rule.id,
+                            status: 'MATCHED',
+                            confidence: match.confidence,
+                            min_confidence: minConfidence,
+                            reasoning: match.reasoning,
+                            reason: 'All conditions met, confidence above threshold'
+                        });
                         validatedMatches.push(match);
-                    } else {
+                    } else if (!isExcluded) {
+                        // Track condition failure for trace UI
+                        validationDetails.push({
+                            rule_name: rule.name,
+                            rule_id: rule.id,
+                            status: 'FILTERED_CONDITIONS',
+                            confidence: match.confidence,
+                            min_confidence: minConfidence,
+                            reason: 'LLM matched but rule conditions not met'
+                        });
+
                         logger.info('Filtered out invalid LLM rule match', {
                             rule_name: rule.name,
                             rule_id: rule.id,
@@ -1090,6 +1183,9 @@ export class EmailProcessorService {
                 analysis.matched_rules = validatedMatches;
             }
 
+            // Mark validation complete
+            timeline.validation = Date.now();
+
             // Log detailed rule evaluation for debugging
             if (eventLogger && rules) {
                 const emailAge = email.date ? Math.floor((Date.now() - new Date(email.date).getTime()) / (1000 * 60 * 60 * 24)) : 0;
@@ -1162,8 +1258,17 @@ export class EmailProcessorService {
                 const learnedCategory = senderDomain && settings?.category_patterns?.[senderDomain];
                 const isVIP = email.sender && settings?.vip_senders?.includes(email.sender);
 
+                // Count validation results
+                const validationSummary = {
+                    llm_matched: validationDetails.length,
+                    final_matched: validationDetails.filter(v => v.status === 'MATCHED').length,
+                    filtered_confidence: validationDetails.filter(v => v.status === 'FILTERED_CONFIDENCE').length,
+                    filtered_negative: validationDetails.filter(v => v.status === 'FILTERED_NEGATIVE_CONDITION').length,
+                    filtered_conditions: validationDetails.filter(v => v.status === 'FILTERED_CONDITIONS').length
+                };
+
                 await eventLogger.info('Rule Evaluation',
-                    `Evaluated ${ruleEvaluations.length} rules: ${analysis.matched_rules.length} matched, ${ruleEvaluations.length - analysis.matched_rules.length} failed`,
+                    `Evaluated ${ruleEvaluations.length} rules: ${validationSummary.final_matched} matched, ${validationSummary.llm_matched - validationSummary.final_matched} filtered`,
                     {
                         ai_analysis: {
                             category: analysis.category,
@@ -1171,6 +1276,8 @@ export class EmailProcessorService {
                             email_age_days: emailAge,
                             summary: analysis.summary
                         },
+                        validation_summary: validationSummary,
+                        validation_details: validationDetails,
                         learned_patterns_applied: {
                             category_override: learnedCategory ? {
                                 domain: senderDomain,
@@ -1373,6 +1480,47 @@ export class EmailProcessorService {
                 );
             }
 
+            // Mark actions complete
+            timeline.actions = Date.now();
+            timeline.end = Date.now();
+
+            // Log performance summary
+            if (eventLogger) {
+                const performanceMetrics = {
+                    total_time_ms: timeline.end - timeline.start,
+                    parse_time_ms: timeline.parsed - timeline.start,
+                    metadata_extraction_ms: timeline.metadata_extracted - timeline.parsed,
+                    llm_analysis_ms: timeline.llm_analysis - timeline.metadata_extracted,
+                    validation_ms: timeline.validation - timeline.llm_analysis,
+                    actions_ms: timeline.actions - timeline.validation,
+                    finalization_ms: timeline.end - timeline.actions
+                };
+
+                const breakdown = [
+                    `Parse: ${performanceMetrics.parse_time_ms}ms`,
+                    `Metadata: ${performanceMetrics.metadata_extraction_ms}ms`,
+                    `LLM: ${performanceMetrics.llm_analysis_ms}ms`,
+                    `Validation: ${performanceMetrics.validation_ms}ms`,
+                    `Actions: ${performanceMetrics.actions_ms}ms`
+                ].join(', ');
+
+                await eventLogger.info('Performance',
+                    `Completed in ${performanceMetrics.total_time_ms}ms (${breakdown})`,
+                    {
+                        timeline: performanceMetrics,
+                        stages: {
+                            parse: { duration_ms: performanceMetrics.parse_time_ms, percent: ((performanceMetrics.parse_time_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) },
+                            metadata: { duration_ms: performanceMetrics.metadata_extraction_ms, percent: ((performanceMetrics.metadata_extraction_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) },
+                            llm: { duration_ms: performanceMetrics.llm_analysis_ms, percent: ((performanceMetrics.llm_analysis_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) },
+                            validation: { duration_ms: performanceMetrics.validation_ms, percent: ((performanceMetrics.validation_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) },
+                            actions: { duration_ms: performanceMetrics.actions_ms, percent: ((performanceMetrics.actions_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) },
+                            finalization: { duration_ms: performanceMetrics.finalization_ms, percent: ((performanceMetrics.finalization_ms / performanceMetrics.total_time_ms) * 100).toFixed(1) }
+                        }
+                    },
+                    email.id
+                );
+            }
+
             // Mark log as success
             if (log) {
                 await this.supabase

package/api/src/utils/encryption.ts

@@ -15,6 +15,10 @@ export function setEncryptionKey(hexKey: string) {
     // console.debug('Encryption key updated from persistence');
 }
 
+export function getEncryptionKeyHex(): string | null {
+    return secretKey ? secretKey.toString('hex') : null;
+}
+
 function getKey(): Buffer {
     if (!secretKey) {
         throw new Error(

package/dist/api/server.js

@@ -13,7 +13,7 @@ import routes from './src/routes/index.js';
 import { logger } from './src/utils/logger.js';
 import { getServerSupabase, getServiceRoleSupabase } from './src/services/supabase.js';
 import { startScheduler, stopScheduler } from './src/services/scheduler.js';
-import { setEncryptionKey } from './src/utils/encryption.js';
+import { setEncryptionKey, getEncryptionKeyHex } from './src/utils/encryption.js';
 // Initialize Persistence Encryption
 // NOTE: In RealTimeX Desktop sandbox, all config is stored in Supabase (no .env files)
 async function initializePersistenceEncryption() {
@@ -71,6 +71,38 @@ async function initializePersistenceEncryption() {
         logger.warn('⚠ IMAP features may not work properly');
     }
 }
+// Periodic sync: Persist in-memory encryption key to users without one
+// This handles the "first user" case and any users created before key was persisted
+async function syncEncryptionKeyToUsers() {
+    try {
+        const supabase = getServiceRoleSupabase();
+        if (!supabase)
+            return;
+        const currentKey = getEncryptionKeyHex();
+        if (!currentKey)
+            return; // No key in memory yet
+        // Find users without encryption key
+        const { data: usersWithoutKey, error } = await supabase
+            .from('user_settings')
+            .select('user_id')
+            .is('encryption_key', null)
+            .limit(100);
+        if (error || !usersWithoutKey || usersWithoutKey.length === 0) {
+            return; // No users to update
+        }
+        logger.info(`Found ${usersWithoutKey.length} user(s) without encryption key, persisting...`);
+        // Update all users without a key
+        const updates = usersWithoutKey.map(user => supabase
+            .from('user_settings')
+            .update({ encryption_key: currentKey })
+            .eq('user_id', user.user_id));
+        await Promise.all(updates);
+        logger.info(`✓ Persisted encryption key to ${usersWithoutKey.length} user(s)`);
+    }
+    catch (err) {
+        logger.warn('Error syncing encryption key to users:', { error: err });
+    }
+}
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 // Validate configuration
@@ -331,6 +363,8 @@ process.on('SIGINT', shutdown);
 const startServer = async () => {
     // Try to load encryption key before accepting requests
     await initializePersistenceEncryption();
+    // Initial sync to persist key to any users without one (handles first user case)
+    setTimeout(() => syncEncryptionKeyToUsers(), 500); // Wait 500ms for DB to be ready
     const server = app.listen(config.port, () => {
         const url = `http://localhost:${config.port}`;
         logger.info(`Server running at ${url}`, {
@@ -341,6 +375,18 @@ const startServer = async () => {
         if (getServerSupabase()) {
             startScheduler();
         }
+        // Periodic sync: persist encryption key to new users every 30 seconds for first 5 minutes
+        // This ensures first user gets the key even if created after server start
+        let syncCount = 0;
+        const syncInterval = setInterval(() => {
+            syncEncryptionKeyToUsers();
+            syncCount++;
+            if (syncCount >= 10) { // 10 * 30s = 5 minutes
+                clearInterval(syncInterval);
+                // After 5 minutes, sync less frequently (every 5 minutes)
+                setInterval(syncEncryptionKeyToUsers, 5 * 60 * 1000);
+            }
+        }, 30 * 1000); // Every 30 seconds
     });
     // Handle server errors
     server.on('error', (error) => {

package/dist/api/src/middleware/index.js

@@ -1,4 +1,4 @@
 export { errorHandler, asyncHandler, AppError, ValidationError, AuthenticationError, AuthorizationError, NotFoundError, RateLimitError } from './errorHandler.js';
 export { authMiddleware, optionalAuth, requireRole } from './auth.js';
-export { rateLimit, authRateLimit, apiRateLimit, syncRateLimit } from './rateLimit.js';
+export { rateLimit, authRateLimit, apiRateLimit, syncRateLimit, connectionRateLimit } from './rateLimit.js';
 export { validateBody, validateQuery, validateParams, schemas } from './validation.js';

package/dist/api/src/middleware/rateLimit.js

@@ -55,3 +55,8 @@ export const syncRateLimit = rateLimit({
     windowMs: 60 * 1000, // 1 minute
     max: 20, // 20 sync requests per minute
 });
+// Connection testing rate limit (lenient for troubleshooting)
+export const connectionRateLimit = rateLimit({
+    windowMs: 15 * 60 * 1000, // 15 minutes
+    max: 30, // 30 connection attempts per 15 minutes
+});

package/dist/api/src/routes/auth.js

@@ -1,17 +1,38 @@
 import { Router } from 'express';
 import { asyncHandler, ValidationError } from '../middleware/errorHandler.js';
 import { authMiddleware } from '../middleware/auth.js';
-import { authRateLimit } from '../middleware/rateLimit.js';
+import { authRateLimit, connectionRateLimit } from '../middleware/rateLimit.js';
 import { validateBody, schemas } from '../middleware/validation.js';
 import { getGmailService } from '../services/gmail.js';
 import { getMicrosoftService } from '../services/microsoft.js';
 import { getImapService } from '../services/imap-service.js';
 import { createLogger } from '../utils/logger.js';
+import { getEncryptionKeyHex } from '../utils/encryption.js';
 const router = Router();
 const logger = createLogger('AuthRoutes');
 // IMAP/SMTP Connection
-router.post('/imap/connect', authRateLimit, authMiddleware, validateBody(schemas.imapConnect), asyncHandler(async (req, res) => {
+router.post('/imap/connect', connectionRateLimit, // More lenient for connection testing (30 attempts / 15 min)
+authMiddleware, validateBody(schemas.imapConnect), asyncHandler(async (req, res) => {
     const { email, password, imapHost, imapPort, imapSecure, smtpHost, smtpPort, smtpSecure } = req.body;
+    // CRITICAL FIX: Ensure user has encryption key before IMAP operations
+    // This handles the "first user connects IMAP immediately after signup" case
+    const currentKey = getEncryptionKeyHex();
+    if (currentKey) {
+        const { data: userSettings } = await req.supabase
+            .from('user_settings')
+            .select('encryption_key')
+            .eq('user_id', req.user.id)
+            .single();
+        if (userSettings && !userSettings.encryption_key) {
+            // User doesn't have encryption key yet - persist it now
+            logger.info('User missing encryption key, persisting immediately', { userId: req.user.id });
+            await req.supabase
+                .from('user_settings')
+                .update({ encryption_key: currentKey })
+                .eq('user_id', req.user.id);
+            logger.info('✓ Encryption key persisted to user');
+        }
+    }
     const imapService = getImapService();
     const imapConfig = {
         host: imapHost,