@realtimex/email-automator 2.28.0 → 2.28.1

package/api/src/services/processor.ts

@@ -5,6 +5,7 @@ import { config } from '../config/index.js';
 import { getGmailService, GmailMessage } from './gmail.js';
 import { getMicrosoftService, OutlookMessage } from './microsoft.js';
 import { getImapService, EmailMessage as ImapMessage } from './imap-service.js';
+import { initializePersistenceEncryption, isEncryptionReady } from './encryptionInit.js';
 import { getIntelligenceService, EmailAnalysis, ContextAwareAnalysis, RuleContext } from './intelligence.js';
 import { getStorageService } from './storage.js';
 import { generateEmailFilename } from '../utils/filename.js';
@@ -144,179 +145,189 @@ export class EmailProcessorService {
     async syncAccount(accountId: string, userId: string): Promise<ProcessingResult> {
         const result: ProcessingResult = { processed: 0, deleted: 0, drafted: 0, errors: 0 };
 
-        // Reset stop request flag at the start of a manual sync
-        await this.resetStopRequest(userId);
-
-        // Zero-Config UX: Auto-seed default rules for new users (self-healing)
         try {
-            const defaultRuleService = new DefaultRuleService(this.supabase);
-            const { installed } = await defaultRuleService.ensureDefaultRules(userId);
-            if (installed) {
-                logger.info(`Seeded default rules for user ${userId}`);
+            // Ensure encryption is ready (especially for background syncs)
+            if (!isEncryptionReady()) {
+                await initializePersistenceEncryption(this.supabase);
             }
-        } catch (error) {
-            // Don't fail sync if pack installation fails
-            logger.error('Failed to auto-install Universal Pack', error);
-        }
 
-        // Create processing log
-        const { data: log } = await this.supabase
-            .from('processing_logs')
-            .insert({
-                user_id: userId,
-                account_id: accountId,
-                status: 'running',
-            })
-            .select()
-            .single();
+            // Reset stop request flag at the start of a manual sync
+            await this.resetStopRequest(userId);
 
-        try {
-            // Fetch account
-            const { data: account, error: accError } = await this.supabase
-                .from('email_accounts')
-                .select('*')
-                .eq('id', accountId)
-                .eq('user_id', userId)
-                .single();
-
-            if (accError || !account) {
-                throw new Error('Account not found or access denied');
-            }
-
-            logger.info('Retrieved account settings', {
-                accountId: account.id,
-                sync_start_date: account.sync_start_date,
-                last_sync_checkpoint: account.last_sync_checkpoint
-            });
-
-            // Refresh token if needed
-            let refreshedAccount = account;
-            if (account.provider === 'gmail') {
-                refreshedAccount = await this.gmailService.refreshTokenIfNeeded(this.supabase, account);
-            } else if (account.provider === 'outlook') {
-                refreshedAccount = await this.microsoftService.refreshTokenIfNeeded(this.supabase, account);
-            } else if (account.provider === 'imap') {
-                // IMAP doesn't need token refresh, but we could verify connection here if we wanted
-                refreshedAccount = account;
+            // Zero-Config UX: Auto-seed default rules for new users (self-healing)
+            try {
+                const defaultRuleService = new DefaultRuleService(this.supabase);
+                const { installed } = await defaultRuleService.ensureDefaultRules(userId);
+                if (installed) {
+                    logger.info(`Seeded default rules for user ${userId}`);
+                }
+            } catch (error) {
+                // Don't fail sync if pack installation fails
+                logger.error('Failed to auto-install Universal Pack', error);
             }
 
-            // Update status to syncing
-            await this.supabase
-                .from('email_accounts')
-                .update({
-                    last_sync_status: 'syncing',
-                    last_sync_at: new Date().toISOString()
+            // Create processing log
+            const { data: log } = await this.supabase
+                .from('processing_logs')
+                .insert({
+                    user_id: userId,
+                    account_id: accountId,
+                    status: 'running',
                 })
-                .eq('id', accountId);
-
-            // Fetch user's rules
-            const { data: rules } = await this.supabase
-                .from('rules')
-                .select('*')
-                .eq('user_id', userId)
-                .eq('is_enabled', true);
-
-            // Fetch user settings for AI preferences
-            const { data: settings } = await this.supabase
-                .from('user_settings')
-                .select('*')
-                .eq('user_id', userId)
+                .select()
                 .single();
 
-            const eventLogger = log ? new EventLogger(this.supabase, log.id) : null;
-            if (eventLogger) await eventLogger.info('Running', 'Starting sync process');
-
-            // --- STOP CHECK ---
-            if (await this.checkStopRequested(userId, eventLogger)) return result;
-
-            // Process based on provider
             try {
-                if (refreshedAccount.provider === 'gmail') {
-                    await this.processGmailAccount(refreshedAccount, rules || [], settings, result, eventLogger);
-                } else if (refreshedAccount.provider === 'outlook') {
-                    await this.processOutlookAccount(refreshedAccount, rules || [], settings, result, eventLogger);
-                } else if (refreshedAccount.provider === 'imap') {
-                    await this.processImapAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+                // Fetch account
+                const { data: account, error: accError } = await this.supabase
+                    .from('email_accounts')
+                    .select('*')
+                    .eq('id', accountId)
+                    .eq('user_id', userId)
+                    .single();
+
+                if (accError || !account) {
+                    throw new Error('Account not found or access denied');
                 }
-            } catch (providerError) {
-                const providerName = refreshedAccount.provider === 'gmail' ? 'Gmail' :
-                    refreshedAccount.provider === 'outlook' ? 'Outlook' :
-                        'IMAP';
-                throw new Error(`${providerName} Sync Error: ${providerError instanceof Error ? providerError.message : String(providerError)}`);
-            }
-
-            // After processing new emails, run retention rules for this account
-            if (await this.checkStopRequested(userId, eventLogger)) return result;
-            await this.runRetentionRules(refreshedAccount, rules || [], settings, result, eventLogger);
 
-            // Wait for background worker to process the queue (ensure sync is fully complete before event)
-            await this.processQueue(userId, settings, result).catch(err =>
-                logger.error('Background worker failed', err)
-            );
+                logger.info('Retrieved account settings', {
+                    accountId: account.id,
+                    sync_start_date: account.sync_start_date,
+                    last_sync_checkpoint: account.last_sync_checkpoint
+                });
 
-            // Update log and account on success
-            if (log) {
-                if (eventLogger) {
-                    await eventLogger.success('Finished', 'Sync run completed', {
-                        total_processed: result.processed,
-                        deleted: result.deleted,
-                        drafted: result.drafted,
-                        errors: result.errors
-                    });
+                // Refresh token if needed
+                let refreshedAccount = account;
+                if (account.provider === 'gmail') {
+                    refreshedAccount = await this.gmailService.refreshTokenIfNeeded(this.supabase, account);
+                } else if (account.provider === 'outlook') {
+                    refreshedAccount = await this.microsoftService.refreshTokenIfNeeded(this.supabase, account);
+                } else if (account.provider === 'imap') {
+                    // IMAP doesn't need token refresh, but we could verify connection here if we wanted
+                    refreshedAccount = account;
                 }
 
+                // Update status to syncing
                 await this.supabase
-                    .from('processing_logs')
+                    .from('email_accounts')
                     .update({
-                        status: 'success',
-                        completed_at: new Date().toISOString(),
-                        emails_processed: result.processed,
-                        emails_deleted: result.deleted,
-                        emails_drafted: result.drafted,
+                        last_sync_status: 'syncing',
+                        last_sync_at: new Date().toISOString()
                     })
-                    .eq('id', log.id);
-            }
+                    .eq('id', accountId);
+
+                // Fetch user's rules
+                const { data: rules } = await this.supabase
+                    .from('rules')
+                    .select('*')
+                    .eq('user_id', userId)
+                    .eq('is_enabled', true);
+
+                // Fetch user settings for AI preferences
+                const { data: settings } = await this.supabase
+                    .from('user_settings')
+                    .select('*')
+                    .eq('user_id', userId)
+                    .single();
+
+                const eventLogger = log ? new EventLogger(this.supabase, log.id) : null;
+                if (eventLogger) await eventLogger.info('Running', 'Starting sync process');
+
+                // --- STOP CHECK ---
+                if (await this.checkStopRequested(userId, eventLogger)) return result;
+
+                // Process based on provider
+                try {
+                    if (refreshedAccount.provider === 'gmail') {
+                        await this.processGmailAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+                    } else if (refreshedAccount.provider === 'outlook') {
+                        await this.processOutlookAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+                    } else if (refreshedAccount.provider === 'imap') {
+                        await this.processImapAccount(refreshedAccount, rules || [], settings, result, eventLogger);
+                    }
+                } catch (providerError) {
+                    const providerName = refreshedAccount.provider === 'gmail' ? 'Gmail' :
+                        refreshedAccount.provider === 'outlook' ? 'Outlook' :
+                            'IMAP';
+                    throw new Error(`${providerName} Sync Error: ${providerError instanceof Error ? providerError.message : String(providerError)}`);
+                }
 
-            await this.supabase
-                .from('email_accounts')
-                .update({
-                    last_sync_status: 'success',
-                    last_sync_error: null,
-                    sync_start_date: null // Clear manual override once used successfully
-                })
-                .eq('id', accountId);
+                // After processing new emails, run retention rules for this account
+                if (await this.checkStopRequested(userId, eventLogger)) return result;
+                await this.runRetentionRules(refreshedAccount, rules || [], settings, result, eventLogger);
 
-            logger.info('Sync completed and override cleared', { accountId, ...result });
-        } catch (error) {
-            logger.error('Sync failed', error, { accountId });
+                // Wait for background worker to process the queue (ensure sync is fully complete before event)
+                await this.processQueue(userId, settings, result).catch(err =>
+                    logger.error('Background worker failed', err)
+                );
+
+                // Update log and account on success
+                if (log) {
+                    if (eventLogger) {
+                        await eventLogger.success('Finished', 'Sync run completed', {
+                            total_processed: result.processed,
+                            deleted: result.deleted,
+                            drafted: result.drafted,
+                            errors: result.errors
+                        });
+                    }
 
-            const errMsg = error instanceof Error ? error.message : 'Unknown error';
+                    await this.supabase
+                        .from('processing_logs')
+                        .update({
+                            status: 'success',
+                            completed_at: new Date().toISOString(),
+                            emails_processed: result.processed,
+                            emails_deleted: result.deleted,
+                            emails_drafted: result.drafted,
+                        })
+                        .eq('id', log.id);
+                }
 
-            if (log) {
                 await this.supabase
-                    .from('processing_logs')
+                    .from('email_accounts')
                     .update({
-                        status: 'failed',
-                        completed_at: new Date().toISOString(),
-                        error_message: errMsg,
+                        last_sync_status: 'success',
+                        last_sync_error: null,
+                        sync_start_date: null // Clear manual override once used successfully
                     })
-                    .eq('id', log.id);
-            }
+                    .eq('id', accountId);
 
-            await this.supabase
-                .from('email_accounts')
-                .update({
-                    last_sync_status: 'error',
-                    last_sync_error: errMsg
-                })
-                .eq('id', accountId);
+                logger.info('Sync completed and override cleared', { accountId, ...result });
+            } catch (error) {
+                logger.error('Sync failed', error, { accountId });
+
+                const errMsg = error instanceof Error ? error.message : 'Unknown error';
+
+                if (log) {
+                    await this.supabase
+                        .from('processing_logs')
+                        .update({
+                            status: 'failed',
+                            completed_at: new Date().toISOString(),
+                            error_message: errMsg,
+                        })
+                        .eq('id', log.id);
+                }
 
-            // If it's a fatal setup error (e.g. Account not found), throw it
-            if (errMsg.includes('Account not found') || errMsg.includes('access denied')) {
-                throw error;
-            }
+                await this.supabase
+                    .from('email_accounts')
+                    .update({
+                        last_sync_status: 'error',
+                        last_sync_error: errMsg
+                    })
+                    .eq('id', accountId);
+
+                // If it's a fatal setup error (e.g. Account not found), throw it
+                if (errMsg.includes('Account not found') || errMsg.includes('access denied')) {
+                    throw error;
+                }
 
-            // Otherwise, increment error count and return partial results
+                // Otherwise, increment error count and return partial results
+                result.errors++;
+            }
+        } catch (globalError) {
+            logger.error('Global sync execution failed', globalError);
             result.errors++;
         }
 

package/dist/api/server.js

@@ -11,112 +11,9 @@ import { errorHandler } from './src/middleware/errorHandler.js';
 import { apiRateLimit } from './src/middleware/rateLimit.js';
 import routes from './src/routes/index.js';
 import { logger } from './src/utils/logger.js';
-import { getServerSupabase, getServiceRoleSupabase } from './src/services/supabase.js';
+import { getServerSupabase } from './src/services/supabase.js';
 import { startScheduler, stopScheduler } from './src/services/scheduler.js';
-import { setEncryptionKey, getEncryptionKeyHex } from './src/utils/encryption.js';
-// Initialize Persistence Encryption
-// NOTE: In RealTimeX Desktop sandbox, all config is stored in Supabase (no .env files)
-async function initializePersistenceEncryption() {
-    try {
-        const supabase = getServiceRoleSupabase();
-        if (!supabase) {
-            // BYOK mode: Supabase not configured at startup (credentials come via HTTP headers)
-            // Generate encryption key anyway and keep it in memory
-            logger.info('Supabase not configured yet (BYOK mode)');
-            logger.info('Generating encryption key in memory - will persist when Supabase becomes available');
-            const newKey = crypto.randomBytes(32).toString('hex');
-            setEncryptionKey(newKey);
-            logger.info('✓ Encryption key generated and loaded in memory');
-            return;
-        }
-        // 1. Check if ANY user has an encryption key stored
-        // In sandbox mode, encryption key is always in database
-        const { data: users, error } = await supabase
-            .from('user_settings')
-            .select('user_id, encryption_key')
-            .not('encryption_key', 'is', null)
-            .limit(1);
-        if (error) {
-            logger.warn('Failed to query user_settings for encryption key', { error });
-            // Still generate a key for in-memory use
-            const newKey = crypto.randomBytes(32).toString('hex');
-            setEncryptionKey(newKey);
-            logger.info('✓ Generated fallback encryption key due to DB error');
-            return;
-        }
-        if (users && users.length > 0 && users[0].encryption_key) {
-            // Found a persisted key in database
-            logger.info('✓ Loaded encryption key from database');
-            setEncryptionKey(users[0].encryption_key);
-            return;
-        }
-        // 2. No key in database - auto-generate and persist
-        // This happens on first run or after fresh database setup
-        logger.info('No encryption key found in database, generating new key...');
-        const newKey = crypto.randomBytes(32).toString('hex');
-        setEncryptionKey(newKey);
-        // 3. Persist to all existing users in database
-        const { data: allUsers } = await supabase
-            .from('user_settings')
-            .select('user_id')
-            .limit(100);
-        if (allUsers && allUsers.length > 0) {
-            logger.info(`Saving encryption key to database for ${allUsers.length} user(s)...`);
-            // Update all users with the new key
-            const updates = allUsers.map(user => supabase
-                .from('user_settings')
-                .update({ encryption_key: newKey })
-                .eq('user_id', user.user_id));
-            await Promise.all(updates);
-            logger.info('✓ Encryption key saved to database');
-        }
-        else {
-            logger.info('No users found yet, encryption key loaded in memory');
-            logger.info('Key will be persisted when users are created');
-        }
-    }
-    catch (err) {
-        logger.error('Error initializing encryption:', err);
-        // Always ensure we have a key, even if there was an error
-        if (!getEncryptionKeyHex()) {
-            logger.warn('Generating emergency fallback encryption key');
-            const fallbackKey = crypto.randomBytes(32).toString('hex');
-            setEncryptionKey(fallbackKey);
-        }
-    }
-}
-// Periodic sync: Persist in-memory encryption key to users without one
-// This handles the "first user" case and any users created before key was persisted
-async function syncEncryptionKeyToUsers() {
-    try {
-        const supabase = getServiceRoleSupabase();
-        if (!supabase)
-            return;
-        const currentKey = getEncryptionKeyHex();
-        if (!currentKey)
-            return; // No key in memory yet
-        // Find users without encryption key
-        const { data: usersWithoutKey, error } = await supabase
-            .from('user_settings')
-            .select('user_id')
-            .is('encryption_key', null)
-            .limit(100);
-        if (error || !usersWithoutKey || usersWithoutKey.length === 0) {
-            return; // No users to update
-        }
-        logger.info(`Found ${usersWithoutKey.length} user(s) without encryption key, persisting...`);
-        // Update all users without a key
-        const updates = usersWithoutKey.map(user => supabase
-            .from('user_settings')
-            .update({ encryption_key: currentKey })
-            .eq('user_id', user.user_id));
-        await Promise.all(updates);
-        logger.info(`✓ Persisted encryption key to ${usersWithoutKey.length} user(s)`);
-    }
-    catch (err) {
-        logger.warn('Error syncing encryption key to users:', { error: err });
-    }
-}
+import { initializePersistenceEncryption, syncEncryptionKeyToUsers } from './src/services/encryptionInit.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 // Validate configuration

package/dist/api/src/middleware/auth.js

@@ -4,6 +4,7 @@ import { AuthenticationError, AuthorizationError } from './errorHandler.js';
 import { createLogger, Logger } from '../utils/logger.js';
 const logger = createLogger('AuthMiddleware');
 import { getServerSupabase, isValidUrl } from '../services/supabase.js';
+import { initializePersistenceEncryption, isEncryptionReady } from '../services/encryptionInit.js';
 // Check if anon key looks valid (JWT or publishable key format)
 function isValidAnonKey(key) {
     if (!key)
@@ -32,6 +33,17 @@ export async function authMiddleware(req, _res, next) {
         const isEnvKeyValid = !!envKey && envKey.length > 0;
         const supabaseUrl = isEnvUrlValid ? envUrl : (headerConfig?.url || '');
         const supabaseAnonKey = isEnvKeyValid ? envKey : (headerConfig?.anonKey || '');
+        // If encryption is not ready, try to initialize it using available Supabase config
+        if (!isEncryptionReady() && supabaseUrl && supabaseAnonKey) {
+            // Note: Ideally we use service role to read encryption_key from user_settings,
+            // but even with anon key it might work if RLS allows or if we just use it 
+            // to check for existence of keys.
+            const initClient = createClient(supabaseUrl, supabaseAnonKey, {
+                auth: { autoRefreshToken: false, persistSession: false },
+            });
+            // Run in background to not block auth
+            initializePersistenceEncryption(initClient).catch(err => logger.warn('Failed to initialize encryption in auth middleware', { error: err.message }));
+        }
         // Development bypass: skip auth if DISABLE_AUTH=true in non-production
         if (config.security.disableAuth && !config.isProduction) {
             logger.warn('Auth disabled for development - creating mock user');

package/dist/api/src/routes/auth.js

@@ -7,57 +7,15 @@ import { getGmailService } from '../services/gmail.js';
 import { getMicrosoftService } from '../services/microsoft.js';
 import { getImapService } from '../services/imap-service.js';
 import { createLogger } from '../utils/logger.js';
-import { getEncryptionKeyHex, setEncryptionKey } from '../utils/encryption.js';
+import { initializePersistenceEncryption } from '../services/encryptionInit.js';
 const router = Router();
 const logger = createLogger('AuthRoutes');
 // IMAP/SMTP Connection
 router.post('/imap/connect', connectionRateLimit, // More lenient for connection testing (30 attempts / 15 min)
 authMiddleware, validateBody(schemas.imapConnect), asyncHandler(async (req, res) => {
     const { email, password, imapHost, imapPort, imapSecure, smtpHost, smtpPort, smtpSecure } = req.body;
-    // CRITICAL FIX: Sync encryption key between server memory and database
-    // In BYOK mode, database trigger generates key but server has random key in memory
-    // We need to sync them: prefer database key (persistent) over server key (ephemeral)
-    const { data: userSettings, error: fetchError } = await req.supabase
-        .from('user_settings')
-        .select('encryption_key')
-        .eq('user_id', req.user.id)
-        .single();
-    if (fetchError) {
-        logger.error('Failed to fetch user settings', { error: fetchError });
-        throw new ValidationError('Failed to load user settings. Please try again.');
-    }
-    const currentServerKey = getEncryptionKeyHex();
-    const databaseKey = userSettings?.encryption_key;
-    logger.info('Encryption key sync check', {
-        hasServerKey: !!currentServerKey,
-        hasDatabaseKey: !!databaseKey,
-        userId: req.user.id
-    });
-    if (databaseKey) {
-        // Database has a key - use it (it's the source of truth)
-        if (currentServerKey !== databaseKey) {
-            logger.info('Loading encryption key from database (overriding server memory)');
-            setEncryptionKey(databaseKey);
-        }
-    }
-    else if (currentServerKey) {
-        // Server has key but database doesn't - persist to database
-        logger.info('Database missing encryption key, persisting server key');
-        const { error: updateError } = await req.supabase
-            .from('user_settings')
-            .update({ encryption_key: currentServerKey })
-            .eq('user_id', req.user.id);
-        if (updateError) {
-            logger.error('Failed to persist encryption key', { error: updateError });
-            throw new ValidationError('Failed to initialize encryption. Please try again.');
-        }
-        logger.info('✓ Encryption key persisted to database');
-    }
-    else {
-        // Neither server nor database has key - this should never happen with new trigger
-        logger.error('No encryption key found in server OR database!');
-        throw new ValidationError('Encryption not initialized. Please contact support.');
-    }
+    // Reconcile encryption key with database before saving credentials
+    await initializePersistenceEncryption(req.supabase);
     const imapService = getImapService();
     const imapConfig = {
         host: imapHost,