@realtimex/email-automator 2.4.5 → 2.6.4

package/api/src/services/storage.ts

@@ -0,0 +1,88 @@
+import fs from 'fs/promises';
+import path from 'path';
+import os from 'os';
+import { createLogger } from '../utils/logger.js';
+
+const logger = createLogger('StorageService');
+
+export class StorageService {
+    private defaultPath: string;
+
+    constructor() {
+        // Default to a folder in the user's home directory or current project
+        // Using project-relative path for now as discussed
+        this.defaultPath = path.resolve(process.cwd(), 'data', 'emails');
+    }
+
+    /**
+     * Ensures the storage directory exists and is writable.
+     */
+    async ensureDirectory(customPath?: string | null): Promise<string> {
+        const targetPath = customPath || this.defaultPath;
+        try {
+            await fs.mkdir(targetPath, { recursive: true });
+            // Test writability
+            const testFile = path.join(targetPath, '.write_test');
+            await fs.writeFile(testFile, 'ok');
+            await fs.unlink(testFile);
+            return targetPath;
+        } catch (error) {
+            logger.error('Storage directory validation failed', error, { targetPath });
+            throw new Error(`Storage path "${targetPath}" is not accessible or writable.`);
+        }
+    }
+
+    /**
+     * Saves raw email content to disk.
+     * Returns the absolute path to the saved file.
+     */
+    async saveEmail(content: string, filename: string, customPath?: string | null): Promise<string> {
+        const baseDir = await this.ensureDirectory(customPath);
+        const filePath = path.join(baseDir, filename);
+        
+        try {
+            await fs.writeFile(filePath, content, 'utf8');
+            logger.debug('Email saved to disk', { filePath });
+            return filePath;
+        } catch (error) {
+            logger.error('Failed to save email to disk', error, { filePath });
+            throw error;
+        }
+    }
+
+    /**
+     * Reads email content from disk.
+     */
+    async readEmail(filePath: string): Promise<string> {
+        try {
+            return await fs.readFile(filePath, 'utf8');
+        } catch (error) {
+            logger.error('Failed to read email from disk', error, { filePath });
+            throw error;
+        }
+    }
+
+    /**
+     * Deletes email from disk.
+     */
+    async deleteEmail(filePath: string): Promise<void> {
+        try {
+            await fs.unlink(filePath);
+            logger.debug('Email deleted from disk', { filePath });
+        } catch (error) {
+            // If file doesn't exist, we don't care much
+            if ((error as any).code !== 'ENOENT') {
+                logger.warn('Failed to delete email from disk', { error, filePath });
+            }
+        }
+    }
+}
+
+let storageService: StorageService | null = null;
+
+export function getStorageService(): StorageService {
+    if (!storageService) {
+        storageService = new StorageService();
+    }
+    return storageService;
+}

package/api/src/services/supabase.ts

@@ -103,6 +103,12 @@ export interface Email {
     action_taken: string | null; // Deprecated
     actions_taken?: string[];
     created_at: string;
+    email_accounts?: EmailAccount;
+    // ETL fields
+    file_path?: string | null;
+    processing_status: 'pending' | 'processing' | 'completed' | 'failed';
+    processing_error?: string | null;
+    retry_count: number;
 }
 
 export interface Rule {

package/api/src/utils/contentCleaner.ts

@@ -5,11 +5,12 @@ export class ContentCleaner {
      */
     static cleanEmailBody(text: string): string {
         if (!text) return "";
+        const originalText = text;
 
         // 0. Lightweight HTML -> Markdown Conversion
         
         // Structure: <br>, <p> -> Newlines
-        text = text.replace(/<br\s*\/?\?>/gi, '\n');
+        text = text.replace(/<br\s*\/?>/gi, '\n');
         text = text.replace(/<\/p>/gi, '\n\n');
         text = text.replace(/<p.*?>/gi, ''); // Open p tags just gone
         
@@ -72,13 +73,13 @@ export class ContentCleaner {
             }
                 
             // 3. Check for specific reply separators
-            // If we hit a reply header, we truncate the rest (Aggressive strategy per Python code)
+            // If we hit a reply header, we truncate the rest
             if (/^On .* wrote:$/i.test(lineStripped)) {
                 break;
             }
 
-            // 4. Footer removal (simple check on short lines)
-            if (lineStripped.length < 100) {
+            // 4. Footer removal (only on very short lines to avoid stripping body content)
+            if (lineStripped.length < 60) {
                 let isFooter = false;
                 for (const pattern of footerPatterns) {
                     if (pattern.test(lineStripped)) {
@@ -97,11 +98,15 @@ export class ContentCleaner {
         // Reassemble
         text = cleanedLines.join('\n');
         
+        // Safety Fallback: If cleaning stripped everything, return original (truncated)
+        if (!text.trim() || text.length < 10) {
+            text = originalText.substring(0, 3000);
+        }
+
         // Collapse multiple newlines
         text = text.replace(/\n{3,}/g, '\n\n');
 
-        // Sanitize LLM Special Tokens (Prevent Prompt Injection/Confusion)
-        // Break sequences like <|channel|>, [INST], <s>
+        // Sanitize LLM Special Tokens
         text = text.replace(/<\|/g, '< |'); 
         text = text.replace(/\|>/g, '| >');
         text = text.replace(/\[INST\]/gi, '[ INST ]');

package/dist/api/src/routes/emails.js

@@ -3,6 +3,7 @@ import { asyncHandler, NotFoundError } from '../middleware/errorHandler.js';
 import { authMiddleware } from '../middleware/auth.js';
 import { apiRateLimit } from '../middleware/rateLimit.js';
 import { createLogger } from '../utils/logger.js';
+import { getStorageService } from '../services/storage.js';
 const router = Router();
 const logger = createLogger('EmailsRoutes');
 // List emails with pagination and filters
@@ -64,19 +65,44 @@ router.get('/:emailId', authMiddleware, asyncHandler(async (req, res) => {
     }
     res.json({ email: data });
 }));
+// Get raw email content (.eml)
+router.get('/:emailId/raw', authMiddleware, asyncHandler(async (req, res) => {
+    const { emailId } = req.params;
+    const { data: email, error } = await req.supabase
+        .from('emails')
+        .select('file_path, subject, email_accounts!inner(user_id)')
+        .eq('id', emailId)
+        .eq('email_accounts.user_id', req.user.id)
+        .single();
+    if (error || !email || !email.file_path) {
+        throw new NotFoundError('Raw Email');
+    }
+    const storageService = getStorageService();
+    const content = await storageService.readEmail(email.file_path);
+    const filename = `${email.subject || 'email'}.eml`.replace(/[^a-z0-9._-]/gi, '_');
+    res.setHeader('Content-Type', 'message/rfc822');
+    res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+    res.send(content);
+}));
 // Delete email record (not the actual email from provider)
 router.delete('/:emailId', apiRateLimit, authMiddleware, asyncHandler(async (req, res) => {
     const { emailId } = req.params;
     // Verify ownership first
     const { data: email } = await req.supabase
         .from('emails')
-        .select('id, email_accounts!inner(user_id)')
+        .select('id, file_path, email_accounts!inner(user_id)')
         .eq('id', emailId)
         .eq('email_accounts.user_id', req.user.id)
         .single();
     if (!email) {
         throw new NotFoundError('Email');
     }
+    // 1. Delete from disk
+    if (email.file_path) {
+        const storageService = getStorageService();
+        await storageService.deleteEmail(email.file_path);
+    }
+    // 2. Delete from DB
     const { error } = await req.supabase
         .from('emails')
         .delete()

package/dist/api/src/routes/rules.js

@@ -46,7 +46,11 @@ router.post('/', apiRateLimit, authMiddleware, validateBody(schemas.createRule),
 // Update rule
 router.patch('/:ruleId', apiRateLimit, authMiddleware, validateParams(z.object({ ruleId: schemas.uuid })), validateBody(schemas.updateRule), asyncHandler(async (req, res) => {
     const { ruleId } = req.params;
-    const updates = req.body;
+    const updates = { ...req.body };
+    // Ensure legacy action is in sync if actions array is provided
+    if (updates.actions && Array.isArray(updates.actions) && updates.actions.length > 0) {
+        updates.action = updates.actions[0];
+    }
     const { data, error } = await req.supabase
         .from('rules')
         .update(updates)
@@ -58,7 +62,7 @@ router.patch('/:ruleId', apiRateLimit, authMiddleware, validateParams(z.object({
         throw error;
     if (!data)
         throw new NotFoundError('Rule');
-    logger.info('Rule updated', { ruleId, userId: req.user.id });
+    logger.info('Rule updated', { ruleId, actions: data.actions, userId: req.user.id });
     res.json({ rule: data });
 }));
 // Delete rule

package/dist/api/src/services/gmail.js

@@ -113,182 +113,93 @@ export class GmailService {
             throw error;
         return data;
     }
-    async fetchMessages(account, options = {}) {
-        const gmail = await this.getAuthenticatedClient(account);
-        const { maxResults = config.processing.batchSize, query, pageToken } = options;
-        const response = await gmail.users.messages.list({
-            userId: 'me',
-            maxResults,
-            q: query,
-            pageToken,
-        });
-        const messages = [];
-        for (const msg of response.data.messages || []) {
-            if (!msg.id)
-                continue;
-            try {
-                const detail = await gmail.users.messages.get({
-                    userId: 'me',
-                    id: msg.id,
-                    format: 'full',
-                });
-                const parsed = this.parseMessage(detail.data);
-                if (parsed) {
-                    messages.push(parsed);
-                }
-            }
-            catch (error) {
-                logger.warn('Failed to fetch message details', { messageId: msg.id, error });
-            }
-        }
-        return {
-            messages,
-            nextPageToken: response.data.nextPageToken ?? undefined,
-        };
-    }
     /**
-     * Fetch messages in OLDEST-FIRST order using "Fetch IDs → Sort → Hydrate" strategy.
+     * Fetch messages in OLDEST-FIRST order using "Fetch IDs → Reverse → Hydrate" strategy.
      *
-     * Gmail API always returns newest first and doesn't support sorting.
-     * To process oldest emails first (critical for checkpoint-based sync), we:
-     * 1. Fetch ALL message IDs matching the query (lightweight, paginated)
-     * 2. Sort by internalDate ascending (oldest first)
+     * Gmail API always returns newest first. To process absolute oldest emails first:
+     * 1. Fetch ALL message IDs matching the query (lightweight)
+     * 2. Reverse the list (turning Newest-First into Oldest-First)
      * 3. Take first N messages (limit)
-     * 4. Hydrate only those N messages with full details
-     *
-     * This ensures we never skip emails when using max_emails pagination.
+     * 4. Hydrate ONLY those N messages
      */
     async fetchMessagesOldestFirst(account, options) {
-        const { limit, query, maxIdsToFetch = 1000 } = options;
-        // Step 1: Fetch all message IDs (lightweight)
-        const allIds = await this.fetchAllMessageIds(account, query, maxIdsToFetch);
+        const { limit, query } = options;
+        // Step 1: Fetch IDs (No hydration yet, so this is fast)
+        const allIds = await this.fetchAllMessageIds(account, query);
         if (allIds.length === 0) {
             return { messages: [], hasMore: false };
         }
-        logger.debug('Fetched message IDs', { count: allIds.length, query });
-        // Step 2: Sort by internalDate ascending (oldest first)
-        allIds.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
-        // Step 3: Take first N IDs
+        // Step 2: Reverse to get oldest first
+        allIds.reverse();
+        // Step 3: Take the window we need
         const idsToHydrate = allIds.slice(0, limit);
         const hasMore = allIds.length > limit;
-        // Step 4: Hydrate those specific messages
-        const messages = await this.hydrateMessages(account, idsToHydrate.map(m => m.id));
-        // Re-sort hydrated messages by internalDate (maintain order)
-        messages.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
+        logger.debug('Hydrating oldest emails', { totalFound: allIds.length, hydrating: idsToHydrate.length });
+        // Step 4: Hydrate only the target messages
+        const messages = await this.hydrateMessages(account, idsToHydrate);
         return { messages, hasMore };
     }
     /**
      * Fetch all message IDs matching a query (lightweight, paginated).
-     * Uses minimal fields for speed: only id and internalDate.
+     * Collects IDs only to remain fast even for large result sets.
      */
-    async fetchAllMessageIds(account, query, maxIds) {
+    async fetchAllMessageIds(account, query) {
         const gmail = await this.getAuthenticatedClient(account);
-        const results = [];
+        const allIds = [];
         let pageToken;
+        const MAX_IDS = 5000; // Efficient chunk size for finding the "bottom" of recent emails
         do {
             const response = await gmail.users.messages.list({
                 userId: 'me',
                 q: query,
                 pageToken,
                 maxResults: 500, // Max allowed per page
-                // Note: messages.list only returns id and threadId, not internalDate
-                // We need to fetch internalDate separately with minimal format
             });
             const messageRefs = response.data.messages || [];
-            // Fetch internalDate for each message (using metadata format for speed)
             for (const ref of messageRefs) {
-                if (!ref.id || results.length >= maxIds)
-                    break;
-                try {
-                    const msg = await gmail.users.messages.get({
-                        userId: 'me',
-                        id: ref.id,
-                        format: 'minimal', // Only returns id, threadId, labelIds, snippet, internalDate
-                    });
-                    if (msg.data.id && msg.data.internalDate) {
-                        results.push({
-                            id: msg.data.id,
-                            internalDate: msg.data.internalDate,
-                        });
-                    }
-                }
-                catch (error) {
-                    logger.warn('Failed to fetch message metadata', { messageId: ref.id });
-                }
+                if (ref.id)
+                    allIds.push(ref.id);
             }
             pageToken = response.data.nextPageToken ?? undefined;
-        } while (pageToken && results.length < maxIds);
-        return results;
+        } while (pageToken && allIds.length < MAX_IDS);
+        logger.info('Collected matching message IDs', { total: allIds.length, query });
+        return allIds;
     }
     /**
-     * Hydrate specific messages by ID (fetch full details).
+     * Hydrate specific messages by ID (fetch raw RFC822 data).
      */
     async hydrateMessages(account, messageIds) {
         const gmail = await this.getAuthenticatedClient(account);
         const messages = [];
-        for (const id of messageIds) {
-            try {
-                const detail = await gmail.users.messages.get({
-                    userId: 'me',
-                    id,
-                    format: 'full',
-                });
-                const parsed = this.parseMessage(detail.data);
-                if (parsed) {
-                    messages.push(parsed);
+        // Hydrate in small parallel batches to avoid rate limits
+        const BATCH_SIZE = 10;
+        for (let i = 0; i < messageIds.length; i += BATCH_SIZE) {
+            const batch = messageIds.slice(i, i + BATCH_SIZE);
+            const hydrated = await Promise.all(batch.map(async (id) => {
+                try {
+                    const detail = await gmail.users.messages.get({
+                        userId: 'me',
+                        id,
+                        format: 'raw',
+                    });
+                    if (detail.data.raw) {
+                        return {
+                            id: detail.data.id,
+                            threadId: detail.data.threadId,
+                            internalDate: detail.data.internalDate,
+                            raw: detail.data.raw
+                        };
+                    }
                 }
-            }
-            catch (error) {
-                logger.warn('Failed to hydrate message', { messageId: id, error });
-            }
+                catch (error) {
+                    logger.warn('Failed to hydrate message', { messageId: id, error });
+                }
+                return null;
+            }));
+            messages.push(...hydrated.filter((m) => m !== null));
         }
         return messages;
     }
-    parseMessage(message) {
-        if (!message.id || !message.threadId)
-            return null;
-        const headers = message.payload?.headers || [];
-        const getHeader = (name) => headers.find(h => h.name?.toLowerCase() === name.toLowerCase())?.value || '';
-        let body = '';
-        const payload = message.payload;
-        if (payload?.parts) {
-            // Multipart message
-            const textPart = payload.parts.find(p => p.mimeType === 'text/plain');
-            const htmlPart = payload.parts.find(p => p.mimeType === 'text/html');
-            const part = textPart || htmlPart || payload.parts[0];
-            body = this.decodeBody(part?.body?.data);
-        }
-        else if (payload?.body?.data) {
-            body = this.decodeBody(payload.body.data);
-        }
-        return {
-            id: message.id,
-            threadId: message.threadId,
-            subject: getHeader('Subject') || 'No Subject',
-            sender: getHeader('From'),
-            recipient: getHeader('To'),
-            date: getHeader('Date'),
-            internalDate: message.internalDate || '', // Gmail's internal timestamp (ms since epoch)
-            body,
-            snippet: message.snippet || '',
-            headers: {
-                importance: getHeader('Importance') || getHeader('X-Priority'),
-                listUnsubscribe: getHeader('List-Unsubscribe'),
-                autoSubmitted: getHeader('Auto-Submitted'),
-                mailer: getHeader('X-Mailer'),
-            }
-        };
-    }
-    decodeBody(data) {
-        if (!data)
-            return '';
-        try {
-            return Buffer.from(data, 'base64').toString('utf-8');
-        }
-        catch {
-            return '';
-        }
-    }
     async trashMessage(account, messageId) {
         const gmail = await this.getAuthenticatedClient(account);
         await gmail.users.messages.trash({ userId: 'me', id: messageId });

package/dist/api/src/services/intelligence.js

@@ -123,7 +123,8 @@ REQUIRED JSON STRUCTURE:
                 await eventLogger.info('Thinking', `Analyzing email: ${context.subject}`, {
                     model: this.model,
                     system_prompt: systemPrompt,
-                    content_preview: cleanedContent
+                    content_preview: cleanedContent,
+                    content_length: cleanedContent.length
                 }, emailId);
             }
             catch (err) {

package/dist/api/src/services/microsoft.js

@@ -103,9 +103,7 @@ export class MicrosoftService {
     async fetchMessages(account, options = {}) {
         const accessToken = account.access_token || '';
         const { top = 20, skip = 0, filter } = options;
-        // IMPORTANT: Use ascending order to fetch OLDEST emails first
-        // This ensures checkpoint-based pagination works correctly and doesn't skip emails
-        let url = `https://graph.microsoft.com/v1.0/me/messages?$top=${top}&$skip=${skip}&$orderby=receivedDateTime asc&$select=id,conversationId,subject,from,toRecipients,receivedDateTime,body,bodyPreview,importance`;
+        let url = `https://graph.microsoft.com/v1.0/me/messages?$top=${top}&$skip=${skip}&$orderby=receivedDateTime asc&$select=id,conversationId`;
         if (filter) {
             url += `&$filter=${encodeURIComponent(filter)}`;
         }
@@ -121,19 +119,29 @@ export class MicrosoftService {
             throw new Error('Failed to fetch messages from Outlook');
         }
         const data = await response.json();
-        const messages = (data.value || []).map((msg) => ({
-            id: msg.id,
-            conversationId: msg.conversationId,
-            subject: msg.subject || 'No Subject',
-            sender: msg.from?.emailAddress?.address || 'Unknown',
-            recipient: msg.toRecipients?.[0]?.emailAddress?.address || '',
-            date: msg.receivedDateTime,
-            body: msg.body?.content || '',
-            snippet: msg.bodyPreview || '',
-            headers: {
-                importance: msg.importance,
+        const messageRefs = data.value || [];
+        const messages = [];
+        // For each message, fetch the raw MIME content
+        for (const ref of messageRefs) {
+            try {
+                const rawResponse = await fetch(`https://graph.microsoft.com/v1.0/me/messages/${ref.id}/$value`, {
+                    headers: {
+                        Authorization: `Bearer ${accessToken}`,
+                    },
+                });
+                if (rawResponse.ok) {
+                    const rawMime = await rawResponse.text();
+                    messages.push({
+                        id: ref.id,
+                        conversationId: ref.conversationId,
+                        raw: rawMime
+                    });
+                }
+            }
+            catch (error) {
+                logger.warn('Failed to fetch raw content for Outlook message', { messageId: ref.id, error });
             }
-        }));
+        }
         return {
             messages,
             hasMore: !!data['@odata.nextLink'],
@@ -195,13 +203,15 @@ export class MicrosoftService {
     }
     async createDraft(account, originalMessageId, replyContent) {
         const accessToken = account.access_token || '';
-        // Get original message
-        const originalResponse = await fetch(`https://graph.microsoft.com/v1.0/me/messages/${originalMessageId}`, {
+        // Get original message (minimal metadata)
+        const originalResponse = await fetch(`https://graph.microsoft.com/v1.0/me/messages/${originalMessageId}?$select=id,conversationId`, {
             headers: {
                 Authorization: `Bearer ${accessToken}`,
             },
         });
-        const original = await originalResponse.json();
+        if (!originalResponse.ok) {
+            throw new Error('Failed to fetch original message metadata');
+        }
         // Create reply draft
         const response = await fetch(`https://graph.microsoft.com/v1.0/me/messages/${originalMessageId}/createReply`, {
             method: 'POST',