@realtimex/email-automator 2.4.0 → 2.6.2

package/api/src/routes/emails.ts

@@ -3,6 +3,7 @@ import { asyncHandler, NotFoundError } from '../middleware/errorHandler.js';
 import { authMiddleware } from '../middleware/auth.js';
 import { apiRateLimit } from '../middleware/rateLimit.js';
 import { createLogger } from '../utils/logger.js';
+import { getStorageService } from '../services/storage.js';
 
 const router = Router();
 const logger = createLogger('EmailsRoutes');
@@ -19,8 +20,15 @@ router.get('/',
             account_id,
             action_taken,
             search,
+            sort_by = 'date',
+            sort_order = 'desc'
         } = req.query;
 
+        // Validate sort params
+        const validSortFields = ['date', 'created_at'];
+        const sortField = validSortFields.includes(sort_by as string) ? sort_by as string : 'date';
+        const isAscending = sort_order === 'asc';
+
         let query = req.supabase!
             .from('emails')
             .select(`
@@ -28,7 +36,7 @@ router.get('/',
                 email_accounts!inner(id, user_id, email_address, provider)
             `, { count: 'exact' })
             .eq('email_accounts.user_id', req.user!.id)
-            .order('date', { ascending: false })
+            .order(sortField, { ascending: isAscending })
             .range(
                 parseInt(offset as string, 10),
                 parseInt(offset as string, 10) + parseInt(limit as string, 10) - 1
@@ -88,6 +96,34 @@ router.get('/:emailId',
     })
 );
 
+// Get raw email content (.eml)
+router.get('/:emailId/raw',
+    authMiddleware,
+    asyncHandler(async (req, res) => {
+        const { emailId } = req.params;
+
+        const { data: email, error } = await req.supabase!
+            .from('emails')
+            .select('file_path, subject, email_accounts!inner(user_id)')
+            .eq('id', emailId)
+            .eq('email_accounts.user_id', req.user!.id)
+            .single();
+
+        if (error || !email || !email.file_path) {
+            throw new NotFoundError('Raw Email');
+        }
+
+        const storageService = getStorageService();
+        const content = await storageService.readEmail(email.file_path);
+
+        const filename = `${email.subject || 'email'}.eml`.replace(/[^a-z0-9._-]/gi, '_');
+        
+        res.setHeader('Content-Type', 'message/rfc822');
+        res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+        res.send(content);
+    })
+);
+
 // Delete email record (not the actual email from provider)
 router.delete('/:emailId',
     apiRateLimit,
@@ -98,7 +134,7 @@ router.delete('/:emailId',
         // Verify ownership first
         const { data: email } = await req.supabase!
             .from('emails')
-            .select('id, email_accounts!inner(user_id)')
+            .select('id, file_path, email_accounts!inner(user_id)')
             .eq('id', emailId)
             .eq('email_accounts.user_id', req.user!.id)
             .single();
@@ -107,6 +143,13 @@ router.delete('/:emailId',
             throw new NotFoundError('Email');
         }
 
+        // 1. Delete from disk
+        if (email.file_path) {
+            const storageService = getStorageService();
+            await storageService.deleteEmail(email.file_path);
+        }
+
+        // 2. Delete from DB
         const { error } = await req.supabase!
             .from('emails')
             .delete()

package/api/src/services/eventLogger.ts

@@ -46,7 +46,15 @@ export class EventLogger {
         await this.log('action', state, { action, reason }, emailId);
     }
 
-    async error(state: string, error: any, emailId?: string) {
-        await this.log('error', state, { error: error.message || error }, emailId);
+    async error(state: string, errorOrDetails: any, emailId?: string) {
+        let details;
+        if (errorOrDetails instanceof Error) {
+            details = { error: errorOrDetails.message };
+        } else if (typeof errorOrDetails === 'object' && errorOrDetails !== null) {
+            details = errorOrDetails;
+        } else {
+            details = { error: String(errorOrDetails) };
+        }
+        await this.log('error', state, details, emailId);
     }
 }

package/api/src/services/gmail.ts

@@ -17,19 +17,8 @@ export interface RuleAttachment {
 export interface GmailMessage {
     id: string;
     threadId: string;
-    subject: string;
-    sender: string;
-    recipient: string;
-    date: string;
-    internalDate: string; // Gmail's internal timestamp (ms since epoch) - use this for checkpointing
-    body: string;
-    snippet: string;
-    headers: {
-        importance?: string;
-        listUnsubscribe?: string;
-        autoSubmitted?: string;
-        mailer?: string;
-    };
+    internalDate: string;
+    raw: string; // Base64url encoded raw RFC822 message
 }
 
 export interface OAuthCredentials {
@@ -184,102 +173,55 @@ export class GmailService {
         return data;
     }
 
-    async fetchMessages(
-        account: EmailAccount,
-        options: { maxResults?: number; query?: string; pageToken?: string } = {}
-    ): Promise<{ messages: GmailMessage[]; nextPageToken?: string }> {
-        const gmail = await this.getAuthenticatedClient(account);
-        const { maxResults = config.processing.batchSize, query, pageToken } = options;
-
-        const response = await gmail.users.messages.list({
-            userId: 'me',
-            maxResults,
-            q: query,
-            pageToken,
-        });
-
-        const messages: GmailMessage[] = [];
-
-        for (const msg of response.data.messages || []) {
-            if (!msg.id) continue;
-
-            try {
-                const detail = await gmail.users.messages.get({
-                    userId: 'me',
-                    id: msg.id,
-                    format: 'full',
-                });
-
-                const parsed = this.parseMessage(detail.data);
-                if (parsed) {
-                    messages.push(parsed);
-                }
-            } catch (error) {
-                logger.warn('Failed to fetch message details', { messageId: msg.id, error });
-            }
-        }
-
-        return {
-            messages,
-            nextPageToken: response.data.nextPageToken ?? undefined,
-        };
-    }
-
     /**
-     * Fetch messages in OLDEST-FIRST order using "Fetch IDs → Sort → Hydrate" strategy.
+     * Fetch messages in OLDEST-FIRST order using "Fetch IDs → Reverse → Hydrate" strategy.
      *
-     * Gmail API always returns newest first and doesn't support sorting.
-     * To process oldest emails first (critical for checkpoint-based sync), we:
-     * 1. Fetch ALL message IDs matching the query (lightweight, paginated)
-     * 2. Sort by internalDate ascending (oldest first)
+     * Gmail API always returns newest first. To process absolute oldest emails first:
+     * 1. Fetch ALL message IDs matching the query (lightweight)
+     * 2. Reverse the list (turning Newest-First into Oldest-First)
      * 3. Take first N messages (limit)
-     * 4. Hydrate only those N messages with full details
-     *
-     * This ensures we never skip emails when using max_emails pagination.
+     * 4. Hydrate ONLY those N messages
      */
     async fetchMessagesOldestFirst(
         account: EmailAccount,
-        options: { limit: number; query?: string; maxIdsToFetch?: number }
+        options: { limit: number; query?: string }
     ): Promise<{ messages: GmailMessage[]; hasMore: boolean }> {
-        const { limit, query, maxIdsToFetch = 1000 } = options;
+        const { limit, query } = options;
 
-        // Step 1: Fetch all message IDs (lightweight)
-        const allIds = await this.fetchAllMessageIds(account, query, maxIdsToFetch);
+        // Step 1: Fetch IDs (No hydration yet, so this is fast)
+        const allIds = await this.fetchAllMessageIds(account, query);
 
         if (allIds.length === 0) {
             return { messages: [], hasMore: false };
         }
 
-        logger.debug('Fetched message IDs', { count: allIds.length, query });
-
-        // Step 2: Sort by internalDate ascending (oldest first)
-        allIds.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
+        // Step 2: Reverse to get oldest first
+        allIds.reverse();
 
-        // Step 3: Take first N IDs
+        // Step 3: Take the window we need
         const idsToHydrate = allIds.slice(0, limit);
         const hasMore = allIds.length > limit;
 
-        // Step 4: Hydrate those specific messages
-        const messages = await this.hydrateMessages(account, idsToHydrate.map(m => m.id));
+        logger.debug('Hydrating oldest emails', { totalFound: allIds.length, hydrating: idsToHydrate.length });
 
-        // Re-sort hydrated messages by internalDate (maintain order)
-        messages.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
+        // Step 4: Hydrate only the target messages
+        const messages = await this.hydrateMessages(account, idsToHydrate);
 
         return { messages, hasMore };
     }
 
     /**
      * Fetch all message IDs matching a query (lightweight, paginated).
-     * Uses minimal fields for speed: only id and internalDate.
+     * Collects IDs only to remain fast even for large result sets.
      */
     private async fetchAllMessageIds(
         account: EmailAccount,
-        query: string | undefined,
-        maxIds: number
-    ): Promise<{ id: string; internalDate: string }[]> {
+        query: string | undefined
+    ): Promise<string[]> {
         const gmail = await this.getAuthenticatedClient(account);
-        const results: { id: string; internalDate: string }[] = [];
+        const allIds: string[] = [];
         let pageToken: string | undefined;
+        const MAX_IDS = 5000; // Efficient chunk size for finding the "bottom" of recent emails
 
         do {
             const response = await gmail.users.messages.list({
@@ -287,42 +229,22 @@ export class GmailService {
                 q: query,
                 pageToken,
                 maxResults: 500, // Max allowed per page
-                // Note: messages.list only returns id and threadId, not internalDate
-                // We need to fetch internalDate separately with minimal format
             });
 
             const messageRefs = response.data.messages || [];
-
-            // Fetch internalDate for each message (using metadata format for speed)
             for (const ref of messageRefs) {
-                if (!ref.id || results.length >= maxIds) break;
-
-                try {
-                    const msg = await gmail.users.messages.get({
-                        userId: 'me',
-                        id: ref.id,
-                        format: 'minimal', // Only returns id, threadId, labelIds, snippet, internalDate
-                    });
-
-                    if (msg.data.id && msg.data.internalDate) {
-                        results.push({
-                            id: msg.data.id,
-                            internalDate: msg.data.internalDate,
-                        });
-                    }
-                } catch (error) {
-                    logger.warn('Failed to fetch message metadata', { messageId: ref.id });
-                }
+                if (ref.id) allIds.push(ref.id);
             }
 
             pageToken = response.data.nextPageToken ?? undefined;
-        } while (pageToken && results.length < maxIds);
+        } while (pageToken && allIds.length < MAX_IDS);
 
-        return results;
+        logger.info('Collected matching message IDs', { total: allIds.length, query });
+        return allIds;
     }
 
     /**
-     * Hydrate specific messages by ID (fetch full details).
+     * Hydrate specific messages by ID (fetch raw RFC822 data).
      */
     private async hydrateMessages(
         account: EmailAccount,
@@ -331,73 +253,38 @@ export class GmailService {
         const gmail = await this.getAuthenticatedClient(account);
         const messages: GmailMessage[] = [];
 
-        for (const id of messageIds) {
-            try {
-                const detail = await gmail.users.messages.get({
-                    userId: 'me',
-                    id,
-                    format: 'full',
-                });
-
-                const parsed = this.parseMessage(detail.data);
-                if (parsed) {
-                    messages.push(parsed);
+        // Hydrate in small parallel batches to avoid rate limits
+        const BATCH_SIZE = 10;
+        for (let i = 0; i < messageIds.length; i += BATCH_SIZE) {
+            const batch = messageIds.slice(i, i + BATCH_SIZE);
+            const hydrated = await Promise.all(batch.map(async (id) => {
+                try {
+                    const detail = await gmail.users.messages.get({
+                        userId: 'me',
+                        id,
+                        format: 'raw',
+                    });
+
+                    if (detail.data.raw) {
+                        return {
+                            id: detail.data.id!,
+                            threadId: detail.data.threadId!,
+                            internalDate: detail.data.internalDate!,
+                            raw: detail.data.raw
+                        };
+                    }
+                } catch (error) {
+                    logger.warn('Failed to hydrate message', { messageId: id, error });
                 }
-            } catch (error) {
-                logger.warn('Failed to hydrate message', { messageId: id, error });
-            }
+                return null;
+            }));
+            
+            messages.push(...hydrated.filter((m): m is GmailMessage => m !== null));
         }
 
         return messages;
     }
 
-    private parseMessage(message: gmail_v1.Schema$Message): GmailMessage | null {
-        if (!message.id || !message.threadId) return null;
-
-        const headers = message.payload?.headers || [];
-        const getHeader = (name: string) => headers.find(h => h.name?.toLowerCase() === name.toLowerCase())?.value || '';
-
-        let body = '';
-        const payload = message.payload;
-
-        if (payload?.parts) {
-            // Multipart message
-            const textPart = payload.parts.find(p => p.mimeType === 'text/plain');
-            const htmlPart = payload.parts.find(p => p.mimeType === 'text/html');
-            const part = textPart || htmlPart || payload.parts[0];
-            body = this.decodeBody(part?.body?.data);
-        } else if (payload?.body?.data) {
-            body = this.decodeBody(payload.body.data);
-        }
-
-        return {
-            id: message.id,
-            threadId: message.threadId,
-            subject: getHeader('Subject') || 'No Subject',
-            sender: getHeader('From'),
-            recipient: getHeader('To'),
-            date: getHeader('Date'),
-            internalDate: message.internalDate || '', // Gmail's internal timestamp (ms since epoch)
-            body,
-            snippet: message.snippet || '',
-            headers: {
-                importance: getHeader('Importance') || getHeader('X-Priority'),
-                listUnsubscribe: getHeader('List-Unsubscribe'),
-                autoSubmitted: getHeader('Auto-Submitted'),
-                mailer: getHeader('X-Mailer'),
-            }
-        };
-    }
-
-    private decodeBody(data?: string | null): string {
-        if (!data) return '';
-        try {
-            return Buffer.from(data, 'base64').toString('utf-8');
-        } catch {
-            return '';
-        }
-    }
-
     async trashMessage(account: EmailAccount, messageId: string): Promise<void> {
         const gmail = await this.getAuthenticatedClient(account);
         await gmail.users.messages.trash({ userId: 'me', id: messageId });

package/api/src/services/intelligence.ts

@@ -151,7 +151,8 @@ REQUIRED JSON STRUCTURE:
                 await eventLogger.info('Thinking', `Analyzing email: ${context.subject}`, { 
                     model: this.model,
                     system_prompt: systemPrompt,
-                    content_preview: cleanedContent
+                    content_preview: cleanedContent,
+                    content_length: cleanedContent.length
                 }, emailId);
             } catch (err) {
                 console.error('[Intelligence] Failed to log thinking event:', err);
@@ -167,7 +168,7 @@ REQUIRED JSON STRUCTURE:
                     { role: 'system', content: systemPrompt },
                     { role: 'user', content: cleanedContent || '[Empty email body]' },
                 ],
-                response_format: { type: 'json_object' },
+                // response_format: { type: 'json_object' }, // Removed for compatibility
                 temperature: 0.1,
             });
 

package/api/src/services/microsoft.ts

@@ -16,17 +16,7 @@ const GRAPH_SCOPES = [
 export interface OutlookMessage {
     id: string;
     conversationId: string;
-    subject: string;
-    sender: string;
-    recipient: string;
-    date: string;
-    body: string;
-    snippet: string;
-    headers: {
-        importance?: string;
-        listUnsubscribe?: string;
-        autoSubmitted?: string;
-    };
+    raw: string; // MIME content
 }
 
 export interface DeviceCodeResponse {
@@ -160,9 +150,7 @@ export class MicrosoftService {
         const accessToken = account.access_token || '';
         const { top = 20, skip = 0, filter } = options;
 
-        // IMPORTANT: Use ascending order to fetch OLDEST emails first
-        // This ensures checkpoint-based pagination works correctly and doesn't skip emails
-        let url = `https://graph.microsoft.com/v1.0/me/messages?$top=${top}&$skip=${skip}&$orderby=receivedDateTime asc&$select=id,conversationId,subject,from,toRecipients,receivedDateTime,body,bodyPreview,importance`;
+        let url = `https://graph.microsoft.com/v1.0/me/messages?$top=${top}&$skip=${skip}&$orderby=receivedDateTime asc&$select=id,conversationId`;
         if (filter) {
             url += `&$filter=${encodeURIComponent(filter)}`;
         }
@@ -184,19 +172,33 @@ export class MicrosoftService {
         }
 
         const data = await response.json();
-        const messages: OutlookMessage[] = (data.value || []).map((msg: any) => ({
-            id: msg.id,
-            conversationId: msg.conversationId,
-            subject: msg.subject || 'No Subject',
-            sender: msg.from?.emailAddress?.address || 'Unknown',
-            recipient: msg.toRecipients?.[0]?.emailAddress?.address || '',
-            date: msg.receivedDateTime,
-            body: msg.body?.content || '',
-            snippet: msg.bodyPreview || '',
-            headers: {
-                importance: msg.importance,
+        const messageRefs = data.value || [];
+        const messages: OutlookMessage[] = [];
+
+        // For each message, fetch the raw MIME content
+        for (const ref of messageRefs) {
+            try {
+                const rawResponse = await fetch(
+                    `https://graph.microsoft.com/v1.0/me/messages/${ref.id}/$value`,
+                    {
+                        headers: {
+                            Authorization: `Bearer ${accessToken}`,
+                        },
+                    }
+                );
+
+                if (rawResponse.ok) {
+                    const rawMime = await rawResponse.text();
+                    messages.push({
+                        id: ref.id,
+                        conversationId: ref.conversationId,
+                        raw: rawMime
+                    });
+                }
+            } catch (error) {
+                logger.warn('Failed to fetch raw content for Outlook message', { messageId: ref.id, error });
             }
-        }));
+        }
 
         return {
             messages,
@@ -287,16 +289,19 @@ export class MicrosoftService {
     ): Promise<string> {
         const accessToken = account.access_token || '';
 
-        // Get original message
+        // Get original message (minimal metadata)
         const originalResponse = await fetch(
-            `https://graph.microsoft.com/v1.0/me/messages/${originalMessageId}`,
+            `https://graph.microsoft.com/v1.0/me/messages/${originalMessageId}?$select=id,conversationId`,
             {
                 headers: {
                     Authorization: `Bearer ${accessToken}`,
                 },
             }
         );
-        const original = await originalResponse.json();
+        
+        if (!originalResponse.ok) {
+            throw new Error('Failed to fetch original message metadata');
+        }
 
         // Create reply draft
         const response = await fetch(