@realtimex/email-automator 2.3.7 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/src/middleware/validation.ts +11 -4
- package/api/src/routes/actions.ts +84 -49
- package/api/src/routes/rules.ts +8 -3
- package/api/src/services/gmail.ts +128 -0
- package/api/src/services/intelligence.ts +11 -6
- package/api/src/services/microsoft.ts +3 -1
- package/api/src/services/processor.ts +89 -58
- package/api/src/services/supabase.ts +2 -1
- package/dist/api/src/middleware/validation.js +11 -4
- package/dist/api/src/routes/actions.js +87 -51
- package/dist/api/src/routes/rules.js +7 -3
- package/dist/api/src/services/gmail.js +99 -0
- package/dist/api/src/services/intelligence.js +11 -6
- package/dist/api/src/services/microsoft.js +3 -1
- package/dist/api/src/services/processor.js +77 -49
- package/dist/assets/index-BFNHevFw.css +1 -0
- package/dist/assets/{index-BQ1uMdFh.js → index-BbSMDtp3.js} +20 -20
- package/dist/index.html +2 -2
- package/package.json +1 -1
- package/supabase/migrations/20260118000000_multi_actions_rules.sql +17 -0
- package/dist/assets/index-Dzi17fx5.css +0 -1
|
@@ -69,11 +69,14 @@ export const schemas = {
|
|
|
69
69
|
accountId: z.string().uuid('Invalid account ID'),
|
|
70
70
|
}),
|
|
71
71
|
|
|
72
|
-
// Action schemas
|
|
72
|
+
// Action schemas - supports both single action (legacy) and actions array
|
|
73
73
|
executeAction: z.object({
|
|
74
74
|
emailId: z.string().uuid('Invalid email ID'),
|
|
75
|
-
action: z.enum(['delete', 'archive', 'draft', 'flag', 'none']),
|
|
75
|
+
action: z.enum(['delete', 'archive', 'draft', 'flag', 'read', 'star', 'none']).optional(),
|
|
76
|
+
actions: z.array(z.enum(['delete', 'archive', 'draft', 'flag', 'read', 'star', 'none'])).optional(),
|
|
76
77
|
draftContent: z.string().optional(),
|
|
78
|
+
}).refine(data => data.action || (data.actions && data.actions.length > 0), {
|
|
79
|
+
message: 'Either action or actions must be provided',
|
|
77
80
|
}),
|
|
78
81
|
|
|
79
82
|
// Migration schemas
|
|
@@ -83,19 +86,23 @@ export const schemas = {
|
|
|
83
86
|
accessToken: z.string().optional(),
|
|
84
87
|
}),
|
|
85
88
|
|
|
86
|
-
// Rule schemas
|
|
89
|
+
// Rule schemas - supports both single action (legacy) and actions array
|
|
87
90
|
createRule: z.object({
|
|
88
91
|
name: z.string().min(1).max(100),
|
|
89
92
|
condition: z.record(z.unknown()),
|
|
90
|
-
action: z.enum(['delete', 'archive', 'draft', 'star', 'read']),
|
|
93
|
+
action: z.enum(['delete', 'archive', 'draft', 'star', 'read']).optional(),
|
|
94
|
+
actions: z.array(z.enum(['delete', 'archive', 'draft', 'star', 'read'])).optional(),
|
|
91
95
|
instructions: z.string().optional(),
|
|
92
96
|
is_enabled: z.boolean().default(true),
|
|
97
|
+
}).refine(data => data.action || (data.actions && data.actions.length > 0), {
|
|
98
|
+
message: 'Either action or actions must be provided',
|
|
93
99
|
}),
|
|
94
100
|
|
|
95
101
|
updateRule: z.object({
|
|
96
102
|
name: z.string().min(1).max(100).optional(),
|
|
97
103
|
condition: z.record(z.unknown()).optional(),
|
|
98
104
|
action: z.enum(['delete', 'archive', 'draft', 'star', 'read']).optional(),
|
|
105
|
+
actions: z.array(z.enum(['delete', 'archive', 'draft', 'star', 'read'])).optional(),
|
|
99
106
|
instructions: z.string().optional(),
|
|
100
107
|
is_enabled: z.boolean().optional(),
|
|
101
108
|
}),
|
|
@@ -11,15 +11,24 @@ import { createLogger } from '../utils/logger.js';
|
|
|
11
11
|
const router = Router();
|
|
12
12
|
const logger = createLogger('ActionRoutes');
|
|
13
13
|
|
|
14
|
-
// Execute action on email
|
|
14
|
+
// Execute action(s) on email - supports both single action and array of actions
|
|
15
15
|
router.post('/execute',
|
|
16
16
|
apiRateLimit,
|
|
17
17
|
authMiddleware,
|
|
18
18
|
validateBody(schemas.executeAction),
|
|
19
19
|
asyncHandler(async (req, res) => {
|
|
20
|
-
const { emailId, action, draftContent } = req.body;
|
|
20
|
+
const { emailId, action, actions, draftContent } = req.body;
|
|
21
21
|
const userId = req.user!.id;
|
|
22
22
|
|
|
23
|
+
// Support both single action (legacy) and actions array
|
|
24
|
+
const actionsToExecute: string[] = actions && actions.length > 0
|
|
25
|
+
? actions
|
|
26
|
+
: (action ? [action] : []);
|
|
27
|
+
|
|
28
|
+
if (actionsToExecute.length === 0) {
|
|
29
|
+
return res.status(400).json({ error: 'No actions specified' });
|
|
30
|
+
}
|
|
31
|
+
|
|
23
32
|
// Fetch email with account info
|
|
24
33
|
const { data: email, error } = await req.supabase!
|
|
25
34
|
.from('emails')
|
|
@@ -37,59 +46,85 @@ router.post('/execute',
|
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
const account = email.email_accounts;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
49
|
+
const actionResults: { action: string; success: boolean; details?: string }[] = [];
|
|
50
|
+
|
|
51
|
+
for (const currentAction of actionsToExecute) {
|
|
52
|
+
try {
|
|
53
|
+
let details = '';
|
|
54
|
+
|
|
55
|
+
if (currentAction === 'none') {
|
|
56
|
+
// Just mark as reviewed, no provider action needed
|
|
57
|
+
} else if (account.provider === 'gmail') {
|
|
58
|
+
const gmailService = getGmailService();
|
|
59
|
+
|
|
60
|
+
if (currentAction === 'delete') {
|
|
61
|
+
await gmailService.trashMessage(account, email.external_id);
|
|
62
|
+
} else if (currentAction === 'archive') {
|
|
63
|
+
await gmailService.archiveMessage(account, email.external_id);
|
|
64
|
+
} else if (currentAction === 'draft') {
|
|
65
|
+
const content = draftContent || email.ai_analysis?.draft_response || '';
|
|
66
|
+
if (content) {
|
|
67
|
+
const draftId = await gmailService.createDraft(account, email.external_id, content);
|
|
68
|
+
details = `Draft created: ${draftId}`;
|
|
69
|
+
}
|
|
70
|
+
} else if (currentAction === 'flag') {
|
|
71
|
+
await gmailService.addLabel(account, email.external_id, ['STARRED']);
|
|
72
|
+
} else if (currentAction === 'read') {
|
|
73
|
+
await gmailService.markAsRead(account, email.external_id);
|
|
74
|
+
} else if (currentAction === 'star') {
|
|
75
|
+
await gmailService.starMessage(account, email.external_id);
|
|
76
|
+
}
|
|
77
|
+
} else if (account.provider === 'outlook') {
|
|
78
|
+
const microsoftService = getMicrosoftService();
|
|
64
79
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
actionResult.details = `Draft created: ${draftId}`;
|
|
80
|
+
if (currentAction === 'delete') {
|
|
81
|
+
await microsoftService.trashMessage(account, email.external_id);
|
|
82
|
+
} else if (currentAction === 'archive') {
|
|
83
|
+
await microsoftService.archiveMessage(account, email.external_id);
|
|
84
|
+
} else if (currentAction === 'draft') {
|
|
85
|
+
const content = draftContent || email.ai_analysis?.draft_response || '';
|
|
86
|
+
if (content) {
|
|
87
|
+
const draftId = await microsoftService.createDraft(account, email.external_id, content);
|
|
88
|
+
details = `Draft created: ${draftId}`;
|
|
89
|
+
}
|
|
90
|
+
} else if (currentAction === 'read') {
|
|
91
|
+
await microsoftService.markAsRead(account, email.external_id);
|
|
92
|
+
} else if (currentAction === 'star' || currentAction === 'flag') {
|
|
93
|
+
await microsoftService.flagMessage(account, email.external_id);
|
|
94
|
+
}
|
|
81
95
|
}
|
|
82
|
-
}
|
|
83
96
|
|
|
84
|
-
|
|
85
|
-
.
|
|
86
|
-
|
|
87
|
-
|
|
97
|
+
// Record this action using atomic append
|
|
98
|
+
await req.supabase!.rpc('append_email_action', {
|
|
99
|
+
p_email_id: emailId,
|
|
100
|
+
p_action: currentAction
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
actionResults.push({ action: currentAction, success: true, details: details || undefined });
|
|
104
|
+
} catch (err) {
|
|
105
|
+
logger.error('Action failed', err, { emailId, action: currentAction });
|
|
106
|
+
actionResults.push({
|
|
107
|
+
action: currentAction,
|
|
108
|
+
success: false,
|
|
109
|
+
details: err instanceof Error ? err.message : 'Unknown error'
|
|
110
|
+
});
|
|
111
|
+
}
|
|
88
112
|
}
|
|
89
113
|
|
|
90
|
-
|
|
114
|
+
// Update legacy column with first action for backward compatibility
|
|
115
|
+
await req.supabase!
|
|
116
|
+
.from('emails')
|
|
117
|
+
.update({ action_taken: actionsToExecute[0] })
|
|
118
|
+
.eq('id', emailId);
|
|
119
|
+
|
|
120
|
+
logger.info('Actions executed', { emailId, actions: actionsToExecute, userId });
|
|
91
121
|
|
|
92
|
-
res.json(
|
|
122
|
+
res.json({
|
|
123
|
+
success: actionResults.every(r => r.success),
|
|
124
|
+
results: actionResults,
|
|
125
|
+
// Legacy field for backward compatibility
|
|
126
|
+
details: actionResults.map(r => r.details).filter(Boolean).join('; ')
|
|
127
|
+
});
|
|
93
128
|
})
|
|
94
129
|
);
|
|
95
130
|
|
package/api/src/routes/rules.ts
CHANGED
|
@@ -31,7 +31,11 @@ router.post('/',
|
|
|
31
31
|
authMiddleware,
|
|
32
32
|
validateBody(schemas.createRule),
|
|
33
33
|
asyncHandler(async (req, res) => {
|
|
34
|
-
const { name, condition, action, is_enabled, instructions, attachments } = req.body;
|
|
34
|
+
const { name, condition, action, actions, is_enabled, instructions, attachments } = req.body;
|
|
35
|
+
|
|
36
|
+
// Use actions array if provided, otherwise use single action for backward compatibility
|
|
37
|
+
const ruleActions = actions && actions.length > 0 ? actions : (action ? [action] : []);
|
|
38
|
+
const primaryAction = ruleActions[0] || 'archive'; // For legacy column
|
|
35
39
|
|
|
36
40
|
const { data, error } = await req.supabase!
|
|
37
41
|
.from('rules')
|
|
@@ -39,7 +43,8 @@ router.post('/',
|
|
|
39
43
|
user_id: req.user!.id,
|
|
40
44
|
name,
|
|
41
45
|
condition,
|
|
42
|
-
action,
|
|
46
|
+
action: primaryAction, // Legacy column
|
|
47
|
+
actions: ruleActions, // New multi-action column
|
|
43
48
|
is_enabled,
|
|
44
49
|
instructions,
|
|
45
50
|
attachments,
|
|
@@ -49,7 +54,7 @@ router.post('/',
|
|
|
49
54
|
|
|
50
55
|
if (error) throw error;
|
|
51
56
|
|
|
52
|
-
logger.info('Rule created', { ruleId: data.id, userId: req.user!.id });
|
|
57
|
+
logger.info('Rule created', { ruleId: data.id, actions: ruleActions, userId: req.user!.id });
|
|
53
58
|
|
|
54
59
|
res.status(201).json({ rule: data });
|
|
55
60
|
})
|
|
@@ -21,6 +21,7 @@ export interface GmailMessage {
|
|
|
21
21
|
sender: string;
|
|
22
22
|
recipient: string;
|
|
23
23
|
date: string;
|
|
24
|
+
internalDate: string; // Gmail's internal timestamp (ms since epoch) - use this for checkpointing
|
|
24
25
|
body: string;
|
|
25
26
|
snippet: string;
|
|
26
27
|
headers: {
|
|
@@ -224,6 +225,132 @@ export class GmailService {
|
|
|
224
225
|
};
|
|
225
226
|
}
|
|
226
227
|
|
|
228
|
+
/**
|
|
229
|
+
* Fetch messages in OLDEST-FIRST order using "Fetch IDs → Sort → Hydrate" strategy.
|
|
230
|
+
*
|
|
231
|
+
* Gmail API always returns newest first and doesn't support sorting.
|
|
232
|
+
* To process oldest emails first (critical for checkpoint-based sync), we:
|
|
233
|
+
* 1. Fetch ALL message IDs matching the query (lightweight, paginated)
|
|
234
|
+
* 2. Sort by internalDate ascending (oldest first)
|
|
235
|
+
* 3. Take first N messages (limit)
|
|
236
|
+
* 4. Hydrate only those N messages with full details
|
|
237
|
+
*
|
|
238
|
+
* This ensures we never skip emails when using max_emails pagination.
|
|
239
|
+
*/
|
|
240
|
+
async fetchMessagesOldestFirst(
|
|
241
|
+
account: EmailAccount,
|
|
242
|
+
options: { limit: number; query?: string; maxIdsToFetch?: number }
|
|
243
|
+
): Promise<{ messages: GmailMessage[]; hasMore: boolean }> {
|
|
244
|
+
const { limit, query, maxIdsToFetch = 1000 } = options;
|
|
245
|
+
|
|
246
|
+
// Step 1: Fetch all message IDs (lightweight)
|
|
247
|
+
const allIds = await this.fetchAllMessageIds(account, query, maxIdsToFetch);
|
|
248
|
+
|
|
249
|
+
if (allIds.length === 0) {
|
|
250
|
+
return { messages: [], hasMore: false };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
logger.debug('Fetched message IDs', { count: allIds.length, query });
|
|
254
|
+
|
|
255
|
+
// Step 2: Sort by internalDate ascending (oldest first)
|
|
256
|
+
allIds.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
|
|
257
|
+
|
|
258
|
+
// Step 3: Take first N IDs
|
|
259
|
+
const idsToHydrate = allIds.slice(0, limit);
|
|
260
|
+
const hasMore = allIds.length > limit;
|
|
261
|
+
|
|
262
|
+
// Step 4: Hydrate those specific messages
|
|
263
|
+
const messages = await this.hydrateMessages(account, idsToHydrate.map(m => m.id));
|
|
264
|
+
|
|
265
|
+
// Re-sort hydrated messages by internalDate (maintain order)
|
|
266
|
+
messages.sort((a, b) => parseInt(a.internalDate) - parseInt(b.internalDate));
|
|
267
|
+
|
|
268
|
+
return { messages, hasMore };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Fetch all message IDs matching a query (lightweight, paginated).
|
|
273
|
+
* Uses minimal fields for speed: only id and internalDate.
|
|
274
|
+
*/
|
|
275
|
+
private async fetchAllMessageIds(
|
|
276
|
+
account: EmailAccount,
|
|
277
|
+
query: string | undefined,
|
|
278
|
+
maxIds: number
|
|
279
|
+
): Promise<{ id: string; internalDate: string }[]> {
|
|
280
|
+
const gmail = await this.getAuthenticatedClient(account);
|
|
281
|
+
const results: { id: string; internalDate: string }[] = [];
|
|
282
|
+
let pageToken: string | undefined;
|
|
283
|
+
|
|
284
|
+
do {
|
|
285
|
+
const response = await gmail.users.messages.list({
|
|
286
|
+
userId: 'me',
|
|
287
|
+
q: query,
|
|
288
|
+
pageToken,
|
|
289
|
+
maxResults: 500, // Max allowed per page
|
|
290
|
+
// Note: messages.list only returns id and threadId, not internalDate
|
|
291
|
+
// We need to fetch internalDate separately with minimal format
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
const messageRefs = response.data.messages || [];
|
|
295
|
+
|
|
296
|
+
// Fetch internalDate for each message (using metadata format for speed)
|
|
297
|
+
for (const ref of messageRefs) {
|
|
298
|
+
if (!ref.id || results.length >= maxIds) break;
|
|
299
|
+
|
|
300
|
+
try {
|
|
301
|
+
const msg = await gmail.users.messages.get({
|
|
302
|
+
userId: 'me',
|
|
303
|
+
id: ref.id,
|
|
304
|
+
format: 'minimal', // Only returns id, threadId, labelIds, snippet, internalDate
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
if (msg.data.id && msg.data.internalDate) {
|
|
308
|
+
results.push({
|
|
309
|
+
id: msg.data.id,
|
|
310
|
+
internalDate: msg.data.internalDate,
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
} catch (error) {
|
|
314
|
+
logger.warn('Failed to fetch message metadata', { messageId: ref.id });
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
pageToken = response.data.nextPageToken ?? undefined;
|
|
319
|
+
} while (pageToken && results.length < maxIds);
|
|
320
|
+
|
|
321
|
+
return results;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Hydrate specific messages by ID (fetch full details).
|
|
326
|
+
*/
|
|
327
|
+
private async hydrateMessages(
|
|
328
|
+
account: EmailAccount,
|
|
329
|
+
messageIds: string[]
|
|
330
|
+
): Promise<GmailMessage[]> {
|
|
331
|
+
const gmail = await this.getAuthenticatedClient(account);
|
|
332
|
+
const messages: GmailMessage[] = [];
|
|
333
|
+
|
|
334
|
+
for (const id of messageIds) {
|
|
335
|
+
try {
|
|
336
|
+
const detail = await gmail.users.messages.get({
|
|
337
|
+
userId: 'me',
|
|
338
|
+
id,
|
|
339
|
+
format: 'full',
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
const parsed = this.parseMessage(detail.data);
|
|
343
|
+
if (parsed) {
|
|
344
|
+
messages.push(parsed);
|
|
345
|
+
}
|
|
346
|
+
} catch (error) {
|
|
347
|
+
logger.warn('Failed to hydrate message', { messageId: id, error });
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return messages;
|
|
352
|
+
}
|
|
353
|
+
|
|
227
354
|
private parseMessage(message: gmail_v1.Schema$Message): GmailMessage | null {
|
|
228
355
|
if (!message.id || !message.threadId) return null;
|
|
229
356
|
|
|
@@ -250,6 +377,7 @@ export class GmailService {
|
|
|
250
377
|
sender: getHeader('From'),
|
|
251
378
|
recipient: getHeader('To'),
|
|
252
379
|
date: getHeader('Date'),
|
|
380
|
+
internalDate: message.internalDate || '', // Gmail's internal timestamp (ms since epoch)
|
|
253
381
|
body,
|
|
254
382
|
snippet: message.snippet || '',
|
|
255
383
|
headers: {
|
|
@@ -48,13 +48,13 @@ export interface EmailContext {
|
|
|
48
48
|
|
|
49
49
|
export class IntelligenceService {
|
|
50
50
|
private client: OpenAI | null = null;
|
|
51
|
-
private model: string;
|
|
51
|
+
private model: string = 'gpt-4o-mini';
|
|
52
52
|
private isConfigured: boolean = false;
|
|
53
53
|
|
|
54
54
|
constructor(overrides?: { model?: string; baseUrl?: string; apiKey?: string }) {
|
|
55
55
|
const apiKey = overrides?.apiKey || config.llm.apiKey;
|
|
56
56
|
const baseUrl = overrides?.baseUrl || config.llm.baseUrl;
|
|
57
|
-
this.model = overrides?.model || config.llm.model;
|
|
57
|
+
this.model = overrides?.model || config.llm.model || 'gpt-4o-mini';
|
|
58
58
|
|
|
59
59
|
// Allow local LLM servers (LM Studio, Ollama) or custom endpoints that don't need API keys
|
|
60
60
|
// We assume any custom baseUrl might be a local/private instance.
|
|
@@ -111,12 +111,16 @@ Do NOT include any greetings, chatter, or special tokens like <|channel|> in you
|
|
|
111
111
|
Return ONLY a valid JSON object.
|
|
112
112
|
|
|
113
113
|
Definitions for Categories:
|
|
114
|
-
- "
|
|
114
|
+
- "spam": Junk, suspicious, unwanted
|
|
115
|
+
- "newsletter": Subscribed content, digests
|
|
115
116
|
- "promotional": Marketing, sales, discounts
|
|
116
117
|
- "transactional": Receipts, shipping, confirmations
|
|
117
118
|
- "social": LinkedIn, friends, social updates
|
|
118
|
-
- "
|
|
119
|
-
- "
|
|
119
|
+
- "support": Help desk, customer service
|
|
120
|
+
- "client": Business clients, customers
|
|
121
|
+
- "internal": Company internal communications
|
|
122
|
+
- "personal": Friends, family, personal matters
|
|
123
|
+
- "other": Anything else
|
|
120
124
|
|
|
121
125
|
Context:
|
|
122
126
|
- Current Date: ${new Date().toISOString()}
|
|
@@ -156,13 +160,14 @@ REQUIRED JSON STRUCTURE:
|
|
|
156
160
|
|
|
157
161
|
let rawResponse = '';
|
|
158
162
|
try {
|
|
159
|
-
//
|
|
163
|
+
// Request JSON response format for reliable parsing
|
|
160
164
|
const response = await this.client!.chat.completions.create({
|
|
161
165
|
model: this.model,
|
|
162
166
|
messages: [
|
|
163
167
|
{ role: 'system', content: systemPrompt },
|
|
164
168
|
{ role: 'user', content: cleanedContent || '[Empty email body]' },
|
|
165
169
|
],
|
|
170
|
+
response_format: { type: 'json_object' },
|
|
166
171
|
temperature: 0.1,
|
|
167
172
|
});
|
|
168
173
|
|
|
@@ -160,7 +160,9 @@ export class MicrosoftService {
|
|
|
160
160
|
const accessToken = account.access_token || '';
|
|
161
161
|
const { top = 20, skip = 0, filter } = options;
|
|
162
162
|
|
|
163
|
-
|
|
163
|
+
// IMPORTANT: Use ascending order to fetch OLDEST emails first
|
|
164
|
+
// This ensures checkpoint-based pagination works correctly and doesn't skip emails
|
|
165
|
+
let url = `https://graph.microsoft.com/v1.0/me/messages?$top=${top}&$skip=${skip}&$orderby=receivedDateTime asc&$select=id,conversationId,subject,from,toRecipients,receivedDateTime,body,bodyPreview,importance`;
|
|
164
166
|
if (filter) {
|
|
165
167
|
url += `&$filter=${encodeURIComponent(filter)}`;
|
|
166
168
|
}
|
|
@@ -167,7 +167,7 @@ export class EmailProcessorService {
|
|
|
167
167
|
if (account.sync_start_date) {
|
|
168
168
|
effectiveStartMs = new Date(account.sync_start_date).getTime();
|
|
169
169
|
}
|
|
170
|
-
|
|
170
|
+
|
|
171
171
|
if (account.last_sync_checkpoint) {
|
|
172
172
|
const checkpointMs = parseInt(account.last_sync_checkpoint);
|
|
173
173
|
if (checkpointMs > effectiveStartMs) {
|
|
@@ -177,33 +177,40 @@ export class EmailProcessorService {
|
|
|
177
177
|
|
|
178
178
|
let query = '';
|
|
179
179
|
if (effectiveStartMs > 0) {
|
|
180
|
-
|
|
180
|
+
// Subtract 1 second to make query inclusive (Gmail's after: is exclusive)
|
|
181
|
+
// This ensures we don't miss emails at the exact checkpoint timestamp
|
|
182
|
+
const startSeconds = Math.floor(effectiveStartMs / 1000) - 1;
|
|
181
183
|
query = `after:${startSeconds}`;
|
|
182
184
|
}
|
|
183
|
-
|
|
184
|
-
if (eventLogger) await eventLogger.info('Fetching', 'Fetching emails from Gmail', { query, batchSize });
|
|
185
185
|
|
|
186
|
-
|
|
187
|
-
|
|
186
|
+
if (eventLogger) await eventLogger.info('Fetching', 'Fetching emails from Gmail (oldest first)', { query, batchSize });
|
|
187
|
+
|
|
188
|
+
// Use "Fetch IDs → Sort → Hydrate" strategy to get OLDEST emails first
|
|
189
|
+
// Gmail API always returns newest first, so we must fetch all IDs, sort, then hydrate
|
|
190
|
+
// This prevents skipping emails when using max_emails pagination
|
|
191
|
+
const { messages, hasMore } = await this.gmailService.fetchMessagesOldestFirst(account, {
|
|
192
|
+
limit: batchSize,
|
|
188
193
|
query: query || undefined,
|
|
194
|
+
maxIdsToFetch: 1000, // Safety limit for ID fetching
|
|
189
195
|
});
|
|
190
|
-
|
|
191
|
-
if (eventLogger) await eventLogger.info('Fetching', `Fetched ${messages.length} emails`);
|
|
192
196
|
|
|
193
|
-
|
|
194
|
-
|
|
197
|
+
if (eventLogger) {
|
|
198
|
+
await eventLogger.info('Fetching', `Fetched ${messages.length} emails (oldest first)${hasMore ? ', more available' : ''}`);
|
|
199
|
+
}
|
|
195
200
|
|
|
201
|
+
// Messages are already sorted oldest-first by fetchMessagesOldestFirst
|
|
196
202
|
let maxInternalDate = account.last_sync_checkpoint ? parseInt(account.last_sync_checkpoint) : 0;
|
|
197
203
|
|
|
198
|
-
for (const message of
|
|
204
|
+
for (const message of messages) {
|
|
199
205
|
try {
|
|
200
206
|
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
201
207
|
|
|
202
|
-
// Checkpoint tracking
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
208
|
+
// Checkpoint tracking: Use Gmail's internalDate for accurate checkpoint
|
|
209
|
+
// internalDate is when Gmail received the email, which matches what after: query uses
|
|
210
|
+
const msgInternalDate = message.internalDate ? parseInt(message.internalDate) : new Date(message.date).getTime();
|
|
211
|
+
if (msgInternalDate > maxInternalDate) {
|
|
212
|
+
maxInternalDate = msgInternalDate;
|
|
213
|
+
|
|
207
214
|
await this.supabase
|
|
208
215
|
.from('email_accounts')
|
|
209
216
|
.update({ last_sync_checkpoint: maxInternalDate.toString() })
|
|
@@ -240,23 +247,28 @@ export class EmailProcessorService {
|
|
|
240
247
|
|
|
241
248
|
let filter = '';
|
|
242
249
|
if (effectiveStartIso) {
|
|
243
|
-
|
|
250
|
+
// Use 'ge' (>=) instead of 'gt' (>) to ensure we don't miss emails at exact checkpoint
|
|
251
|
+
// The duplicate check in processMessage() will skip already-processed emails
|
|
252
|
+
filter = `receivedDateTime ge ${effectiveStartIso}`;
|
|
244
253
|
}
|
|
245
254
|
|
|
246
|
-
if (eventLogger) await eventLogger.info('Fetching', 'Fetching emails from Outlook', { filter, batchSize });
|
|
255
|
+
if (eventLogger) await eventLogger.info('Fetching', 'Fetching emails from Outlook (oldest first)', { filter, batchSize });
|
|
247
256
|
|
|
248
|
-
|
|
257
|
+
// Outlook API now returns messages sorted by receivedDateTime ascending (oldest first)
|
|
258
|
+
// This ensures checkpoint-based pagination works correctly
|
|
259
|
+
const { messages, hasMore } = await this.microsoftService.fetchMessages(account, {
|
|
249
260
|
top: batchSize,
|
|
250
261
|
filter: filter || undefined,
|
|
251
262
|
});
|
|
252
|
-
|
|
253
|
-
if (eventLogger) await eventLogger.info('Fetching', `Fetched ${messages.length} emails`);
|
|
254
263
|
|
|
255
|
-
|
|
264
|
+
if (eventLogger) {
|
|
265
|
+
await eventLogger.info('Fetching', `Fetched ${messages.length} emails (oldest first)${hasMore ? ', more available' : ''}`);
|
|
266
|
+
}
|
|
256
267
|
|
|
268
|
+
// Messages are already sorted oldest-first by the API
|
|
257
269
|
let latestCheckpoint = account.last_sync_checkpoint || '';
|
|
258
270
|
|
|
259
|
-
for (const message of
|
|
271
|
+
for (const message of messages) {
|
|
260
272
|
try {
|
|
261
273
|
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
262
274
|
|
|
@@ -387,37 +399,49 @@ export class EmailProcessorService {
|
|
|
387
399
|
// User-defined and System rules (Unified)
|
|
388
400
|
for (const rule of rules) {
|
|
389
401
|
if (this.matchesCondition(email, analysis, rule.condition as any)) {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
|
|
398
|
-
? {
|
|
399
|
-
model: settings.llm_model,
|
|
400
|
-
baseUrl: settings.llm_base_url,
|
|
401
|
-
apiKey: settings.llm_api_key,
|
|
402
|
-
}
|
|
403
|
-
: undefined
|
|
404
|
-
);
|
|
405
|
-
|
|
406
|
-
const customizedDraft = await intelligenceService.generateDraftReply({
|
|
407
|
-
subject: email.subject || '',
|
|
408
|
-
sender: email.sender || '',
|
|
409
|
-
body: email.body_snippet || '' // Note: body_snippet is used here, might want full body if available
|
|
410
|
-
}, rule.instructions);
|
|
411
|
-
|
|
412
|
-
if (customizedDraft) {
|
|
413
|
-
draftContent = customizedDraft;
|
|
414
|
-
}
|
|
402
|
+
// Get actions array (fallback to single action for backward compatibility)
|
|
403
|
+
const actions = rule.actions && rule.actions.length > 0
|
|
404
|
+
? rule.actions
|
|
405
|
+
: (rule.action ? [rule.action] : []);
|
|
406
|
+
|
|
407
|
+
if (eventLogger && actions.length > 1) {
|
|
408
|
+
await eventLogger.info('Multi-Action', `Executing ${actions.length} actions for rule: ${rule.name}`, { actions }, email.id);
|
|
415
409
|
}
|
|
416
410
|
|
|
417
|
-
|
|
411
|
+
// Execute each action in the rule
|
|
412
|
+
for (const action of actions) {
|
|
413
|
+
let draftContent = undefined;
|
|
414
|
+
|
|
415
|
+
// If the action is to draft, and it has specific instructions, generate it now
|
|
416
|
+
if (action === 'draft' && rule.instructions) {
|
|
417
|
+
if (eventLogger) await eventLogger.info('Thinking', `Generating customized draft based on rule: ${rule.name}`, undefined, email.id);
|
|
418
|
+
|
|
419
|
+
const intelligenceService = getIntelligenceService(
|
|
420
|
+
settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
|
|
421
|
+
? {
|
|
422
|
+
model: settings.llm_model,
|
|
423
|
+
baseUrl: settings.llm_base_url,
|
|
424
|
+
apiKey: settings.llm_api_key,
|
|
425
|
+
}
|
|
426
|
+
: undefined
|
|
427
|
+
);
|
|
428
|
+
|
|
429
|
+
const customizedDraft = await intelligenceService.generateDraftReply({
|
|
430
|
+
subject: email.subject || '',
|
|
431
|
+
sender: email.sender || '',
|
|
432
|
+
body: email.body_snippet || ''
|
|
433
|
+
}, rule.instructions);
|
|
434
|
+
|
|
435
|
+
if (customizedDraft) {
|
|
436
|
+
draftContent = customizedDraft;
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
await this.executeAction(account, email, action, draftContent, eventLogger, `Rule: ${rule.name}`, rule.attachments);
|
|
418
441
|
|
|
419
|
-
|
|
420
|
-
|
|
442
|
+
if (action === 'delete') result.deleted++;
|
|
443
|
+
else if (action === 'draft') result.drafted++;
|
|
444
|
+
}
|
|
421
445
|
}
|
|
422
446
|
}
|
|
423
447
|
}
|
|
@@ -511,13 +535,20 @@ export class EmailProcessorService {
|
|
|
511
535
|
for (const rule of retentionRules) {
|
|
512
536
|
if (this.matchesCondition(email, email.ai_analysis as any, rule.condition as any)) {
|
|
513
537
|
if (eventLogger) await eventLogger.info('Retention', `Applying retention rule: ${rule.name} to ${email.subject}`);
|
|
514
|
-
|
|
515
|
-
//
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
538
|
+
|
|
539
|
+
// Get actions array (fallback to single action for backward compatibility)
|
|
540
|
+
const actions = rule.actions && rule.actions.length > 0
|
|
541
|
+
? rule.actions
|
|
542
|
+
: (rule.action ? [rule.action] : []);
|
|
543
|
+
|
|
544
|
+
// Execute each action
|
|
545
|
+
for (const action of actions) {
|
|
546
|
+
await this.executeAction(account, email, action, undefined, eventLogger, `Retention Rule: ${rule.name}`);
|
|
547
|
+
|
|
548
|
+
if (action === 'delete') result.deleted++;
|
|
549
|
+
else if (action === 'draft') result.drafted++;
|
|
550
|
+
}
|
|
551
|
+
|
|
521
552
|
break; // Only one rule per email
|
|
522
553
|
}
|
|
523
554
|
}
|