@realtimex/email-automator 2.4.5 → 2.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/src/routes/emails.ts +37 -1
- package/api/src/routes/rules.ts +7 -2
- package/api/src/services/gmail.ts +53 -166
- package/api/src/services/intelligence.ts +2 -1
- package/api/src/services/microsoft.ts +34 -29
- package/api/src/services/processor.ts +376 -106
- package/api/src/services/storage.ts +88 -0
- package/api/src/services/supabase.ts +6 -0
- package/api/src/utils/contentCleaner.ts +11 -6
- package/dist/api/src/routes/emails.js +27 -1
- package/dist/api/src/routes/rules.js +6 -2
- package/dist/api/src/services/gmail.js +49 -138
- package/dist/api/src/services/intelligence.js +2 -1
- package/dist/api/src/services/microsoft.js +28 -18
- package/dist/api/src/services/processor.js +335 -96
- package/dist/api/src/services/storage.js +81 -0
- package/dist/api/src/utils/contentCleaner.js +10 -6
- package/dist/assets/index-BSHZ3lFn.js +97 -0
- package/dist/assets/index-CRQKk5IW.css +1 -0
- package/dist/index.html +2 -2
- package/package.json +3 -1
- package/supabase/functions/api-v1-accounts/index.ts +28 -7
- package/supabase/migrations/20260118000001_async_etl_storage.sql +24 -0
- package/dist/assets/index-C3PlbplS.css +0 -1
- package/dist/assets/index-DfGa9R7j.js +0 -97
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { SupabaseClient } from '@supabase/supabase-js';
|
|
2
|
+
import { simpleParser } from 'mailparser';
|
|
2
3
|
import { createLogger } from '../utils/logger.js';
|
|
3
4
|
import { config } from '../config/index.js';
|
|
4
5
|
import { getGmailService, GmailMessage } from './gmail.js';
|
|
5
6
|
import { getMicrosoftService, OutlookMessage } from './microsoft.js';
|
|
6
7
|
import { getIntelligenceService, EmailAnalysis } from './intelligence.js';
|
|
8
|
+
import { getStorageService } from './storage.js';
|
|
7
9
|
import { EmailAccount, Email, Rule, ProcessingLog } from './supabase.js';
|
|
8
10
|
import { EventLogger } from './eventLogger.js';
|
|
9
11
|
|
|
@@ -20,6 +22,7 @@ export class EmailProcessorService {
|
|
|
20
22
|
private supabase: SupabaseClient;
|
|
21
23
|
private gmailService = getGmailService();
|
|
22
24
|
private microsoftService = getMicrosoftService();
|
|
25
|
+
private storageService = getStorageService();
|
|
23
26
|
|
|
24
27
|
constructor(supabase: SupabaseClient) {
|
|
25
28
|
this.supabase = supabase;
|
|
@@ -52,6 +55,12 @@ export class EmailProcessorService {
|
|
|
52
55
|
throw new Error('Account not found or access denied');
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
logger.info('Retrieved account settings', {
|
|
59
|
+
accountId: account.id,
|
|
60
|
+
sync_start_date: account.sync_start_date,
|
|
61
|
+
last_sync_checkpoint: account.last_sync_checkpoint
|
|
62
|
+
});
|
|
63
|
+
|
|
55
64
|
// Refresh token if needed
|
|
56
65
|
let refreshedAccount = account;
|
|
57
66
|
if (account.provider === 'gmail') {
|
|
@@ -94,6 +103,11 @@ export class EmailProcessorService {
|
|
|
94
103
|
// After processing new emails, run retention rules for this account
|
|
95
104
|
await this.runRetentionRules(refreshedAccount, rules || [], settings, result, eventLogger);
|
|
96
105
|
|
|
106
|
+
// Trigger background worker (async) to process the queue
|
|
107
|
+
this.processQueue(userId, settings).catch(err =>
|
|
108
|
+
logger.error('Background worker failed', err)
|
|
109
|
+
);
|
|
110
|
+
|
|
97
111
|
// Update log and account on success
|
|
98
112
|
if (log) {
|
|
99
113
|
await this.supabase
|
|
@@ -112,11 +126,12 @@ export class EmailProcessorService {
|
|
|
112
126
|
.from('email_accounts')
|
|
113
127
|
.update({
|
|
114
128
|
last_sync_status: 'success',
|
|
115
|
-
last_sync_error: null
|
|
129
|
+
last_sync_error: null,
|
|
130
|
+
sync_start_date: null // Clear manual override once used successfully
|
|
116
131
|
})
|
|
117
132
|
.eq('id', accountId);
|
|
118
133
|
|
|
119
|
-
logger.info('Sync completed', { accountId, ...result });
|
|
134
|
+
logger.info('Sync completed and override cleared', { accountId, ...result });
|
|
120
135
|
} catch (error) {
|
|
121
136
|
logger.error('Sync failed', error, { accountId });
|
|
122
137
|
|
|
@@ -162,59 +177,81 @@ export class EmailProcessorService {
|
|
|
162
177
|
): Promise<void> {
|
|
163
178
|
const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
|
|
164
179
|
|
|
165
|
-
//
|
|
180
|
+
// Debug: Log account sync settings
|
|
181
|
+
logger.info('Gmail sync settings', {
|
|
182
|
+
accountId: account.id,
|
|
183
|
+
sync_start_date: account.sync_start_date,
|
|
184
|
+
last_sync_checkpoint: account.last_sync_checkpoint,
|
|
185
|
+
sync_max_emails_per_run: account.sync_max_emails_per_run,
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
// Construct query: Use a sliding window for efficiency and determinism
|
|
166
189
|
let effectiveStartMs = 0;
|
|
167
190
|
if (account.sync_start_date) {
|
|
168
191
|
effectiveStartMs = new Date(account.sync_start_date).getTime();
|
|
192
|
+
logger.info('Using sync_start_date override', { effectiveStartMs, date: new Date(effectiveStartMs).toISOString() });
|
|
193
|
+
} else if (account.last_sync_checkpoint) {
|
|
194
|
+
effectiveStartMs = parseInt(account.last_sync_checkpoint);
|
|
195
|
+
logger.info('Using last_sync_checkpoint', { effectiveStartMs, date: new Date(effectiveStartMs).toISOString() });
|
|
169
196
|
}
|
|
170
197
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
198
|
+
// Use a 7-day sliding window. If empty, skip forward (up to 10 weeks per run)
|
|
199
|
+
const windowSizeMs = 7 * 24 * 60 * 60 * 1000;
|
|
200
|
+
const nowMs = Date.now();
|
|
201
|
+
const tomorrowMs = nowMs + (24 * 60 * 60 * 1000);
|
|
202
|
+
|
|
203
|
+
let currentStartMs = effectiveStartMs;
|
|
204
|
+
let messages: GmailMessage[] = [];
|
|
205
|
+
let hasMore = false;
|
|
206
|
+
let attempts = 0;
|
|
207
|
+
const maxAttempts = 10;
|
|
177
208
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
// This ensures we don't miss emails at the exact checkpoint timestamp
|
|
182
|
-
const startSeconds = Math.floor(effectiveStartMs / 1000) - 1;
|
|
183
|
-
query = `after:${startSeconds}`;
|
|
184
|
-
}
|
|
209
|
+
while (attempts < maxAttempts && currentStartMs < nowMs) {
|
|
210
|
+
let effectiveEndMs = currentStartMs + windowSizeMs;
|
|
211
|
+
if (effectiveEndMs > tomorrowMs) effectiveEndMs = tomorrowMs;
|
|
185
212
|
|
|
186
|
-
|
|
213
|
+
const startSec = Math.floor(currentStartMs / 1000) - 1;
|
|
214
|
+
const endSec = Math.floor(effectiveEndMs / 1000);
|
|
215
|
+
const query = `after:${startSec} before:${endSec}`;
|
|
187
216
|
|
|
188
|
-
|
|
189
|
-
// Gmail API always returns newest first, so we must fetch all IDs, sort, then hydrate
|
|
190
|
-
// This prevents skipping emails when using max_emails pagination
|
|
191
|
-
const { messages, hasMore } = await this.gmailService.fetchMessagesOldestFirst(account, {
|
|
192
|
-
limit: batchSize,
|
|
193
|
-
query: query || undefined,
|
|
194
|
-
maxIdsToFetch: 1000, // Safety limit for ID fetching
|
|
195
|
-
});
|
|
217
|
+
logger.info('Gmail window attempt', { attempt: attempts + 1, query });
|
|
196
218
|
|
|
197
|
-
|
|
198
|
-
|
|
219
|
+
const result = await this.gmailService.fetchMessagesOldestFirst(account, {
|
|
220
|
+
limit: batchSize,
|
|
221
|
+
query,
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
if (result.messages.length > 0) {
|
|
225
|
+
messages = result.messages;
|
|
226
|
+
hasMore = result.hasMore;
|
|
227
|
+
break; // Found emails, stop skipping
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// No emails found in this week, move to next week
|
|
231
|
+
logger.info('No emails in 7-day window, skipping forward', { start: new Date(currentStartMs).toISOString() });
|
|
232
|
+
currentStartMs = effectiveEndMs;
|
|
233
|
+
attempts++;
|
|
234
|
+
|
|
235
|
+
if (eventLogger && attempts % 3 === 0) {
|
|
236
|
+
await eventLogger.info('Sync', `Scanning history... reached ${new Date(currentStartMs).toLocaleDateString()}`);
|
|
237
|
+
}
|
|
199
238
|
}
|
|
200
239
|
|
|
201
|
-
|
|
202
|
-
|
|
240
|
+
if (eventLogger && messages.length > 0) {
|
|
241
|
+
await eventLogger.info('Fetching', `Fetched ${messages.length} emails in window${hasMore ? ', more available' : ''}`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Initialize max tracking with the point we reached
|
|
245
|
+
let maxInternalDate = currentStartMs;
|
|
203
246
|
|
|
204
247
|
for (const message of messages) {
|
|
205
248
|
try {
|
|
206
249
|
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
207
250
|
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
const msgInternalDate = message.internalDate ? parseInt(message.internalDate) : new Date(message.date).getTime();
|
|
251
|
+
// Track highest internalDate in memory
|
|
252
|
+
const msgInternalDate = parseInt(message.internalDate);
|
|
211
253
|
if (msgInternalDate > maxInternalDate) {
|
|
212
254
|
maxInternalDate = msgInternalDate;
|
|
213
|
-
|
|
214
|
-
await this.supabase
|
|
215
|
-
.from('email_accounts')
|
|
216
|
-
.update({ last_sync_checkpoint: maxInternalDate.toString() })
|
|
217
|
-
.eq('id', account.id);
|
|
218
255
|
}
|
|
219
256
|
} catch (error) {
|
|
220
257
|
logger.error('Failed to process Gmail message', error, { messageId: message.id });
|
|
@@ -222,6 +259,24 @@ export class EmailProcessorService {
|
|
|
222
259
|
result.errors++;
|
|
223
260
|
}
|
|
224
261
|
}
|
|
262
|
+
|
|
263
|
+
// Update checkpoint once at the end of the batch if we made progress
|
|
264
|
+
if (maxInternalDate > effectiveStartMs) {
|
|
265
|
+
logger.info('Updating Gmail checkpoint', {
|
|
266
|
+
accountId: account.id,
|
|
267
|
+
oldCheckpoint: account.last_sync_checkpoint,
|
|
268
|
+
newCheckpoint: maxInternalDate.toString()
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
const { error: updateError } = await this.supabase
|
|
272
|
+
.from('email_accounts')
|
|
273
|
+
.update({ last_sync_checkpoint: maxInternalDate.toString() })
|
|
274
|
+
.eq('id', account.id);
|
|
275
|
+
|
|
276
|
+
if (updateError) {
|
|
277
|
+
logger.error('Failed to update Gmail checkpoint', updateError);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
225
280
|
}
|
|
226
281
|
|
|
227
282
|
private async processOutlookAccount(
|
|
@@ -233,16 +288,22 @@ export class EmailProcessorService {
|
|
|
233
288
|
): Promise<void> {
|
|
234
289
|
const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
|
|
235
290
|
|
|
236
|
-
//
|
|
291
|
+
// Debug: Log account sync settings
|
|
292
|
+
logger.info('Outlook sync settings', {
|
|
293
|
+
accountId: account.id,
|
|
294
|
+
sync_start_date: account.sync_start_date,
|
|
295
|
+
last_sync_checkpoint: account.last_sync_checkpoint,
|
|
296
|
+
sync_max_emails_per_run: account.sync_max_emails_per_run,
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
// Construct filter: Use sync_start_date if present (Override), otherwise checkpoint
|
|
237
300
|
let effectiveStartIso = '';
|
|
238
301
|
if (account.sync_start_date) {
|
|
239
302
|
effectiveStartIso = new Date(account.sync_start_date).toISOString();
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
effectiveStartIso = account.last_sync_checkpoint;
|
|
245
|
-
}
|
|
303
|
+
logger.info('Using sync_start_date override', { effectiveStartIso });
|
|
304
|
+
} else if (account.last_sync_checkpoint) {
|
|
305
|
+
effectiveStartIso = account.last_sync_checkpoint;
|
|
306
|
+
logger.info('Using last_sync_checkpoint', { effectiveStartIso });
|
|
246
307
|
}
|
|
247
308
|
|
|
248
309
|
let filter = '';
|
|
@@ -250,6 +311,7 @@ export class EmailProcessorService {
|
|
|
250
311
|
// Use 'ge' (>=) instead of 'gt' (>) to ensure we don't miss emails at exact checkpoint
|
|
251
312
|
// The duplicate check in processMessage() will skip already-processed emails
|
|
252
313
|
filter = `receivedDateTime ge ${effectiveStartIso}`;
|
|
314
|
+
logger.info('Final Outlook filter', { filter });
|
|
253
315
|
}
|
|
254
316
|
|
|
255
317
|
if (eventLogger) await eventLogger.info('Fetching', 'Fetching emails from Outlook (oldest first)', { filter, batchSize });
|
|
@@ -266,19 +328,14 @@ export class EmailProcessorService {
|
|
|
266
328
|
}
|
|
267
329
|
|
|
268
330
|
// Messages are already sorted oldest-first by the API
|
|
269
|
-
let latestCheckpoint =
|
|
331
|
+
let latestCheckpoint = effectiveStartIso;
|
|
270
332
|
|
|
271
333
|
for (const message of messages) {
|
|
272
334
|
try {
|
|
273
|
-
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
335
|
+
const processResult = await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
274
336
|
|
|
275
|
-
if (!latestCheckpoint ||
|
|
276
|
-
latestCheckpoint =
|
|
277
|
-
|
|
278
|
-
await this.supabase
|
|
279
|
-
.from('email_accounts')
|
|
280
|
-
.update({ last_sync_checkpoint: latestCheckpoint })
|
|
281
|
-
.eq('id', account.id);
|
|
337
|
+
if (processResult && (!latestCheckpoint || processResult.date > latestCheckpoint)) {
|
|
338
|
+
latestCheckpoint = processResult.date;
|
|
282
339
|
}
|
|
283
340
|
} catch (error) {
|
|
284
341
|
logger.error('Failed to process Outlook message', error, { messageId: message.id });
|
|
@@ -286,6 +343,29 @@ export class EmailProcessorService {
|
|
|
286
343
|
result.errors++;
|
|
287
344
|
}
|
|
288
345
|
}
|
|
346
|
+
|
|
347
|
+
// Update checkpoint once at the end of the batch if we made progress
|
|
348
|
+
if (latestCheckpoint && latestCheckpoint !== effectiveStartIso) {
|
|
349
|
+
logger.info('Updating Outlook checkpoint', {
|
|
350
|
+
accountId: account.id,
|
|
351
|
+
oldCheckpoint: account.last_sync_checkpoint,
|
|
352
|
+
newCheckpoint: latestCheckpoint
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
const { error: updateError } = await this.supabase
|
|
356
|
+
.from('email_accounts')
|
|
357
|
+
.update({ last_sync_checkpoint: latestCheckpoint })
|
|
358
|
+
.eq('id', account.id);
|
|
359
|
+
|
|
360
|
+
if (updateError) {
|
|
361
|
+
logger.error('Failed to update Outlook checkpoint', updateError);
|
|
362
|
+
}
|
|
363
|
+
} else {
|
|
364
|
+
logger.info('Outlook checkpoint not updated (no newer emails found in this batch)', {
|
|
365
|
+
latestCheckpoint,
|
|
366
|
+
effectiveStartIso
|
|
367
|
+
});
|
|
368
|
+
}
|
|
289
369
|
}
|
|
290
370
|
|
|
291
371
|
private async processMessage(
|
|
@@ -295,7 +375,7 @@ export class EmailProcessorService {
|
|
|
295
375
|
settings: any,
|
|
296
376
|
result: ProcessingResult,
|
|
297
377
|
eventLogger: EventLogger | null
|
|
298
|
-
): Promise<void> {
|
|
378
|
+
): Promise<{ date: string } | void> {
|
|
299
379
|
// Check if already processed
|
|
300
380
|
const { data: existing } = await this.supabase
|
|
301
381
|
.from('emails')
|
|
@@ -306,21 +386,64 @@ export class EmailProcessorService {
|
|
|
306
386
|
|
|
307
387
|
if (existing) {
|
|
308
388
|
logger.debug('Message already processed', { messageId: message.id });
|
|
309
|
-
if (eventLogger) await eventLogger.info('Skipped', `Already processed: ${message.
|
|
389
|
+
if (eventLogger) await eventLogger.info('Skipped', `Already processed ID: ${message.id}`);
|
|
390
|
+
|
|
391
|
+
// Still need to return the date for checkpointing even if skipped
|
|
392
|
+
const rawMime = 'raw' in message
|
|
393
|
+
? (account.provider === 'gmail'
|
|
394
|
+
? Buffer.from(message.raw, 'base64').toString('utf-8')
|
|
395
|
+
: message.raw)
|
|
396
|
+
: '';
|
|
397
|
+
if (rawMime) {
|
|
398
|
+
const parsed = await simpleParser(rawMime);
|
|
399
|
+
return { date: parsed.date ? parsed.date.toISOString() : new Date().toISOString() };
|
|
400
|
+
}
|
|
310
401
|
return;
|
|
311
402
|
}
|
|
312
403
|
|
|
313
|
-
|
|
404
|
+
// Extract raw content string (Gmail is base64url, Outlook is raw text from $value)
|
|
405
|
+
const rawMime = 'raw' in message
|
|
406
|
+
? (account.provider === 'gmail'
|
|
407
|
+
? Buffer.from(message.raw, 'base64').toString('utf-8')
|
|
408
|
+
: message.raw)
|
|
409
|
+
: '';
|
|
410
|
+
|
|
411
|
+
if (!rawMime) {
|
|
412
|
+
throw new Error(`No raw MIME content found for message ${message.id}`);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// 1. Extract metadata from raw MIME using mailparser for the DB record
|
|
416
|
+
const parsed = await simpleParser(rawMime);
|
|
417
|
+
const subject = parsed.subject || 'No Subject';
|
|
418
|
+
const sender = parsed.from?.text || 'Unknown';
|
|
419
|
+
const recipient = parsed.to ? (Array.isArray(parsed.to) ? parsed.to[0].text : parsed.to.text) : '';
|
|
420
|
+
const date = parsed.date ? parsed.date.toISOString() : new Date().toISOString();
|
|
421
|
+
const bodySnippet = (parsed.text || parsed.textAsHtml || '').substring(0, 500);
|
|
422
|
+
|
|
423
|
+
if (eventLogger) await eventLogger.info('Ingesting', `Ingesting email: ${subject}`);
|
|
314
424
|
|
|
315
|
-
//
|
|
425
|
+
// 2. Save raw content to local storage (.eml format)
|
|
426
|
+
let filePath = '';
|
|
427
|
+
try {
|
|
428
|
+
const filename = `${account.id}_${message.id}.eml`.replace(/[^a-z0-9._-]/gi, '_');
|
|
429
|
+
filePath = await this.storageService.saveEmail(rawMime, filename, settings?.storage_path);
|
|
430
|
+
} catch (storageError) {
|
|
431
|
+
logger.error('Failed to save raw email content', storageError);
|
|
432
|
+
if (eventLogger) await eventLogger.error('Storage Error', storageError);
|
|
433
|
+
throw storageError;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// 3. Create a "Skeleton" record with 'pending' status
|
|
316
437
|
const emailData: Partial<Email> = {
|
|
317
438
|
account_id: account.id,
|
|
318
439
|
external_id: message.id,
|
|
319
|
-
subject
|
|
320
|
-
sender
|
|
321
|
-
recipient
|
|
322
|
-
date
|
|
323
|
-
body_snippet:
|
|
440
|
+
subject,
|
|
441
|
+
sender,
|
|
442
|
+
recipient,
|
|
443
|
+
date,
|
|
444
|
+
body_snippet: bodySnippet,
|
|
445
|
+
file_path: filePath,
|
|
446
|
+
processing_status: 'pending',
|
|
324
447
|
};
|
|
325
448
|
|
|
326
449
|
const { data: savedEmail, error: saveError } = await this.supabase
|
|
@@ -332,59 +455,199 @@ export class EmailProcessorService {
|
|
|
332
455
|
if (saveError || !savedEmail) {
|
|
333
456
|
logger.error('Failed to create initial email record', saveError);
|
|
334
457
|
if (eventLogger) await eventLogger.error('Database Error', saveError);
|
|
458
|
+
return { date };
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// Log successful ingestion linked to email ID
|
|
462
|
+
if (eventLogger) await eventLogger.info('Ingested', `Successfully ingested email: ${subject}`, { filePath }, savedEmail.id);
|
|
463
|
+
|
|
464
|
+
result.processed++;
|
|
465
|
+
|
|
466
|
+
return { date };
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Background Worker: Processes pending emails for a user recursively until empty.
|
|
471
|
+
*/
|
|
472
|
+
async processQueue(userId: string, settings: any): Promise<void> {
|
|
473
|
+
logger.info('Worker: Checking queue', { userId });
|
|
474
|
+
|
|
475
|
+
// Fetch up to 5 pending emails for this user
|
|
476
|
+
const { data: pendingEmails, error } = await this.supabase
|
|
477
|
+
.from('emails')
|
|
478
|
+
.select('*, email_accounts!inner(id, user_id, provider)')
|
|
479
|
+
.eq('email_accounts.user_id', userId)
|
|
480
|
+
.eq('processing_status', 'pending')
|
|
481
|
+
.limit(5);
|
|
482
|
+
|
|
483
|
+
if (error) {
|
|
484
|
+
logger.error('Worker: Failed to fetch queue', error);
|
|
335
485
|
return;
|
|
336
486
|
}
|
|
337
487
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
|
|
341
|
-
? {
|
|
342
|
-
model: settings.llm_model,
|
|
343
|
-
baseUrl: settings.llm_base_url,
|
|
344
|
-
apiKey: settings.llm_api_key,
|
|
345
|
-
}
|
|
346
|
-
: undefined
|
|
347
|
-
);
|
|
348
|
-
|
|
349
|
-
const analysis = await intelligenceService.analyzeEmail(message.body, {
|
|
350
|
-
subject: message.subject,
|
|
351
|
-
sender: message.sender,
|
|
352
|
-
date: message.date,
|
|
353
|
-
metadata: message.headers,
|
|
354
|
-
userPreferences: {
|
|
355
|
-
autoTrashSpam: settings?.auto_trash_spam,
|
|
356
|
-
smartDrafts: settings?.smart_drafts,
|
|
357
|
-
},
|
|
358
|
-
}, eventLogger || undefined, savedEmail.id);
|
|
359
|
-
|
|
360
|
-
if (!analysis) {
|
|
361
|
-
result.errors++;
|
|
488
|
+
if (!pendingEmails || pendingEmails.length === 0) {
|
|
489
|
+
logger.info('Worker: Queue empty', { userId });
|
|
362
490
|
return;
|
|
363
491
|
}
|
|
364
492
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
.
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
493
|
+
logger.info('Worker: Processing batch', { userId, count: pendingEmails.length });
|
|
494
|
+
|
|
495
|
+
for (const email of pendingEmails) {
|
|
496
|
+
await this.processPendingEmail(email, userId, settings);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
// Slight delay to prevent hitting rate limits too fast, then check again
|
|
500
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
501
|
+
return this.processQueue(userId, settings);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
private async processPendingEmail(email: Email, userId: string, settings: any): Promise<void> {
|
|
505
|
+
// Create a real processing log entry for this background task to ensure RLS compliance
|
|
506
|
+
const { data: log } = await this.supabase
|
|
507
|
+
.from('processing_logs')
|
|
508
|
+
.insert({
|
|
509
|
+
user_id: userId,
|
|
510
|
+
account_id: email.account_id,
|
|
511
|
+
status: 'running',
|
|
374
512
|
})
|
|
375
|
-
.
|
|
513
|
+
.select()
|
|
514
|
+
.single();
|
|
376
515
|
|
|
377
|
-
const
|
|
378
|
-
...savedEmail,
|
|
379
|
-
category: analysis.category,
|
|
380
|
-
is_useless: analysis.is_useless,
|
|
381
|
-
suggested_actions: analysis.suggested_actions
|
|
382
|
-
};
|
|
516
|
+
const eventLogger = log ? new EventLogger(this.supabase, log.id) : null;
|
|
383
517
|
|
|
384
|
-
|
|
518
|
+
try {
|
|
519
|
+
// 1. Double-check status and mark as processing (Atomic-ish)
|
|
520
|
+
const { data: current } = await this.supabase
|
|
521
|
+
.from('emails')
|
|
522
|
+
.select('processing_status')
|
|
523
|
+
.eq('id', email.id)
|
|
524
|
+
.single();
|
|
525
|
+
|
|
526
|
+
if (current?.processing_status !== 'pending') {
|
|
527
|
+
if (log) await this.supabase.from('processing_logs').delete().eq('id', log.id);
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
await this.supabase
|
|
532
|
+
.from('emails')
|
|
533
|
+
.update({ processing_status: 'processing' })
|
|
534
|
+
.eq('id', email.id);
|
|
535
|
+
|
|
536
|
+
if (eventLogger) await eventLogger.info('Processing', `Background processing: ${email.subject}`, undefined, email.id);
|
|
537
|
+
|
|
538
|
+
// 2. Read content from disk and parse with mailparser
|
|
539
|
+
if (!email.file_path) throw new Error('No file path found for email');
|
|
540
|
+
const rawMime = await this.storageService.readEmail(email.file_path);
|
|
541
|
+
const parsed = await simpleParser(rawMime);
|
|
542
|
+
|
|
543
|
+
// Extract clean content (prioritize text)
|
|
544
|
+
const cleanContent = parsed.text || parsed.textAsHtml || '';
|
|
545
|
+
|
|
546
|
+
// Extract metadata signals from headers
|
|
547
|
+
const metadata = {
|
|
548
|
+
importance: parsed.headers.get('importance')?.toString() || parsed.headers.get('x-priority')?.toString(),
|
|
549
|
+
listUnsubscribe: parsed.headers.get('list-unsubscribe')?.toString(),
|
|
550
|
+
autoSubmitted: parsed.headers.get('auto-submitted')?.toString(),
|
|
551
|
+
mailer: parsed.headers.get('x-mailer')?.toString()
|
|
552
|
+
};
|
|
553
|
+
|
|
554
|
+
// 3. Analyze with AI
|
|
555
|
+
const intelligenceService = getIntelligenceService(
|
|
556
|
+
settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
|
|
557
|
+
? {
|
|
558
|
+
model: settings.llm_model,
|
|
559
|
+
baseUrl: settings.llm_base_url,
|
|
560
|
+
apiKey: settings.llm_api_key,
|
|
561
|
+
}
|
|
562
|
+
: undefined
|
|
563
|
+
);
|
|
564
|
+
|
|
565
|
+
const analysis = await intelligenceService.analyzeEmail(cleanContent, {
|
|
566
|
+
subject: email.subject || '',
|
|
567
|
+
sender: email.sender || '',
|
|
568
|
+
date: email.date || '',
|
|
569
|
+
metadata,
|
|
570
|
+
userPreferences: {
|
|
571
|
+
autoTrashSpam: settings?.auto_trash_spam,
|
|
572
|
+
smartDrafts: settings?.smart_drafts,
|
|
573
|
+
},
|
|
574
|
+
}, eventLogger || undefined, email.id);
|
|
575
|
+
|
|
576
|
+
if (!analysis) {
|
|
577
|
+
throw new Error('AI analysis returned no result');
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// 4. Update the email record with results
|
|
581
|
+
await this.supabase
|
|
582
|
+
.from('emails')
|
|
583
|
+
.update({
|
|
584
|
+
category: analysis.category,
|
|
585
|
+
is_useless: analysis.is_useless,
|
|
586
|
+
ai_analysis: analysis as any,
|
|
587
|
+
suggested_actions: analysis.suggested_actions || [],
|
|
588
|
+
suggested_action: analysis.suggested_actions?.[0] || 'none',
|
|
589
|
+
processing_status: 'completed'
|
|
590
|
+
})
|
|
591
|
+
.eq('id', email.id);
|
|
592
|
+
|
|
593
|
+
// 5. Execute automation rules
|
|
594
|
+
// Fetch account and rules needed for execution
|
|
595
|
+
const { data: account } = await this.supabase
|
|
596
|
+
.from('email_accounts')
|
|
597
|
+
.select('*')
|
|
598
|
+
.eq('id', email.account_id)
|
|
599
|
+
.single();
|
|
600
|
+
|
|
601
|
+
const { data: rules } = await this.supabase
|
|
602
|
+
.from('rules')
|
|
603
|
+
.select('*')
|
|
604
|
+
.eq('user_id', userId)
|
|
605
|
+
.eq('is_enabled', true);
|
|
606
|
+
|
|
607
|
+
if (account && rules) {
|
|
608
|
+
const tempResult = { processed: 0, deleted: 0, drafted: 0, errors: 0 };
|
|
609
|
+
// Ensure email object for rules has the analysis fields merged in
|
|
610
|
+
const emailForRules = { ...email, ...analysis };
|
|
611
|
+
await this.executeRules(account, emailForRules as any, analysis, rules, settings, tempResult, eventLogger);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
// Mark log as success
|
|
615
|
+
if (log) {
|
|
616
|
+
await this.supabase
|
|
617
|
+
.from('processing_logs')
|
|
618
|
+
.update({
|
|
619
|
+
status: 'success',
|
|
620
|
+
completed_at: new Date().toISOString(),
|
|
621
|
+
emails_processed: 1
|
|
622
|
+
})
|
|
623
|
+
.eq('id', log.id);
|
|
624
|
+
}
|
|
385
625
|
|
|
386
|
-
|
|
387
|
-
|
|
626
|
+
} catch (error) {
|
|
627
|
+
logger.error('Failed to process pending email', error, { emailId: email.id });
|
|
628
|
+
if (eventLogger) await eventLogger.error('Processing Failed', error, email.id);
|
|
629
|
+
|
|
630
|
+
// Mark log as failed
|
|
631
|
+
if (log) {
|
|
632
|
+
await this.supabase
|
|
633
|
+
.from('processing_logs')
|
|
634
|
+
.update({
|
|
635
|
+
status: 'failed',
|
|
636
|
+
completed_at: new Date().toISOString(),
|
|
637
|
+
error_message: error instanceof Error ? error.message : String(error)
|
|
638
|
+
})
|
|
639
|
+
.eq('id', log.id);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
await this.supabase
|
|
643
|
+
.from('emails')
|
|
644
|
+
.update({
|
|
645
|
+
processing_status: 'failed',
|
|
646
|
+
processing_error: error instanceof Error ? error.message : String(error),
|
|
647
|
+
retry_count: (email.retry_count || 0) + 1
|
|
648
|
+
})
|
|
649
|
+
.eq('id', email.id);
|
|
650
|
+
}
|
|
388
651
|
}
|
|
389
652
|
|
|
390
653
|
private async executeRules(
|
|
@@ -491,8 +754,9 @@ export class EmailProcessorService {
|
|
|
491
754
|
case 'suggested_actions':
|
|
492
755
|
// Handle array membership check (e.g. if condition expects "reply" to be in actions)
|
|
493
756
|
const requiredActions = Array.isArray(value) ? value : [value];
|
|
757
|
+
const actualActions = analysis.suggested_actions || [];
|
|
494
758
|
const hasAllActions = requiredActions.every(req =>
|
|
495
|
-
|
|
759
|
+
actualActions.includes(req as any)
|
|
496
760
|
);
|
|
497
761
|
if (!hasAllActions) return false;
|
|
498
762
|
break;
|
|
@@ -572,6 +836,9 @@ export class EmailProcessorService {
|
|
|
572
836
|
if (account.provider === 'gmail') {
|
|
573
837
|
if (action === 'delete') {
|
|
574
838
|
await this.gmailService.trashMessage(account, email.external_id);
|
|
839
|
+
if (email.file_path) {
|
|
840
|
+
await this.storageService.deleteEmail(email.file_path);
|
|
841
|
+
}
|
|
575
842
|
} else if (action === 'archive') {
|
|
576
843
|
await this.gmailService.archiveMessage(account, email.external_id);
|
|
577
844
|
} else if (action === 'draft' && draftContent) {
|
|
@@ -587,6 +854,9 @@ export class EmailProcessorService {
|
|
|
587
854
|
} else if (account.provider === 'outlook') {
|
|
588
855
|
if (action === 'delete') {
|
|
589
856
|
await this.microsoftService.trashMessage(account, email.external_id);
|
|
857
|
+
if (email.file_path) {
|
|
858
|
+
await this.storageService.deleteEmail(email.file_path);
|
|
859
|
+
}
|
|
590
860
|
} else if (action === 'archive') {
|
|
591
861
|
await this.microsoftService.archiveMessage(account, email.external_id);
|
|
592
862
|
} else if (action === 'draft' && draftContent) {
|