@realtimex/email-automator 2.4.5 → 2.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/src/routes/emails.ts +37 -1
- package/api/src/routes/rules.ts +7 -2
- package/api/src/services/gmail.ts +53 -166
- package/api/src/services/intelligence.ts +2 -1
- package/api/src/services/microsoft.ts +34 -29
- package/api/src/services/processor.ts +376 -106
- package/api/src/services/storage.ts +88 -0
- package/api/src/services/supabase.ts +6 -0
- package/api/src/utils/contentCleaner.ts +11 -6
- package/dist/api/src/routes/emails.js +27 -1
- package/dist/api/src/routes/rules.js +6 -2
- package/dist/api/src/services/gmail.js +49 -138
- package/dist/api/src/services/intelligence.js +2 -1
- package/dist/api/src/services/microsoft.js +28 -18
- package/dist/api/src/services/processor.js +335 -96
- package/dist/api/src/services/storage.js +81 -0
- package/dist/api/src/utils/contentCleaner.js +10 -6
- package/dist/assets/index-BSHZ3lFn.js +97 -0
- package/dist/assets/index-CRQKk5IW.css +1 -0
- package/dist/index.html +2 -2
- package/package.json +3 -1
- package/supabase/functions/api-v1-accounts/index.ts +28 -7
- package/supabase/migrations/20260118000001_async_etl_storage.sql +24 -0
- package/dist/assets/index-C3PlbplS.css +0 -1
- package/dist/assets/index-DfGa9R7j.js +0 -97
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
import { simpleParser } from 'mailparser';
|
|
1
2
|
import { createLogger } from '../utils/logger.js';
|
|
2
3
|
import { config } from '../config/index.js';
|
|
3
4
|
import { getGmailService } from './gmail.js';
|
|
4
5
|
import { getMicrosoftService } from './microsoft.js';
|
|
5
6
|
import { getIntelligenceService } from './intelligence.js';
|
|
7
|
+
import { getStorageService } from './storage.js';
|
|
6
8
|
import { EventLogger } from './eventLogger.js';
|
|
7
9
|
const logger = createLogger('Processor');
|
|
8
10
|
export class EmailProcessorService {
|
|
9
11
|
supabase;
|
|
10
12
|
gmailService = getGmailService();
|
|
11
13
|
microsoftService = getMicrosoftService();
|
|
14
|
+
storageService = getStorageService();
|
|
12
15
|
constructor(supabase) {
|
|
13
16
|
this.supabase = supabase;
|
|
14
17
|
}
|
|
@@ -35,6 +38,11 @@ export class EmailProcessorService {
|
|
|
35
38
|
if (accError || !account) {
|
|
36
39
|
throw new Error('Account not found or access denied');
|
|
37
40
|
}
|
|
41
|
+
logger.info('Retrieved account settings', {
|
|
42
|
+
accountId: account.id,
|
|
43
|
+
sync_start_date: account.sync_start_date,
|
|
44
|
+
last_sync_checkpoint: account.last_sync_checkpoint
|
|
45
|
+
});
|
|
38
46
|
// Refresh token if needed
|
|
39
47
|
let refreshedAccount = account;
|
|
40
48
|
if (account.provider === 'gmail') {
|
|
@@ -72,6 +80,8 @@ export class EmailProcessorService {
|
|
|
72
80
|
}
|
|
73
81
|
// After processing new emails, run retention rules for this account
|
|
74
82
|
await this.runRetentionRules(refreshedAccount, rules || [], settings, result, eventLogger);
|
|
83
|
+
// Trigger background worker (async) to process the queue
|
|
84
|
+
this.processQueue(userId, settings).catch(err => logger.error('Background worker failed', err));
|
|
75
85
|
// Update log and account on success
|
|
76
86
|
if (log) {
|
|
77
87
|
await this.supabase
|
|
@@ -89,10 +99,11 @@ export class EmailProcessorService {
|
|
|
89
99
|
.from('email_accounts')
|
|
90
100
|
.update({
|
|
91
101
|
last_sync_status: 'success',
|
|
92
|
-
last_sync_error: null
|
|
102
|
+
last_sync_error: null,
|
|
103
|
+
sync_start_date: null // Clear manual override once used successfully
|
|
93
104
|
})
|
|
94
105
|
.eq('id', accountId);
|
|
95
|
-
logger.info('Sync completed', { accountId, ...result });
|
|
106
|
+
logger.info('Sync completed and override cleared', { accountId, ...result });
|
|
96
107
|
}
|
|
97
108
|
catch (error) {
|
|
98
109
|
logger.error('Sync failed', error, { accountId });
|
|
@@ -125,51 +136,69 @@ export class EmailProcessorService {
|
|
|
125
136
|
}
|
|
126
137
|
async processGmailAccount(account, rules, settings, result, eventLogger) {
|
|
127
138
|
const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
|
|
128
|
-
//
|
|
139
|
+
// Debug: Log account sync settings
|
|
140
|
+
logger.info('Gmail sync settings', {
|
|
141
|
+
accountId: account.id,
|
|
142
|
+
sync_start_date: account.sync_start_date,
|
|
143
|
+
last_sync_checkpoint: account.last_sync_checkpoint,
|
|
144
|
+
sync_max_emails_per_run: account.sync_max_emails_per_run,
|
|
145
|
+
});
|
|
146
|
+
// Construct query: Use a sliding window for efficiency and determinism
|
|
129
147
|
let effectiveStartMs = 0;
|
|
130
148
|
if (account.sync_start_date) {
|
|
131
149
|
effectiveStartMs = new Date(account.sync_start_date).getTime();
|
|
150
|
+
logger.info('Using sync_start_date override', { effectiveStartMs, date: new Date(effectiveStartMs).toISOString() });
|
|
132
151
|
}
|
|
133
|
-
if (account.last_sync_checkpoint) {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
152
|
+
else if (account.last_sync_checkpoint) {
|
|
153
|
+
effectiveStartMs = parseInt(account.last_sync_checkpoint);
|
|
154
|
+
logger.info('Using last_sync_checkpoint', { effectiveStartMs, date: new Date(effectiveStartMs).toISOString() });
|
|
155
|
+
}
|
|
156
|
+
// Use a 7-day sliding window. If empty, skip forward (up to 10 weeks per run)
|
|
157
|
+
const windowSizeMs = 7 * 24 * 60 * 60 * 1000;
|
|
158
|
+
const nowMs = Date.now();
|
|
159
|
+
const tomorrowMs = nowMs + (24 * 60 * 60 * 1000);
|
|
160
|
+
let currentStartMs = effectiveStartMs;
|
|
161
|
+
let messages = [];
|
|
162
|
+
let hasMore = false;
|
|
163
|
+
let attempts = 0;
|
|
164
|
+
const maxAttempts = 10;
|
|
165
|
+
while (attempts < maxAttempts && currentStartMs < nowMs) {
|
|
166
|
+
let effectiveEndMs = currentStartMs + windowSizeMs;
|
|
167
|
+
if (effectiveEndMs > tomorrowMs)
|
|
168
|
+
effectiveEndMs = tomorrowMs;
|
|
169
|
+
const startSec = Math.floor(currentStartMs / 1000) - 1;
|
|
170
|
+
const endSec = Math.floor(effectiveEndMs / 1000);
|
|
171
|
+
const query = `after:${startSec} before:${endSec}`;
|
|
172
|
+
logger.info('Gmail window attempt', { attempt: attempts + 1, query });
|
|
173
|
+
const result = await this.gmailService.fetchMessagesOldestFirst(account, {
|
|
174
|
+
limit: batchSize,
|
|
175
|
+
query,
|
|
176
|
+
});
|
|
177
|
+
if (result.messages.length > 0) {
|
|
178
|
+
messages = result.messages;
|
|
179
|
+
hasMore = result.hasMore;
|
|
180
|
+
break; // Found emails, stop skipping
|
|
181
|
+
}
|
|
182
|
+
// No emails found in this week, move to next week
|
|
183
|
+
logger.info('No emails in 7-day window, skipping forward', { start: new Date(currentStartMs).toISOString() });
|
|
184
|
+
currentStartMs = effectiveEndMs;
|
|
185
|
+
attempts++;
|
|
186
|
+
if (eventLogger && attempts % 3 === 0) {
|
|
187
|
+
await eventLogger.info('Sync', `Scanning history... reached ${new Date(currentStartMs).toLocaleDateString()}`);
|
|
137
188
|
}
|
|
138
189
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// Subtract 1 second to make query inclusive (Gmail's after: is exclusive)
|
|
142
|
-
// This ensures we don't miss emails at the exact checkpoint timestamp
|
|
143
|
-
const startSeconds = Math.floor(effectiveStartMs / 1000) - 1;
|
|
144
|
-
query = `after:${startSeconds}`;
|
|
190
|
+
if (eventLogger && messages.length > 0) {
|
|
191
|
+
await eventLogger.info('Fetching', `Fetched ${messages.length} emails in window${hasMore ? ', more available' : ''}`);
|
|
145
192
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
// Use "Fetch IDs → Sort → Hydrate" strategy to get OLDEST emails first
|
|
149
|
-
// Gmail API always returns newest first, so we must fetch all IDs, sort, then hydrate
|
|
150
|
-
// This prevents skipping emails when using max_emails pagination
|
|
151
|
-
const { messages, hasMore } = await this.gmailService.fetchMessagesOldestFirst(account, {
|
|
152
|
-
limit: batchSize,
|
|
153
|
-
query: query || undefined,
|
|
154
|
-
maxIdsToFetch: 1000, // Safety limit for ID fetching
|
|
155
|
-
});
|
|
156
|
-
if (eventLogger) {
|
|
157
|
-
await eventLogger.info('Fetching', `Fetched ${messages.length} emails (oldest first)${hasMore ? ', more available' : ''}`);
|
|
158
|
-
}
|
|
159
|
-
// Messages are already sorted oldest-first by fetchMessagesOldestFirst
|
|
160
|
-
let maxInternalDate = account.last_sync_checkpoint ? parseInt(account.last_sync_checkpoint) : 0;
|
|
193
|
+
// Initialize max tracking with the point we reached
|
|
194
|
+
let maxInternalDate = currentStartMs;
|
|
161
195
|
for (const message of messages) {
|
|
162
196
|
try {
|
|
163
197
|
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
const msgInternalDate = message.internalDate ? parseInt(message.internalDate) : new Date(message.date).getTime();
|
|
198
|
+
// Track highest internalDate in memory
|
|
199
|
+
const msgInternalDate = parseInt(message.internalDate);
|
|
167
200
|
if (msgInternalDate > maxInternalDate) {
|
|
168
201
|
maxInternalDate = msgInternalDate;
|
|
169
|
-
await this.supabase
|
|
170
|
-
.from('email_accounts')
|
|
171
|
-
.update({ last_sync_checkpoint: maxInternalDate.toString() })
|
|
172
|
-
.eq('id', account.id);
|
|
173
202
|
}
|
|
174
203
|
}
|
|
175
204
|
catch (error) {
|
|
@@ -179,24 +208,47 @@ export class EmailProcessorService {
|
|
|
179
208
|
result.errors++;
|
|
180
209
|
}
|
|
181
210
|
}
|
|
211
|
+
// Update checkpoint once at the end of the batch if we made progress
|
|
212
|
+
if (maxInternalDate > effectiveStartMs) {
|
|
213
|
+
logger.info('Updating Gmail checkpoint', {
|
|
214
|
+
accountId: account.id,
|
|
215
|
+
oldCheckpoint: account.last_sync_checkpoint,
|
|
216
|
+
newCheckpoint: maxInternalDate.toString()
|
|
217
|
+
});
|
|
218
|
+
const { error: updateError } = await this.supabase
|
|
219
|
+
.from('email_accounts')
|
|
220
|
+
.update({ last_sync_checkpoint: maxInternalDate.toString() })
|
|
221
|
+
.eq('id', account.id);
|
|
222
|
+
if (updateError) {
|
|
223
|
+
logger.error('Failed to update Gmail checkpoint', updateError);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
182
226
|
}
|
|
183
227
|
async processOutlookAccount(account, rules, settings, result, eventLogger) {
|
|
184
228
|
const batchSize = account.sync_max_emails_per_run || config.processing.batchSize;
|
|
185
|
-
//
|
|
229
|
+
// Debug: Log account sync settings
|
|
230
|
+
logger.info('Outlook sync settings', {
|
|
231
|
+
accountId: account.id,
|
|
232
|
+
sync_start_date: account.sync_start_date,
|
|
233
|
+
last_sync_checkpoint: account.last_sync_checkpoint,
|
|
234
|
+
sync_max_emails_per_run: account.sync_max_emails_per_run,
|
|
235
|
+
});
|
|
236
|
+
// Construct filter: Use sync_start_date if present (Override), otherwise checkpoint
|
|
186
237
|
let effectiveStartIso = '';
|
|
187
238
|
if (account.sync_start_date) {
|
|
188
239
|
effectiveStartIso = new Date(account.sync_start_date).toISOString();
|
|
240
|
+
logger.info('Using sync_start_date override', { effectiveStartIso });
|
|
189
241
|
}
|
|
190
|
-
if (account.last_sync_checkpoint) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
242
|
+
else if (account.last_sync_checkpoint) {
|
|
243
|
+
effectiveStartIso = account.last_sync_checkpoint;
|
|
244
|
+
logger.info('Using last_sync_checkpoint', { effectiveStartIso });
|
|
194
245
|
}
|
|
195
246
|
let filter = '';
|
|
196
247
|
if (effectiveStartIso) {
|
|
197
248
|
// Use 'ge' (>=) instead of 'gt' (>) to ensure we don't miss emails at exact checkpoint
|
|
198
249
|
// The duplicate check in processMessage() will skip already-processed emails
|
|
199
250
|
filter = `receivedDateTime ge ${effectiveStartIso}`;
|
|
251
|
+
logger.info('Final Outlook filter', { filter });
|
|
200
252
|
}
|
|
201
253
|
if (eventLogger)
|
|
202
254
|
await eventLogger.info('Fetching', 'Fetching emails from Outlook (oldest first)', { filter, batchSize });
|
|
@@ -210,16 +262,12 @@ export class EmailProcessorService {
|
|
|
210
262
|
await eventLogger.info('Fetching', `Fetched ${messages.length} emails (oldest first)${hasMore ? ', more available' : ''}`);
|
|
211
263
|
}
|
|
212
264
|
// Messages are already sorted oldest-first by the API
|
|
213
|
-
let latestCheckpoint =
|
|
265
|
+
let latestCheckpoint = effectiveStartIso;
|
|
214
266
|
for (const message of messages) {
|
|
215
267
|
try {
|
|
216
|
-
await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
217
|
-
if (!latestCheckpoint ||
|
|
218
|
-
latestCheckpoint =
|
|
219
|
-
await this.supabase
|
|
220
|
-
.from('email_accounts')
|
|
221
|
-
.update({ last_sync_checkpoint: latestCheckpoint })
|
|
222
|
-
.eq('id', account.id);
|
|
268
|
+
const processResult = await this.processMessage(account, message, rules, settings, result, eventLogger);
|
|
269
|
+
if (processResult && (!latestCheckpoint || processResult.date > latestCheckpoint)) {
|
|
270
|
+
latestCheckpoint = processResult.date;
|
|
223
271
|
}
|
|
224
272
|
}
|
|
225
273
|
catch (error) {
|
|
@@ -229,6 +277,27 @@ export class EmailProcessorService {
|
|
|
229
277
|
result.errors++;
|
|
230
278
|
}
|
|
231
279
|
}
|
|
280
|
+
// Update checkpoint once at the end of the batch if we made progress
|
|
281
|
+
if (latestCheckpoint && latestCheckpoint !== effectiveStartIso) {
|
|
282
|
+
logger.info('Updating Outlook checkpoint', {
|
|
283
|
+
accountId: account.id,
|
|
284
|
+
oldCheckpoint: account.last_sync_checkpoint,
|
|
285
|
+
newCheckpoint: latestCheckpoint
|
|
286
|
+
});
|
|
287
|
+
const { error: updateError } = await this.supabase
|
|
288
|
+
.from('email_accounts')
|
|
289
|
+
.update({ last_sync_checkpoint: latestCheckpoint })
|
|
290
|
+
.eq('id', account.id);
|
|
291
|
+
if (updateError) {
|
|
292
|
+
logger.error('Failed to update Outlook checkpoint', updateError);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
logger.info('Outlook checkpoint not updated (no newer emails found in this batch)', {
|
|
297
|
+
latestCheckpoint,
|
|
298
|
+
effectiveStartIso
|
|
299
|
+
});
|
|
300
|
+
}
|
|
232
301
|
}
|
|
233
302
|
async processMessage(account, message, rules, settings, result, eventLogger) {
|
|
234
303
|
// Check if already processed
|
|
@@ -241,20 +310,60 @@ export class EmailProcessorService {
|
|
|
241
310
|
if (existing) {
|
|
242
311
|
logger.debug('Message already processed', { messageId: message.id });
|
|
243
312
|
if (eventLogger)
|
|
244
|
-
await eventLogger.info('Skipped', `Already processed: ${message.
|
|
313
|
+
await eventLogger.info('Skipped', `Already processed ID: ${message.id}`);
|
|
314
|
+
// Still need to return the date for checkpointing even if skipped
|
|
315
|
+
const rawMime = 'raw' in message
|
|
316
|
+
? (account.provider === 'gmail'
|
|
317
|
+
? Buffer.from(message.raw, 'base64').toString('utf-8')
|
|
318
|
+
: message.raw)
|
|
319
|
+
: '';
|
|
320
|
+
if (rawMime) {
|
|
321
|
+
const parsed = await simpleParser(rawMime);
|
|
322
|
+
return { date: parsed.date ? parsed.date.toISOString() : new Date().toISOString() };
|
|
323
|
+
}
|
|
245
324
|
return;
|
|
246
325
|
}
|
|
326
|
+
// Extract raw content string (Gmail is base64url, Outlook is raw text from $value)
|
|
327
|
+
const rawMime = 'raw' in message
|
|
328
|
+
? (account.provider === 'gmail'
|
|
329
|
+
? Buffer.from(message.raw, 'base64').toString('utf-8')
|
|
330
|
+
: message.raw)
|
|
331
|
+
: '';
|
|
332
|
+
if (!rawMime) {
|
|
333
|
+
throw new Error(`No raw MIME content found for message ${message.id}`);
|
|
334
|
+
}
|
|
335
|
+
// 1. Extract metadata from raw MIME using mailparser for the DB record
|
|
336
|
+
const parsed = await simpleParser(rawMime);
|
|
337
|
+
const subject = parsed.subject || 'No Subject';
|
|
338
|
+
const sender = parsed.from?.text || 'Unknown';
|
|
339
|
+
const recipient = parsed.to ? (Array.isArray(parsed.to) ? parsed.to[0].text : parsed.to.text) : '';
|
|
340
|
+
const date = parsed.date ? parsed.date.toISOString() : new Date().toISOString();
|
|
341
|
+
const bodySnippet = (parsed.text || parsed.textAsHtml || '').substring(0, 500);
|
|
247
342
|
if (eventLogger)
|
|
248
|
-
await eventLogger.info('
|
|
249
|
-
//
|
|
343
|
+
await eventLogger.info('Ingesting', `Ingesting email: ${subject}`);
|
|
344
|
+
// 2. Save raw content to local storage (.eml format)
|
|
345
|
+
let filePath = '';
|
|
346
|
+
try {
|
|
347
|
+
const filename = `${account.id}_${message.id}.eml`.replace(/[^a-z0-9._-]/gi, '_');
|
|
348
|
+
filePath = await this.storageService.saveEmail(rawMime, filename, settings?.storage_path);
|
|
349
|
+
}
|
|
350
|
+
catch (storageError) {
|
|
351
|
+
logger.error('Failed to save raw email content', storageError);
|
|
352
|
+
if (eventLogger)
|
|
353
|
+
await eventLogger.error('Storage Error', storageError);
|
|
354
|
+
throw storageError;
|
|
355
|
+
}
|
|
356
|
+
// 3. Create a "Skeleton" record with 'pending' status
|
|
250
357
|
const emailData = {
|
|
251
358
|
account_id: account.id,
|
|
252
359
|
external_id: message.id,
|
|
253
|
-
subject
|
|
254
|
-
sender
|
|
255
|
-
recipient
|
|
256
|
-
date
|
|
257
|
-
body_snippet:
|
|
360
|
+
subject,
|
|
361
|
+
sender,
|
|
362
|
+
recipient,
|
|
363
|
+
date,
|
|
364
|
+
body_snippet: bodySnippet,
|
|
365
|
+
file_path: filePath,
|
|
366
|
+
processing_status: 'pending',
|
|
258
367
|
};
|
|
259
368
|
const { data: savedEmail, error: saveError } = await this.supabase
|
|
260
369
|
.from('emails')
|
|
@@ -265,50 +374,173 @@ export class EmailProcessorService {
|
|
|
265
374
|
logger.error('Failed to create initial email record', saveError);
|
|
266
375
|
if (eventLogger)
|
|
267
376
|
await eventLogger.error('Database Error', saveError);
|
|
377
|
+
return { date };
|
|
378
|
+
}
|
|
379
|
+
// Log successful ingestion linked to email ID
|
|
380
|
+
if (eventLogger)
|
|
381
|
+
await eventLogger.info('Ingested', `Successfully ingested email: ${subject}`, { filePath }, savedEmail.id);
|
|
382
|
+
result.processed++;
|
|
383
|
+
return { date };
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Background Worker: Processes pending emails for a user recursively until empty.
|
|
387
|
+
*/
|
|
388
|
+
async processQueue(userId, settings) {
|
|
389
|
+
logger.info('Worker: Checking queue', { userId });
|
|
390
|
+
// Fetch up to 5 pending emails for this user
|
|
391
|
+
const { data: pendingEmails, error } = await this.supabase
|
|
392
|
+
.from('emails')
|
|
393
|
+
.select('*, email_accounts!inner(id, user_id, provider)')
|
|
394
|
+
.eq('email_accounts.user_id', userId)
|
|
395
|
+
.eq('processing_status', 'pending')
|
|
396
|
+
.limit(5);
|
|
397
|
+
if (error) {
|
|
398
|
+
logger.error('Worker: Failed to fetch queue', error);
|
|
268
399
|
return;
|
|
269
400
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
? {
|
|
273
|
-
model: settings.llm_model,
|
|
274
|
-
baseUrl: settings.llm_base_url,
|
|
275
|
-
apiKey: settings.llm_api_key,
|
|
276
|
-
}
|
|
277
|
-
: undefined);
|
|
278
|
-
const analysis = await intelligenceService.analyzeEmail(message.body, {
|
|
279
|
-
subject: message.subject,
|
|
280
|
-
sender: message.sender,
|
|
281
|
-
date: message.date,
|
|
282
|
-
metadata: message.headers,
|
|
283
|
-
userPreferences: {
|
|
284
|
-
autoTrashSpam: settings?.auto_trash_spam,
|
|
285
|
-
smartDrafts: settings?.smart_drafts,
|
|
286
|
-
},
|
|
287
|
-
}, eventLogger || undefined, savedEmail.id);
|
|
288
|
-
if (!analysis) {
|
|
289
|
-
result.errors++;
|
|
401
|
+
if (!pendingEmails || pendingEmails.length === 0) {
|
|
402
|
+
logger.info('Worker: Queue empty', { userId });
|
|
290
403
|
return;
|
|
291
404
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
405
|
+
logger.info('Worker: Processing batch', { userId, count: pendingEmails.length });
|
|
406
|
+
for (const email of pendingEmails) {
|
|
407
|
+
await this.processPendingEmail(email, userId, settings);
|
|
408
|
+
}
|
|
409
|
+
// Slight delay to prevent hitting rate limits too fast, then check again
|
|
410
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
411
|
+
return this.processQueue(userId, settings);
|
|
412
|
+
}
|
|
413
|
+
async processPendingEmail(email, userId, settings) {
|
|
414
|
+
// Create a real processing log entry for this background task to ensure RLS compliance
|
|
415
|
+
const { data: log } = await this.supabase
|
|
416
|
+
.from('processing_logs')
|
|
417
|
+
.insert({
|
|
418
|
+
user_id: userId,
|
|
419
|
+
account_id: email.account_id,
|
|
420
|
+
status: 'running',
|
|
301
421
|
})
|
|
302
|
-
.
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
422
|
+
.select()
|
|
423
|
+
.single();
|
|
424
|
+
const eventLogger = log ? new EventLogger(this.supabase, log.id) : null;
|
|
425
|
+
try {
|
|
426
|
+
// 1. Double-check status and mark as processing (Atomic-ish)
|
|
427
|
+
const { data: current } = await this.supabase
|
|
428
|
+
.from('emails')
|
|
429
|
+
.select('processing_status')
|
|
430
|
+
.eq('id', email.id)
|
|
431
|
+
.single();
|
|
432
|
+
if (current?.processing_status !== 'pending') {
|
|
433
|
+
if (log)
|
|
434
|
+
await this.supabase.from('processing_logs').delete().eq('id', log.id);
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
await this.supabase
|
|
438
|
+
.from('emails')
|
|
439
|
+
.update({ processing_status: 'processing' })
|
|
440
|
+
.eq('id', email.id);
|
|
441
|
+
if (eventLogger)
|
|
442
|
+
await eventLogger.info('Processing', `Background processing: ${email.subject}`, undefined, email.id);
|
|
443
|
+
// 2. Read content from disk and parse with mailparser
|
|
444
|
+
if (!email.file_path)
|
|
445
|
+
throw new Error('No file path found for email');
|
|
446
|
+
const rawMime = await this.storageService.readEmail(email.file_path);
|
|
447
|
+
const parsed = await simpleParser(rawMime);
|
|
448
|
+
// Extract clean content (prioritize text)
|
|
449
|
+
const cleanContent = parsed.text || parsed.textAsHtml || '';
|
|
450
|
+
// Extract metadata signals from headers
|
|
451
|
+
const metadata = {
|
|
452
|
+
importance: parsed.headers.get('importance')?.toString() || parsed.headers.get('x-priority')?.toString(),
|
|
453
|
+
listUnsubscribe: parsed.headers.get('list-unsubscribe')?.toString(),
|
|
454
|
+
autoSubmitted: parsed.headers.get('auto-submitted')?.toString(),
|
|
455
|
+
mailer: parsed.headers.get('x-mailer')?.toString()
|
|
456
|
+
};
|
|
457
|
+
// 3. Analyze with AI
|
|
458
|
+
const intelligenceService = getIntelligenceService(settings?.llm_model || settings?.llm_base_url || settings?.llm_api_key
|
|
459
|
+
? {
|
|
460
|
+
model: settings.llm_model,
|
|
461
|
+
baseUrl: settings.llm_base_url,
|
|
462
|
+
apiKey: settings.llm_api_key,
|
|
463
|
+
}
|
|
464
|
+
: undefined);
|
|
465
|
+
const analysis = await intelligenceService.analyzeEmail(cleanContent, {
|
|
466
|
+
subject: email.subject || '',
|
|
467
|
+
sender: email.sender || '',
|
|
468
|
+
date: email.date || '',
|
|
469
|
+
metadata,
|
|
470
|
+
userPreferences: {
|
|
471
|
+
autoTrashSpam: settings?.auto_trash_spam,
|
|
472
|
+
smartDrafts: settings?.smart_drafts,
|
|
473
|
+
},
|
|
474
|
+
}, eventLogger || undefined, email.id);
|
|
475
|
+
if (!analysis) {
|
|
476
|
+
throw new Error('AI analysis returned no result');
|
|
477
|
+
}
|
|
478
|
+
// 4. Update the email record with results
|
|
479
|
+
await this.supabase
|
|
480
|
+
.from('emails')
|
|
481
|
+
.update({
|
|
482
|
+
category: analysis.category,
|
|
483
|
+
is_useless: analysis.is_useless,
|
|
484
|
+
ai_analysis: analysis,
|
|
485
|
+
suggested_actions: analysis.suggested_actions || [],
|
|
486
|
+
suggested_action: analysis.suggested_actions?.[0] || 'none',
|
|
487
|
+
processing_status: 'completed'
|
|
488
|
+
})
|
|
489
|
+
.eq('id', email.id);
|
|
490
|
+
// 5. Execute automation rules
|
|
491
|
+
// Fetch account and rules needed for execution
|
|
492
|
+
const { data: account } = await this.supabase
|
|
493
|
+
.from('email_accounts')
|
|
494
|
+
.select('*')
|
|
495
|
+
.eq('id', email.account_id)
|
|
496
|
+
.single();
|
|
497
|
+
const { data: rules } = await this.supabase
|
|
498
|
+
.from('rules')
|
|
499
|
+
.select('*')
|
|
500
|
+
.eq('user_id', userId)
|
|
501
|
+
.eq('is_enabled', true);
|
|
502
|
+
if (account && rules) {
|
|
503
|
+
const tempResult = { processed: 0, deleted: 0, drafted: 0, errors: 0 };
|
|
504
|
+
// Ensure email object for rules has the analysis fields merged in
|
|
505
|
+
const emailForRules = { ...email, ...analysis };
|
|
506
|
+
await this.executeRules(account, emailForRules, analysis, rules, settings, tempResult, eventLogger);
|
|
507
|
+
}
|
|
508
|
+
// Mark log as success
|
|
509
|
+
if (log) {
|
|
510
|
+
await this.supabase
|
|
511
|
+
.from('processing_logs')
|
|
512
|
+
.update({
|
|
513
|
+
status: 'success',
|
|
514
|
+
completed_at: new Date().toISOString(),
|
|
515
|
+
emails_processed: 1
|
|
516
|
+
})
|
|
517
|
+
.eq('id', log.id);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
catch (error) {
|
|
521
|
+
logger.error('Failed to process pending email', error, { emailId: email.id });
|
|
522
|
+
if (eventLogger)
|
|
523
|
+
await eventLogger.error('Processing Failed', error, email.id);
|
|
524
|
+
// Mark log as failed
|
|
525
|
+
if (log) {
|
|
526
|
+
await this.supabase
|
|
527
|
+
.from('processing_logs')
|
|
528
|
+
.update({
|
|
529
|
+
status: 'failed',
|
|
530
|
+
completed_at: new Date().toISOString(),
|
|
531
|
+
error_message: error instanceof Error ? error.message : String(error)
|
|
532
|
+
})
|
|
533
|
+
.eq('id', log.id);
|
|
534
|
+
}
|
|
535
|
+
await this.supabase
|
|
536
|
+
.from('emails')
|
|
537
|
+
.update({
|
|
538
|
+
processing_status: 'failed',
|
|
539
|
+
processing_error: error instanceof Error ? error.message : String(error),
|
|
540
|
+
retry_count: (email.retry_count || 0) + 1
|
|
541
|
+
})
|
|
542
|
+
.eq('id', email.id);
|
|
543
|
+
}
|
|
312
544
|
}
|
|
313
545
|
async executeRules(account, email, analysis, rules, settings, result, eventLogger) {
|
|
314
546
|
// User-defined and System rules (Unified)
|
|
@@ -408,7 +640,8 @@ export class EmailProcessorService {
|
|
|
408
640
|
case 'suggested_actions':
|
|
409
641
|
// Handle array membership check (e.g. if condition expects "reply" to be in actions)
|
|
410
642
|
const requiredActions = Array.isArray(value) ? value : [value];
|
|
411
|
-
const
|
|
643
|
+
const actualActions = analysis.suggested_actions || [];
|
|
644
|
+
const hasAllActions = requiredActions.every(req => actualActions.includes(req));
|
|
412
645
|
if (!hasAllActions)
|
|
413
646
|
return false;
|
|
414
647
|
break;
|
|
@@ -470,6 +703,9 @@ export class EmailProcessorService {
|
|
|
470
703
|
if (account.provider === 'gmail') {
|
|
471
704
|
if (action === 'delete') {
|
|
472
705
|
await this.gmailService.trashMessage(account, email.external_id);
|
|
706
|
+
if (email.file_path) {
|
|
707
|
+
await this.storageService.deleteEmail(email.file_path);
|
|
708
|
+
}
|
|
473
709
|
}
|
|
474
710
|
else if (action === 'archive') {
|
|
475
711
|
await this.gmailService.archiveMessage(account, email.external_id);
|
|
@@ -490,6 +726,9 @@ export class EmailProcessorService {
|
|
|
490
726
|
else if (account.provider === 'outlook') {
|
|
491
727
|
if (action === 'delete') {
|
|
492
728
|
await this.microsoftService.trashMessage(account, email.external_id);
|
|
729
|
+
if (email.file_path) {
|
|
730
|
+
await this.storageService.deleteEmail(email.file_path);
|
|
731
|
+
}
|
|
493
732
|
}
|
|
494
733
|
else if (action === 'archive') {
|
|
495
734
|
await this.microsoftService.archiveMessage(account, email.external_id);
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { createLogger } from '../utils/logger.js';
|
|
4
|
+
const logger = createLogger('StorageService');
|
|
5
|
+
export class StorageService {
|
|
6
|
+
defaultPath;
|
|
7
|
+
constructor() {
|
|
8
|
+
// Default to a folder in the user's home directory or current project
|
|
9
|
+
// Using project-relative path for now as discussed
|
|
10
|
+
this.defaultPath = path.resolve(process.cwd(), 'data', 'emails');
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Ensures the storage directory exists and is writable.
|
|
14
|
+
*/
|
|
15
|
+
async ensureDirectory(customPath) {
|
|
16
|
+
const targetPath = customPath || this.defaultPath;
|
|
17
|
+
try {
|
|
18
|
+
await fs.mkdir(targetPath, { recursive: true });
|
|
19
|
+
// Test writability
|
|
20
|
+
const testFile = path.join(targetPath, '.write_test');
|
|
21
|
+
await fs.writeFile(testFile, 'ok');
|
|
22
|
+
await fs.unlink(testFile);
|
|
23
|
+
return targetPath;
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
logger.error('Storage directory validation failed', error, { targetPath });
|
|
27
|
+
throw new Error(`Storage path "${targetPath}" is not accessible or writable.`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Saves raw email content to disk.
|
|
32
|
+
* Returns the absolute path to the saved file.
|
|
33
|
+
*/
|
|
34
|
+
async saveEmail(content, filename, customPath) {
|
|
35
|
+
const baseDir = await this.ensureDirectory(customPath);
|
|
36
|
+
const filePath = path.join(baseDir, filename);
|
|
37
|
+
try {
|
|
38
|
+
await fs.writeFile(filePath, content, 'utf8');
|
|
39
|
+
logger.debug('Email saved to disk', { filePath });
|
|
40
|
+
return filePath;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
logger.error('Failed to save email to disk', error, { filePath });
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Reads email content from disk.
|
|
49
|
+
*/
|
|
50
|
+
async readEmail(filePath) {
|
|
51
|
+
try {
|
|
52
|
+
return await fs.readFile(filePath, 'utf8');
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
logger.error('Failed to read email from disk', error, { filePath });
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Deletes email from disk.
|
|
61
|
+
*/
|
|
62
|
+
async deleteEmail(filePath) {
|
|
63
|
+
try {
|
|
64
|
+
await fs.unlink(filePath);
|
|
65
|
+
logger.debug('Email deleted from disk', { filePath });
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
// If file doesn't exist, we don't care much
|
|
69
|
+
if (error.code !== 'ENOENT') {
|
|
70
|
+
logger.warn('Failed to delete email from disk', { error, filePath });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
let storageService = null;
|
|
76
|
+
export function getStorageService() {
|
|
77
|
+
if (!storageService) {
|
|
78
|
+
storageService = new StorageService();
|
|
79
|
+
}
|
|
80
|
+
return storageService;
|
|
81
|
+
}
|