emailengine-app 2.61.5 → 2.62.0

package/lib/export.js

@@ -0,0 +1,528 @@
+'use strict';
+
+const os = require('os');
+const crypto = require('crypto');
+const fs = require('fs');
+const pathlib = require('path');
+const msgpack = require('msgpack5')();
+const { redis, exportQueue } = require('./db');
+const settings = require('./settings');
+const logger = require('./logger');
+const getSecret = require('./get-secret');
+
+const {
+    REDIS_PREFIX,
+    DEFAULT_EXPORT_MAX_AGE,
+    DEFAULT_EXPORT_MAX_CONCURRENT,
+    DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT,
+    DEFAULT_EXPORT_MAX_MESSAGE_SIZE
+} = require('./consts');
+
+const EXPORT_ID_PREFIX = 'exp_';
+const ACTIVE_EXPORTS_KEY = `${REDIS_PREFIX}exp:active`;
+
+// Atomically check concurrent limits AND add to active set if under limits.
+// Eliminates TOCTOU race by combining check and add in a single operation.
+const CONCURRENT_CHECK_AND_ADD_SCRIPT = `
+local activeKey = KEYS[1]
+local maxConcurrent = tonumber(ARGV[1])
+local maxGlobal = tonumber(ARGV[2])
+local accountPrefix = ARGV[3]
+local activeEntry = ARGV[4]
+
+local members = redis.call('SMEMBERS', activeKey)
+local globalCount = #members
+
+-- Check global limit first (faster)
+if globalCount >= maxGlobal then
+    return 0
+end
+
+-- Check per-account limit
+local accountCount = 0
+for _, member in ipairs(members) do
+    if string.sub(member, 1, #accountPrefix) == accountPrefix then
+        accountCount = accountCount + 1
+    end
+end
+
+if accountCount >= maxConcurrent then
+    return 0
+end
+
+-- Under both limits - atomically add to active set
+redis.call('SADD', activeKey, activeEntry)
+return 1
+`;
+
+function generateExportId() {
+    return EXPORT_ID_PREFIX + crypto.randomBytes(12).toString('hex');
+}
+
+function createError(message, code, statusCode) {
+    const err = new Error(message);
+    err.code = code;
+    err.statusCode = statusCode;
+    return err;
+}
+
+async function tryAddToActiveSet(account, exportId) {
+    const maxConcurrent = (await settings.get('exportMaxConcurrent')) || DEFAULT_EXPORT_MAX_CONCURRENT;
+    const maxGlobalConcurrent = (await settings.get('exportMaxGlobalConcurrent')) || DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT;
+
+    const accountPrefix = `${account}:`;
+    const activeEntry = `${account}:${exportId}`;
+
+    const added = await redis.eval(CONCURRENT_CHECK_AND_ADD_SCRIPT, 1, ACTIVE_EXPORTS_KEY, maxConcurrent, maxGlobalConcurrent, accountPrefix, activeEntry);
+
+    if (!added) {
+        throw createError('Maximum concurrent exports reached', 'TooManyExports', 429);
+    }
+
+    return activeEntry;
+}
+
+async function scanKeys(pattern) {
+    const keys = [];
+    let cursor = '0';
+    do {
+        const [nextCursor, batch] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
+        cursor = nextCursor;
+        keys.push(...batch);
+    } while (cursor !== '0');
+    return keys;
+}
+
+function getExportKey(account, exportId) {
+    return `${REDIS_PREFIX}exp:${account}:${exportId}`;
+}
+
+function getExportQueueKey(account, exportId) {
+    return `${REDIS_PREFIX}exq:${account}:${exportId}`;
+}
+
+async function getExportPath() {
+    return (await settings.get('exportPath')) || process.env.EENGINE_EXPORT_PATH || os.tmpdir();
+}
+
+async function getExportMaxAge() {
+    const settingsMaxAge = await settings.get('exportMaxAge');
+    const envMaxAge = process.env.EENGINE_EXPORT_MAX_AGE;
+
+    if (settingsMaxAge && !isNaN(settingsMaxAge)) return Number(settingsMaxAge);
+    if (envMaxAge && !isNaN(envMaxAge)) return Number(envMaxAge);
+    return DEFAULT_EXPORT_MAX_AGE;
+}
+
+function toTimestamp(date) {
+    const ts = new Date(date).getTime();
+    if (isNaN(ts)) {
+        throw new Error('Invalid date value');
+    }
+    return ts;
+}
+
+function calculateScore(timestamp, folder, messageId, uid) {
+    const baseTimestamp = timestamp instanceof Date ? timestamp.getTime() : Number(timestamp) || Date.now();
+    const baseSeconds = Math.floor(baseTimestamp / 1000);
+
+    const uniqueKey = `${folder || ''}:${messageId || ''}:${uid || ''}`;
+    const hash = crypto.createHash('sha256').update(uniqueKey).digest();
+    const tiebreaker = (((hash[0] << 24) | (hash[1] << 16) | (hash[2] << 8) | hash[3]) >>> 0) % 1000000;
+
+    return baseSeconds * 1000000 + tiebreaker;
+}
+
+class Export {
+    static async create(account, options) {
+        const accountData = await redis.hgetall(`${REDIS_PREFIX}iad:${account}`);
+        if (!accountData || !accountData.account) {
+            throw createError('Account not found', 'AccountNotFound', 404);
+        }
+
+        const exportId = generateExportId();
+        const exportKey = getExportKey(account, exportId);
+
+        const maxAge = await getExportMaxAge();
+        const exportPath = await getExportPath();
+        const now = Date.now();
+        const expiresAt = now + maxAge;
+
+        const activeEntry = await tryAddToActiveSet(account, exportId);
+
+        await fs.promises.mkdir(exportPath, { recursive: true });
+
+        const secret = await getSecret();
+        const isEncrypted = !!secret;
+        const fileExtension = isEncrypted ? '.ndjson.gz.enc' : '.ndjson.gz';
+        const filePath = pathlib.join(exportPath, `${exportId}${fileExtension}`);
+        const startDate = toTimestamp(options.startDate);
+        const endDate = toTimestamp(options.endDate);
+
+        const exportData = {
+            exportId,
+            account,
+            status: 'queued',
+            phase: 'pending',
+            folders: JSON.stringify(options.folders || []),
+            startDate,
+            endDate,
+            textType: options.textType || '*',
+            maxBytes: options.maxBytes || 5 * 1024 * 1024,
+            includeAttachments: options.includeAttachments ? '1' : '0',
+            isEncrypted: isEncrypted ? '1' : '0',
+            foldersScanned: 0,
+            foldersTotal: 0,
+            messagesQueued: 0,
+            messagesExported: 0,
+            messagesSkipped: 0,
+            bytesWritten: 0,
+            filePath,
+            lastProcessedScore: 0,
+            created: now,
+            expiresAt,
+            error: ''
+        };
+
+        const ttl = Math.ceil(maxAge / 1000);
+
+        await redis.multi().hmset(exportKey, exportData).expire(exportKey, ttl).exec();
+
+        try {
+            await exportQueue.add('export', { account, exportId }, { jobId: exportId, removeOnComplete: true, removeOnFail: true });
+        } catch (err) {
+            await redis.multi().srem(ACTIVE_EXPORTS_KEY, activeEntry).del(exportKey).exec();
+            throw err;
+        }
+
+        logger.info({
+            msg: 'Export job created',
+            account,
+            exportId,
+            folders: options.folders,
+            startDate: new Date(startDate).toISOString(),
+            endDate: new Date(endDate).toISOString()
+        });
+
+        return {
+            exportId,
+            status: 'queued',
+            created: new Date(now).toISOString()
+        };
+    }
+
+    static async isCancelled(account, exportId) {
+        const status = await redis.hget(getExportKey(account, exportId), 'status');
+        return status === 'cancelled' || !status;
+    }
+
+    static async get(account, exportId) {
+        const data = await redis.hgetall(getExportKey(account, exportId));
+        return data && data.exportId ? Export.formatStatus(data) : null;
+    }
+
+    static formatStatus(data) {
+        const toIsoDate = value => (value ? new Date(Number(value)).toISOString() : undefined);
+
+        const result = {
+            exportId: data.exportId,
+            status: data.status,
+            phase: data.phase !== 'pending' ? data.phase : undefined,
+            folders: data.folders ? JSON.parse(data.folders) : [],
+            startDate: toIsoDate(data.startDate),
+            endDate: toIsoDate(data.endDate),
+            isEncrypted: data.isEncrypted === '1',
+            progress: {
+                foldersScanned: Number(data.foldersScanned) || 0,
+                foldersTotal: Number(data.foldersTotal) || 0,
+                messagesQueued: Number(data.messagesQueued) || 0,
+                messagesExported: Number(data.messagesExported) || 0,
+                messagesSkipped: Number(data.messagesSkipped) || 0,
+                bytesWritten: Number(data.bytesWritten) || 0
+            },
+            created: toIsoDate(data.created),
+            expiresAt: toIsoDate(data.expiresAt),
+            error: data.error || null
+        };
+
+        return result;
+    }
+
+    static async list(account, options = {}) {
+        const page = Number(options.page) || 0;
+        const pageSize = Number(options.pageSize) || 20;
+
+        const pattern = `${REDIS_PREFIX}exp:${account}:${EXPORT_ID_PREFIX}*`;
+        const keys = await scanKeys(pattern);
+
+        const exports = [];
+        for (const key of keys) {
+            const data = await redis.hgetall(key);
+            if (data && data.exportId) {
+                exports.push({
+                    exportId: data.exportId,
+                    status: data.status,
+                    created: data.created ? new Date(Number(data.created)).toISOString() : undefined,
+                    expiresAt: data.expiresAt ? new Date(Number(data.expiresAt)).toISOString() : undefined
+                });
+            }
+        }
+
+        exports.sort((a, b) => new Date(b.created) - new Date(a.created));
+
+        const total = exports.length;
+        const pages = Math.ceil(total / pageSize) || 1;
+
+        return {
+            total,
+            page,
+            pages,
+            exports: exports.slice(page * pageSize, (page + 1) * pageSize)
+        };
+    }
+
+    static async delete(account, exportId) {
+        const exportKey = getExportKey(account, exportId);
+        const queueKey = getExportQueueKey(account, exportId);
+
+        const data = await redis.hgetall(exportKey);
+        if (!data || !data.exportId) {
+            return false;
+        }
+
+        const isActive = data.status === 'processing';
+
+        // Signal cancellation for active exports
+        if (isActive) {
+            await redis.hset(exportKey, 'status', 'cancelled');
+        }
+
+        const job = await exportQueue.getJob(exportId).catch(() => null);
+        if (job) {
+            await job.remove().catch(() => {});
+        }
+
+        // Active exports: worker will clean up when it detects cancellation
+        // Non-active exports: clean up immediately
+        if (!isActive) {
+            await redis.multi().del(exportKey).del(queueKey).srem(ACTIVE_EXPORTS_KEY, `${account}:${exportId}`).exec();
+
+            if (data.filePath) {
+                await fs.promises.unlink(data.filePath).catch(() => {});
+            }
+        }
+
+        logger.info({ msg: 'Export deleted', account, exportId });
+        return true;
+    }
+
+    static async update(account, exportId, updates) {
+        if (Object.keys(updates).length > 0) {
+            await redis.hmset(getExportKey(account, exportId), updates);
+        }
+    }
+
+    static async queueMessage(account, exportId, messageInfo) {
+        const queueKey = getExportQueueKey(account, exportId);
+        const exportKey = getExportKey(account, exportId);
+
+        const score = calculateScore(messageInfo.date, messageInfo.folder, messageInfo.messageId, messageInfo.uid);
+
+        const value = msgpack
+            .encode({
+                folder: messageInfo.folder,
+                messageId: messageInfo.messageId,
+                uid: messageInfo.uid,
+                size: messageInfo.size || 0
+            })
+            .toString('base64url');
+
+        const ttl = await redis.ttl(exportKey);
+        const multi = redis.multi().zadd(queueKey, score, value).hincrby(exportKey, 'messagesQueued', 1);
+        if (ttl > 0) {
+            multi.expire(queueKey, ttl);
+        }
+        await multi.exec();
+    }
+
+    static async getNextBatch(account, exportId, lastScore, limit) {
+        const queueKey = getExportQueueKey(account, exportId);
+        // Use exclusive lower bound to avoid re-processing messages at batch boundaries
+        const minScore = lastScore > 0 ? '(' + lastScore : lastScore;
+        const results = await redis.zrangebyscore(queueKey, minScore, '+inf', 'WITHSCORES', 'LIMIT', 0, limit);
+
+        const messages = [];
+        for (let i = 0; i < results.length; i += 2) {
+            try {
+                const info = msgpack.decode(Buffer.from(results[i], 'base64url'));
+                messages.push({ ...info, score: Number(results[i + 1]) });
+            } catch (err) {
+                logger.error({ msg: 'Failed to decode message info', account, exportId, err });
+            }
+        }
+
+        return messages;
+    }
+
+    static async incrementExported(account, exportId, bytesWritten = 0) {
+        const exportKey = getExportKey(account, exportId);
+        await redis.multi().hincrby(exportKey, 'messagesExported', 1).hincrby(exportKey, 'bytesWritten', bytesWritten).exec();
+    }
+
+    static async incrementSkipped(account, exportId) {
+        await redis.hincrby(getExportKey(account, exportId), 'messagesSkipped', 1);
+    }
+
+    static async updateLastProcessedScore(account, exportId, score) {
+        await redis.hset(getExportKey(account, exportId), 'lastProcessedScore', score);
+    }
+
+    static async complete(account, exportId) {
+        const exportKey = getExportKey(account, exportId);
+        const queueKey = getExportQueueKey(account, exportId);
+
+        await redis
+            .multi()
+            .hmset(exportKey, { status: 'completed', phase: 'complete' })
+            .del(queueKey)
+            .srem(ACTIVE_EXPORTS_KEY, `${account}:${exportId}`)
+            .exec();
+
+        logger.info({ msg: 'Export completed', account, exportId });
+    }
+
+    static async fail(account, exportId, error) {
+        const exportKey = getExportKey(account, exportId);
+        const queueKey = getExportQueueKey(account, exportId);
+
+        const exportData = await redis.hgetall(exportKey);
+        if (!exportData || !exportData.exportId) {
+            // Key already deleted, just clean up active set
+            await redis.srem(ACTIVE_EXPORTS_KEY, `${account}:${exportId}`);
+            return;
+        }
+
+        await redis
+            .multi()
+            .hmset(exportKey, {
+                status: 'failed',
+                error: error || 'Unknown error'
+            })
+            .del(queueKey)
+            .srem(ACTIVE_EXPORTS_KEY, `${account}:${exportId}`)
+            .exec();
+
+        logger.error({ msg: 'Export failed', account, exportId, error });
+    }
+
+    static async markInterruptedAsFailed() {
+        const activeExports = await redis.smembers(ACTIVE_EXPORTS_KEY);
+
+        for (const entry of activeExports) {
+            try {
+                // Find ':exp_' as separator since account IDs may contain colons
+                const separatorIndex = entry.indexOf(':exp_');
+                if (separatorIndex === -1) continue;
+                const account = entry.substring(0, separatorIndex);
+                const exportId = entry.substring(separatorIndex + 1);
+                if (!account || !exportId) continue;
+
+                const data = await redis.hgetall(getExportKey(account, exportId));
+
+                if (!data || !data.exportId) {
+                    // Stale entry -- export hash expired, clean up active set
+                    await redis.srem(ACTIVE_EXPORTS_KEY, entry);
+                    continue;
+                }
+
+                if (data && (data.status === 'processing' || data.status === 'queued' || data.status === 'indexing' || data.status === 'cancelled')) {
+                    const job = await exportQueue.getJob(exportId).catch(() => null);
+                    if (job) {
+                        await job.remove().catch(() => {});
+                        logger.info({ msg: 'Removed interrupted export job from queue', account, exportId });
+                    }
+
+                    await Export.fail(account, exportId, 'Export interrupted by application restart');
+
+                    if (data.filePath) {
+                        await fs.promises.unlink(data.filePath).catch(() => {});
+                    }
+                }
+            } catch (err) {
+                logger.error({ msg: 'Failed to mark interrupted export as failed', entry, err });
+            }
+        }
+    }
+
+    static async cleanup() {
+        const exportPath = await getExportPath();
+        let cleaned = 0;
+
+        try {
+            const files = await fs.promises.readdir(exportPath);
+
+            for (const file of files) {
+                if (!file.startsWith(EXPORT_ID_PREFIX)) continue;
+
+                const exportId = file.split('.')[0];
+                const pattern = `${REDIS_PREFIX}exp:*:${exportId}`;
+                const keys = await scanKeys(pattern);
+
+                if (keys.length === 0) {
+                    try {
+                        await fs.promises.unlink(pathlib.join(exportPath, file));
+                        cleaned++;
+                        logger.info({ msg: 'Cleaned up orphaned export file', file });
+                    } catch (err) {
+                        logger.error({ msg: 'Failed to clean up export file', file, err });
+                    }
+                }
+            }
+        } catch (err) {
+            logger.error({ msg: 'Failed to list export directory', exportPath, err });
+        }
+
+        return cleaned;
+    }
+
+    static async getFile(account, exportId) {
+        const data = await redis.hgetall(getExportKey(account, exportId));
+
+        if (!data || !data.exportId) {
+            return null;
+        }
+
+        if (data.status !== 'completed') {
+            throw createError('Export not completed', 'ExportNotReady', 400);
+        }
+
+        if (!data.filePath) {
+            throw createError('Export file not found', 'FileNotFound', 404);
+        }
+
+        try {
+            await fs.promises.access(data.filePath, fs.constants.R_OK);
+        } catch {
+            throw createError('Export file not found', 'FileNotFound', 404);
+        }
+
+        const isEncrypted = data.isEncrypted === '1';
+
+        return {
+            filePath: data.filePath,
+            filename: `${exportId}.ndjson.gz`,
+            isEncrypted
+        };
+    }
+}
+
+module.exports = {
+    Export,
+    generateExportId,
+    calculateScore,
+    getExportKey,
+    getExportQueueKey,
+    getExportPath,
+    getExportMaxAge,
+    DEFAULT_EXPORT_MAX_MESSAGE_SIZE
+};

package/lib/oauth/gmail.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const packageData = require('../../package.json');
-const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');
+const { formatPartialSecretKey, structuredClone, fetchAgent, retryAgent, formatTokenError } = require('../tools');
 const crypto = require('crypto');
 
 const { fetch: fetchCmd } = require('undici');
@@ -125,10 +125,10 @@ const checkForUserFlags = err => {
 };
 
 const formatFetchBody = (searchParams, logRaw) => {
-    let data = Object.fromEntries(searchParams);
+    let entries = typeof searchParams === 'string' ? new URLSearchParams(searchParams) : searchParams;
+    let data = Object.fromEntries(entries);
 
     if (logRaw) {
-        // no changes needed
         return data;
     }
 
@@ -138,7 +138,7 @@ const formatFetchBody = (searchParams, logRaw) => {
         }
     }
 
-    if (data.assertion && !logRaw) {
+    if (data.assertion) {
         let [payload, signature] = data.assertion.toString().split('.');
         data.assertion = [payload, formatPartialSecretKey(signature)].join('.');
     }
@@ -256,7 +256,7 @@ class GmailOauth {
 
         return {
             url: url.origin + url.pathname,
-            body: url.searchParams
+            body: url.searchParams.toString()
         };
     }
 
@@ -326,9 +326,9 @@ class GmailOauth {
                 code
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
-                if (this.clientSecret && EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response.error_description)) {
+                if (this.clientSecret && EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response?.error_description)) {
                     // key might have been invalidated or renewed
                     err.tokenRequest.clientSecret = formatPartialSecretKey(this.clientSecret);
                 }
@@ -342,6 +342,7 @@ class GmailOauth {
                 // ignore
             }
 
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
@@ -381,13 +382,14 @@ class GmailOauth {
         let requestUrl = url.origin + url.pathname;
         let method = 'post';
 
+        const bodyString = url.searchParams.toString();
         const fetchOpts = {
             method,
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded',
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
-            body: url.searchParams
+            body: bodyString
         };
 
         let res = await fetchCmd(
@@ -438,9 +440,9 @@ class GmailOauth {
                 scopes: this.scopes
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
-                if (this.clientSecret && EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response.error_description)) {
+                if (this.clientSecret && EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response?.error_description)) {
                     // key might have been invalidated or renewed
                     err.tokenRequest.clientSecret = formatPartialSecretKey(this.clientSecret);
                 }
@@ -459,13 +461,14 @@ class GmailOauth {
             } catch (e) {
                 // ignore
             }
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
         // clear potential auth flag
         await this.setFlag();
 
-        return await responseJson;
+        return responseJson;
     }
 
     async request(accessToken, url, method, payload, options) {
@@ -485,11 +488,16 @@ class GmailOauth {
             if (!Buffer.isBuffer(payload)) {
                 reqData.headers.Accept = 'application/json';
                 reqData.headers['Content-Type'] = options?.contentType || 'application/json';
-                payload = Buffer.from(JSON.stringify(payload));
+                // Use string body instead of Buffer to avoid ArrayBuffer detachment on retry
+                reqData.body = JSON.stringify(payload);
             } else {
                 reqData.headers['Content-Type'] = options?.contentType || 'application/x-www-form-urlencoded';
+                reqData.body = payload;
+                if (payload.length > 0) {
+                    // Non-empty buffers use non-retry dispatcher to prevent ArrayBuffer detachment
+                    reqData.dispatcher = fetchAgent;
+                }
             }
-            reqData.body = payload;
         } else if (payload && method === 'get') {
             let parsedUrl = new URL(url);
             for (let key of Object.keys(payload)) {