emailengine-app 2.68.1 → 2.69.0

package/lib/api-routes/token-routes.js

@@ -0,0 +1,297 @@
+'use strict';
+
+const Joi = require('joi');
+const { redis } = require('../db');
+const { Account } = require('../account');
+const getSecret = require('../get-secret');
+const tokens = require('../tokens');
+const { failAction } = require('../tools');
+const { handleError } = require('./route-helpers');
+const { accountIdSchema, tokenRestrictionsSchema, ipSchema, tokenIdSchema } = require('../schemas');
+
+async function init(args) {
+    const { server, call, CORS_CONFIG } = args;
+
+    server.route({
+        method: 'POST',
+        path: '/v1/token',
+
+        async handler(request) {
+            let accountObject = new Account({
+                redis,
+                account: request.payload.account,
+                call,
+                secret: await getSecret(),
+                timeout: request.headers['x-ee-timeout']
+            });
+
+            try {
+                // throws if account does not exist
+                await accountObject.loadAccountData();
+
+                let token = await tokens.provision(Object.assign({}, request.payload, { remoteAddress: request.app.ip }));
+
+                return { token };
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'Provision an access token',
+            notes: 'Provisions a new access token for an account',
+            tags: ['api', 'Access Tokens'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+
+                payload: Joi.object({
+                    account: accountIdSchema.required(),
+
+                    description: Joi.string().empty('').trim().max(1024).required().example('Token description').description('Token description'),
+
+                    scopes: Joi.array()
+                        .items(Joi.string().valid('api', 'smtp', 'imap-proxy').label('TokenScope'))
+                        .single()
+                        .default(['api'])
+                        .required()
+                        .description(
+                            'Token permission scopes: "api" for REST API access, "smtp" for SMTP submission, "imap-proxy" for IMAP proxy authentication'
+                        )
+                        .label('Scopes'),
+
+                    metadata: Joi.string()
+                        .empty('')
+                        .max(1024 * 1024)
+                        .custom((value, helpers) => {
+                            try {
+                                // check if parsing fails
+                                JSON.parse(value);
+                                return value;
+                            } catch (err) {
+                                return helpers.message('Metadata must be a valid JSON string');
+                            }
+                        })
+                        .example('{"example": "value"}')
+                        .description('Related metadata in JSON format')
+                        .label('JsonMetaData'),
+
+                    restrictions: tokenRestrictionsSchema,
+
+                    ip: ipSchema.description('IP address of the requester').label('TokenIP')
+                }).label('CreateToken')
+            },
+
+            response: {
+                schema: Joi.object({
+                    token: Joi.string().length(64).hex().required().example('123456').description('Access token')
+                }).label('CreateTokenResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
+    server.route({
+        method: 'DELETE',
+        path: '/v1/token/{token}',
+
+        async handler(request) {
+            try {
+                return { deleted: await tokens.delete(request.params.token, { remoteAddress: request.app.ip }) };
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+        options: {
+            description: 'Remove a token',
+            notes: 'Delete an access token',
+            tags: ['api', 'Access Tokens'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+
+                params: Joi.object({
+                    token: Joi.string().length(64).hex().required().example('123456').description('Access token')
+                }).label('DeleteTokenRequest')
+            },
+
+            response: {
+                schema: Joi.object({
+                    deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the token deleted')
+                }).label('DeleteTokenRequestResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
+    server.route({
+        method: 'GET',
+        path: '/v1/tokens',
+
+        async handler(request) {
+            try {
+                // TODO: allow paging
+                return { tokens: (await tokens.list(null, 0, 1000)).tokens };
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'List root tokens',
+            notes: 'Lists access tokens registered for root access',
+            tags: ['api', 'Access Tokens'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction
+            },
+
+            response: {
+                schema: Joi.object({
+                    tokens: Joi.array()
+                        .items(
+                            Joi.object({
+                                account: accountIdSchema.required(),
+                                description: Joi.string().empty('').trim().max(1024).required().example('Token description').description('Token description'),
+                                metadata: Joi.string()
+                                    .empty('')
+                                    .max(1024 * 1024)
+                                    .custom((value, helpers) => {
+                                        try {
+                                            // check if parsing fails
+                                            JSON.parse(value);
+                                            return value;
+                                        } catch (err) {
+                                            return helpers.message('Metadata must be a valid JSON string');
+                                        }
+                                    })
+                                    .example('{"example": "value"}')
+                                    .description('Related metadata in JSON format')
+                                    .label('JsonMetaData'),
+                                ip: ipSchema.description('IP address of the requester').label('TokenIP'),
+                                id: tokenIdSchema
+                            }).label('RootTokensItem')
+                        )
+                        .label('RootTokensEntries')
+                }).label('RootTokensResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
+    server.route({
+        method: 'GET',
+        path: '/v1/tokens/account/{account}',
+
+        async handler(request) {
+            try {
+                // TODO: allow paging
+                return { tokens: (await tokens.list(request.params.account, 0, 1000)).tokens };
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'List account tokens',
+            notes: 'Lists access tokens registered for an account',
+            tags: ['api', 'Access Tokens'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: accountIdSchema.required()
+                })
+            },
+
+            response: {
+                schema: Joi.object({
+                    tokens: Joi.array()
+                        .items(
+                            Joi.object({
+                                account: accountIdSchema.required(),
+                                description: Joi.string().empty('').trim().max(1024).required().example('Token description').description('Token description'),
+                                metadata: Joi.string()
+                                    .empty('')
+                                    .max(1024 * 1024)
+                                    .custom((value, helpers) => {
+                                        try {
+                                            // check if parsing fails
+                                            JSON.parse(value);
+                                            return value;
+                                        } catch (err) {
+                                            return helpers.message('Metadata must be a valid JSON string');
+                                        }
+                                    })
+                                    .example('{"example": "value"}')
+                                    .description('Related metadata in JSON format')
+                                    .label('JsonMetaData'),
+
+                                restrictions: tokenRestrictionsSchema,
+
+                                ip: ipSchema.description('IP address of the requester').label('TokenIP'),
+
+                                id: tokenIdSchema
+                            }).label('AccountTokensItem')
+                        )
+                        .label('AccountTokensEntries')
+                }).label('AccountsTokensResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+}
+
+module.exports = init;

package/lib/api-routes/webhook-route-routes.js

@@ -0,0 +1,152 @@
+'use strict';
+
+const Joi = require('joi');
+const { webhooks: Webhooks } = require('../webhooks');
+const { failAction } = require('../tools');
+const { handleError } = require('./route-helpers');
+const { settingsSchema } = require('../schemas');
+
+async function init(args) {
+    const { server, CORS_CONFIG } = args;
+
+    server.route({
+        method: 'GET',
+        path: '/v1/webhookRoutes',
+
+        async handler(request) {
+            try {
+                return await Webhooks.list(request.query.page, request.query.pageSize);
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'List webhook routes',
+            notes: 'List custom webhook routes',
+            tags: ['api', 'Webhooks'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+
+                query: Joi.object({
+                    page: Joi.number()
+                        .integer()
+                        .min(0)
+                        .max(1024 * 1024)
+                        .default(0)
+                        .example(0)
+                        .description('Page number (zero indexed, so use 0 for first page)')
+                        .label('PageNumber'),
+                    pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page').label('PageSize')
+                }).label('WebhookRoutesListRequest')
+            },
+
+            response: {
+                schema: Joi.object({
+                    total: Joi.number().integer().example(120).description('How many matching entries').label('TotalNumber'),
+                    page: Joi.number().integer().example(0).description('Current page (0-based index)').label('PageNumber'),
+                    pages: Joi.number().integer().example(24).description('Total page count').label('PagesNumber'),
+
+                    webhooks: Joi.array()
+                        .items(
+                            Joi.object({
+                                id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Webhook ID'),
+                                name: Joi.string().max(256).example('Send to Slack').description('Name of the route').label('WebhookRouteName').required(),
+                                description: Joi.string()
+                                    .allow('')
+                                    .max(1024)
+                                    .example('Something about the route')
+                                    .description('Optional description of the webhook route')
+                                    .label('WebhookRouteDescription'),
+                                created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was created'),
+                                updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was last updated'),
+                                enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
+                                targetUrl: settingsSchema.webhooks,
+                                tcount: Joi.number().integer().example(123).description('How many times this route has been applied')
+                            }).label('WebhookRoutesListEntry')
+                        )
+                        .label('WebhookRoutesList')
+                }).label('WebhookRoutesListResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
+    server.route({
+        method: 'GET',
+        path: '/v1/webhookRoutes/webhookRoute/{webhookRoute}',
+
+        async handler(request) {
+            try {
+                return await Webhooks.get(request.params.webhookRoute);
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'Get webhook route information',
+            notes: 'Retrieve webhook route content and information',
+            tags: ['api', 'Webhooks'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    webhookRoute: Joi.string().max(256).required().example('example').description('Webhook ID')
+                }).label('GetWebhookRouteRequest')
+            },
+
+            response: {
+                schema: Joi.object({
+                    id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Webhook ID'),
+                    name: Joi.string().max(256).example('Send to Slack').description('Name of the route').label('WebhookRouteName').required(),
+                    description: Joi.string()
+                        .allow('')
+                        .max(1024)
+                        .example('Something about the route')
+                        .description('Optional description of the webhook route')
+                        .label('WebhookRouteDescription'),
+                    created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was created'),
+                    updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was last updated'),
+                    enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
+                    targetUrl: settingsSchema.webhooks,
+                    tcount: Joi.number().integer().example(123).description('How many times this route has been applied'),
+                    content: Joi.object({
+                        fn: Joi.string().example('return true;').description('Filter function'),
+                        map: Joi.string().example('payload.ts = Date.now(); return payload;').description('Mapping function')
+                    }).label('WebhookRouteContent')
+                }).label('WebhookRouteResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+}
+
+module.exports = init;

package/lib/email-client/gmail-client.js

@@ -2700,6 +2700,20 @@ class GmailClient extends BaseClient {
             queryParts.push(search.gmailRaw);
         }
 
+        // Label filters - "has" matches messages with the label, "not" excludes them
+        if (search.labels && typeof search.labels === 'object') {
+            for (let label of [].concat(search.labels.has || [])) {
+                if (label) {
+                    queryParts.push(`label:${this.formatSearchTerm(label)}`);
+                }
+            }
+            for (let label of [].concat(search.labels.not || [])) {
+                if (label) {
+                    queryParts.push(`-label:${this.formatSearchTerm(label)}`);
+                }
+            }
+        }
+
         // body search
         if (search.body && typeof search.body === 'string') {
             queryParts.push(`${this.formatSearchTerm(search.body)}`);

package/lib/email-client/imap/mailbox.js

@@ -23,7 +23,6 @@ const {
     MESSAGE_DELETED_NOTIFY,
     MESSAGE_UPDATED_NOTIFY,
     MESSAGE_MISSING_NOTIFY,
-    MAILBOX_RESET_NOTIFY,
     MAILBOX_NEW_NOTIFY,
     EMAIL_BOUNCE_NOTIFY,
     EMAIL_COMPLAINT_NOTIFY,
@@ -41,6 +40,7 @@ const {
     canUseCondstorePartialSync,
     canUseSimplePartialSync,
     canSkipSync,
+    shouldSeedLostIndex,
     FULL_SYNC_DELAY
 } = require('./sync-operations');
 
@@ -154,6 +154,8 @@ class Mailbox {
         let data = await this.connection.redis.hgetall(this.getMailboxKey());
         data = data || {};
 
+        let hasStoredState = Object.keys(data).length > 0;
+
         // Log diagnostic info if stored uidValidity is invalid or missing
         if (!validUidValidity(data.uidValidity)) {
             this.logger.warn({
@@ -162,12 +164,16 @@ class Mailbox {
                 redisKey: this.getMailboxKey(),
                 rawUidValidity: data.uidValidity,
                 rawUidValidityType: typeof data.uidValidity,
-                hasData: Object.keys(data).length > 0,
+                hasData: hasStoredState,
                 storedKeys: Object.keys(data)
             });
         }
 
         return {
+            // True when the mailbox hash held any fields at all. Redis eviction removes
+            // whole keys, so this distinguishes "state lost" from "individual fields
+            // never persisted" (e.g. uidNext on servers that omit UIDNEXT from SELECT)
+            hasStoredState,
             path: data.path || this.path,
             uidValidity: validUidValidity(data.uidValidity) ? BigInt(data.uidValidity) : false,
             highestModseq: data.highestModseq && !isNaN(data.highestModseq) ? BigInt(data.highestModseq) : false,
@@ -1934,6 +1940,16 @@ class Mailbox {
         return this.syncOps.partialSync(storedStatus);
     }
 
+    /**
+     * Silently rebuilds the message index after lost or invalidated sync state
+     * Delegates to SyncOperations
+     * @param {Object} mailboxStatus - Current mailbox status from IMAP
+     * @param {Object} [options] - Reseed options (reason, prevUidValidity)
+     */
+    async seedMailboxIndex(mailboxStatus, options) {
+        return this.syncOps.seedMailboxIndex(mailboxStatus, options);
+    }
+
     /**
      * Processes queued notification events after sync
      * Fetches full message details and sends notifications
@@ -2060,6 +2076,15 @@ class Mailbox {
         try {
             let storedStatus = await this.getStoredStatus();
 
+            // Lost-index recovery: the account has synced in a previous session but this folder
+            // has no stored state at all (e.g. Redis evicted the whole hash) while the server
+            // still has messages. Rebuild the baseline silently instead of replaying every
+            // message as a new email.
+            if (shouldSeedLostIndex(storedStatus, mailboxStatus, this.previouslyConnected)) {
+                await this.seedMailboxIndex(mailboxStatus, { reason: 'syncStateLost' });
+                return false;
+            }
+
             // Store initial UID on first sync
             if (storedStatus.uidNext === false && typeof mailboxStatus.uidNext === 'number') {
                 // update first UID
@@ -2088,19 +2113,17 @@ class Mailbox {
                 });
 
                 this.logger.debug({ msg: 'Mailbox reset', path: this.listingEntry.path });
-                await this.connection.notify(this, MAILBOX_RESET_NOTIFY, {
-                    path: this.listingEntry.path,
-                    name: this.listingEntry.name,
-                    specialUse: this.listingEntry.specialUse || false,
-                    uidValidity: validUidValidity(mailboxStatus.uidValidity) ? mailboxStatus.uidValidity.toString() : false,
-                    prevUidValidity: validUidValidity(storedStatus.uidValidity) ? storedStatus.uidValidity.toString() : false
-                });
 
                 // do not advertise messages as new
                 this.listingEntry.isNew = true;
 
-                // generates blank stored status as the Redis key was deleted
-                storedStatus = await this.getStoredStatus();
+                // Rebuild the index silently from the recreated mailbox and emit a single
+                // mailboxReset (with prev/current UIDVALIDITY) instead of replaying every message.
+                await this.seedMailboxIndex(mailboxStatus, {
+                    reason: 'uidValidityChange',
+                    prevUidValidity: validUidValidity(storedStatus.uidValidity) ? storedStatus.uidValidity.toString() : false
+                });
+                return false;
             }
 
             // Determine sync strategy using helper functions

package/lib/email-client/imap/subconnection.js

@@ -210,24 +210,32 @@ class Subconnection extends EventEmitter {
         let response = await imapClient.connect();
 
         // Process untagged EXISTS responses
-        imapClient.on('exists', async event => {
-            if (!event || !event.path) {
-                return; //?
-            }
+        imapClient.on('exists', event => {
+            try {
+                if (!event || !event.path) {
+                    return; //?
+                }
 
-            this.logger.info({ msg: 'Exists notification', account: this.account, event });
+                this.logger.info({ msg: 'Exists notification', account: this.account, event });
 
-            this.requestSync();
+                this.requestSync();
+            } catch (err) {
+                this.logger.error({ msg: 'Exists notification handling failed', account: this.account, err });
+            }
         });
 
-        imapClient.on('flags', async event => {
-            if (!event || !event.path) {
-                return; //?
-            }
+        imapClient.on('flags', event => {
+            try {
+                if (!event || !event.path) {
+                    return; //?
+                }
 
-            this.logger.info({ msg: 'Flags notification', account: this.account, event });
+                this.logger.info({ msg: 'Flags notification', account: this.account, event });
 
-            this.requestSync();
+                this.requestSync();
+            } catch (err) {
+                this.logger.error({ msg: 'Flags notification handling failed', account: this.account, err });
+            }
         });
 
         return response;