emailengine-app 2.69.0 → 2.71.0

package/lib/api-routes/template-routes.js

@@ -6,9 +6,12 @@ const getSecret = require('../get-secret');
 const { templates } = require('../templates');
 const Joi = require('joi');
 const { failAction } = require('../tools');
-const { handleError } = require('./route-helpers');
+const { handleError, throwNotFound } = require('./route-helpers');
 
-const { templateSchemas, accountIdSchema } = require('../schemas');
+const { templateSchemas, accountIdSchema, errorResponses } = require('../schemas');
+
+// Template owner field shared by all template response schemas
+const templateAccountIdSchema = accountIdSchema.required().allow(null).description('Account ID. Null for public templates');
 
 async function init(args) {
     const { server, call, CORS_CONFIG } = args;
@@ -52,7 +55,11 @@ async function init(args) {
             notes: 'Create a new stored template. Templates can be used when sending emails as the content of the message.',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -104,7 +111,7 @@ async function init(args) {
             response: {
                 schema: Joi.object({
                     created: Joi.boolean().description('Was the template created or not'),
-                    account: accountIdSchema.required(),
+                    account: templateAccountIdSchema,
                     id: Joi.string().max(256).required().example('example').description('Template ID')
                 }).label('CreateTemplateResponse'),
                 failAction: 'log'
@@ -136,7 +143,11 @@ async function init(args) {
             notes: 'Update a stored template.',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -182,7 +193,7 @@ async function init(args) {
             response: {
                 schema: Joi.object({
                     updated: Joi.boolean().description('Was the template updated or not'),
-                    account: accountIdSchema.required(),
+                    account: templateAccountIdSchema,
                     id: Joi.string().max(256).required().example('example').description('Template ID')
                 }).label('UpdateTemplateResponse'),
                 failAction: 'log'
@@ -213,7 +224,11 @@ async function init(args) {
             notes: 'Lists stored templates for the account',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -246,7 +261,7 @@ async function init(args) {
 
             response: {
                 schema: Joi.object({
-                    account: accountIdSchema.required(),
+                    account: templateAccountIdSchema,
                     total: Joi.number().integer().example(120).description('How many matching entries').label('TotalNumber'),
                     page: Joi.number().integer().example(0).description('Current page (0-based index)').label('PageNumber'),
                     pages: Joi.number().integer().example(24).description('Total page count').label('PagesNumber'),
@@ -284,7 +299,11 @@ async function init(args) {
 
         async handler(request) {
             try {
-                return await templates.get(request.params.template);
+                let templateData = await templates.get(request.params.template);
+                if (!templateData) {
+                    throwNotFound();
+                }
+                return templateData;
             } catch (err) {
                 handleError(request, err);
             }
@@ -295,7 +314,11 @@ async function init(args) {
             notes: 'Retrieve template content and information',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -317,7 +340,7 @@ async function init(args) {
 
             response: {
                 schema: Joi.object({
-                    account: accountIdSchema.required(),
+                    account: templateAccountIdSchema,
                     id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Template ID'),
                     name: Joi.string().max(256).example('Transaction receipt').description('Name of the template').label('TemplateName').required(),
                     description: Joi.string()
@@ -337,12 +360,7 @@ async function init(args) {
                         subject: templateSchemas.subject,
                         text: templateSchemas.text,
                         html: templateSchemas.html,
-                        previewText: templateSchemas.previewText,
-                        format: Joi.string()
-                            .valid('html', 'markdown')
-                            .default('html')
-                            .description('Markup language for HTML ("html" or "markdown")')
-                            .label('TemplateContentFormat')
+                        previewText: templateSchemas.previewText
                     }).label('RequestTemplateContent')
                 }).label('AccountTemplateResponse'),
                 failAction: 'log'
@@ -366,7 +384,11 @@ async function init(args) {
             notes: 'Delete a stored template',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -390,7 +412,7 @@ async function init(args) {
             response: {
                 schema: Joi.object({
                     deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the template deleted'),
-                    account: accountIdSchema.required(),
+                    account: templateAccountIdSchema,
                     id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Template ID')
                 }).label('DeleteTemplateResponse'),
                 failAction: 'log'
@@ -406,6 +428,13 @@ async function init(args) {
             let accountObject = new Account({ redis, account: request.params.account, call, secret: await getSecret() });
 
             try {
+                if (!request.query.force) {
+                    let err = new Error('Set the force=true query parameter to flush account templates');
+                    err.code = 'ForceRequired';
+                    err.statusCode = 400;
+                    throw err;
+                }
+
                 // throws if account does not exist
                 await accountObject.loadAccountData();
 
@@ -419,7 +448,11 @@ async function init(args) {
             notes: 'Delete all stored templates for an account',
             tags: ['api', 'Templates'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -451,7 +484,7 @@ async function init(args) {
 
             response: {
                 schema: Joi.object({
-                    deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Were the templates flushed'),
+                    flushed: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Were the templates flushed'),
                     account: accountIdSchema.required()
                 }).label('FlushTemplatesResponse'),
                 failAction: 'log'

package/lib/api-routes/token-routes.js

@@ -7,7 +7,50 @@ const getSecret = require('../get-secret');
 const tokens = require('../tokens');
 const { failAction } = require('../tools');
 const { handleError } = require('./route-helpers');
-const { accountIdSchema, tokenRestrictionsSchema, ipSchema, tokenIdSchema } = require('../schemas');
+const { accountIdSchema, tokenRestrictionsSchema, ipSchema, tokenIdSchema, errorResponses } = require('../schemas');
+
+const tokenAccessSchema = Joi.object({
+    time: Joi.date().iso().allow(null).example('2021-02-17T13:43:18.860Z').description('Last time this token was used. Null if the token has never been used'),
+    ip: ipSchema.allow(null).description('IP address of the last request that used this token. Null if the address was not available')
+})
+    .unknown()
+    .description('Token usage information')
+    .label('TokenAccess');
+
+const tokenScopesSchema = Joi.array()
+    .items(Joi.string().example('api').label('TokenScopeEntry'))
+    .description('Scopes this token is valid for')
+    .label('TokenScopes');
+
+const tokenMetadataSchema = Joi.string()
+    .empty('')
+    .max(1024 * 1024)
+    .custom((value, helpers) => {
+        try {
+            // check if parsing fails
+            JSON.parse(value);
+            return value;
+        } catch (err) {
+            return helpers.message('Metadata must be a valid JSON string');
+        }
+    })
+    .example('{"example": "value"}')
+    .description('Related metadata in JSON format')
+    .label('JsonMetaData');
+
+// Both token listings (root and account) return the same item shape from tokens.list(),
+// the account listing just adds the account ID
+const tokenListItemFields = {
+    description: Joi.string().empty('').trim().max(1024).example('Token description').description('Token description'),
+    metadata: tokenMetadataSchema,
+    ip: ipSchema.description('IP address of the requester').label('TokenIP'),
+    remoteAddress: ipSchema.description('IP address of the client that created the token').label('TokenRemoteAddress'),
+    scopes: tokenScopesSchema,
+    restrictions: tokenRestrictionsSchema,
+    created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this token was created'),
+    access: tokenAccessSchema,
+    id: tokenIdSchema
+};
 
 async function init(args) {
     const { server, call, CORS_CONFIG } = args;
@@ -42,7 +85,11 @@ async function init(args) {
             notes: 'Provisions a new access token for an account',
             tags: ['api', 'Access Tokens'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -73,21 +120,7 @@ async function init(args) {
                         )
                         .label('Scopes'),
 
-                    metadata: Joi.string()
-                        .empty('')
-                        .max(1024 * 1024)
-                        .custom((value, helpers) => {
-                            try {
-                                // check if parsing fails
-                                JSON.parse(value);
-                                return value;
-                            } catch (err) {
-                                return helpers.message('Metadata must be a valid JSON string');
-                            }
-                        })
-                        .example('{"example": "value"}')
-                        .description('Related metadata in JSON format')
-                        .label('JsonMetaData'),
+                    metadata: tokenMetadataSchema,
 
                     restrictions: tokenRestrictionsSchema,
 
@@ -120,7 +153,11 @@ async function init(args) {
             notes: 'Delete an access token',
             tags: ['api', 'Access Tokens'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -168,7 +205,11 @@ async function init(args) {
             notes: 'Lists access tokens registered for root access',
             tags: ['api', 'Access Tokens'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(401, 403, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -187,31 +228,7 @@ async function init(args) {
 
             response: {
                 schema: Joi.object({
-                    tokens: Joi.array()
-                        .items(
-                            Joi.object({
-                                account: accountIdSchema.required(),
-                                description: Joi.string().empty('').trim().max(1024).required().example('Token description').description('Token description'),
-                                metadata: Joi.string()
-                                    .empty('')
-                                    .max(1024 * 1024)
-                                    .custom((value, helpers) => {
-                                        try {
-                                            // check if parsing fails
-                                            JSON.parse(value);
-                                            return value;
-                                        } catch (err) {
-                                            return helpers.message('Metadata must be a valid JSON string');
-                                        }
-                                    })
-                                    .example('{"example": "value"}')
-                                    .description('Related metadata in JSON format')
-                                    .label('JsonMetaData'),
-                                ip: ipSchema.description('IP address of the requester').label('TokenIP'),
-                                id: tokenIdSchema
-                            }).label('RootTokensItem')
-                        )
-                        .label('RootTokensEntries')
+                    tokens: Joi.array().items(Joi.object(tokenListItemFields).label('RootTokensItem')).label('RootTokensEntries')
                 }).label('RootTokensResponse'),
                 failAction: 'log'
             }
@@ -236,7 +253,11 @@ async function init(args) {
             notes: 'Lists access tokens registered for an account',
             tags: ['api', 'Access Tokens'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -262,28 +283,7 @@ async function init(args) {
                         .items(
                             Joi.object({
                                 account: accountIdSchema.required(),
-                                description: Joi.string().empty('').trim().max(1024).required().example('Token description').description('Token description'),
-                                metadata: Joi.string()
-                                    .empty('')
-                                    .max(1024 * 1024)
-                                    .custom((value, helpers) => {
-                                        try {
-                                            // check if parsing fails
-                                            JSON.parse(value);
-                                            return value;
-                                        } catch (err) {
-                                            return helpers.message('Metadata must be a valid JSON string');
-                                        }
-                                    })
-                                    .example('{"example": "value"}')
-                                    .description('Related metadata in JSON format')
-                                    .label('JsonMetaData'),
-
-                                restrictions: tokenRestrictionsSchema,
-
-                                ip: ipSchema.description('IP address of the requester').label('TokenIP'),
-
-                                id: tokenIdSchema
+                                ...tokenListItemFields
                             }).label('AccountTokensItem')
                         )
                         .label('AccountTokensEntries')

package/lib/api-routes/webhook-route-routes.js

@@ -3,8 +3,20 @@
 const Joi = require('joi');
 const { webhooks: Webhooks } = require('../webhooks');
 const { failAction } = require('../tools');
-const { handleError } = require('./route-helpers');
-const { settingsSchema } = require('../schemas');
+const { handleError, throwNotFound } = require('./route-helpers');
+const { settingsSchema, errorResponses } = require('../schemas');
+
+const webhookErrorFlagSchema = Joi.object({
+    message: Joi.string().example('Request failed with status 500').description('Error message from the last failed delivery')
+})
+    .unknown()
+    .allow(null)
+    .description('Information about the last webhook delivery error. Null if no errors have been registered')
+    .label('WebhookRouteErrorFlag');
+
+const webhookCustomHeadersSchema = settingsSchema.webhooksCustomHeaders
+    .description('Custom HTTP headers added to webhook requests for this route')
+    .label('WebhookRouteCustomHeaders');
 
 async function init(args) {
     const { server, CORS_CONFIG } = args;
@@ -26,7 +38,11 @@ async function init(args) {
             notes: 'List custom webhook routes',
             tags: ['api', 'Webhooks'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -76,7 +92,9 @@ async function init(args) {
                                 updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was last updated'),
                                 enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
                                 targetUrl: settingsSchema.webhooks,
-                                tcount: Joi.number().integer().example(123).description('How many times this route has been applied')
+                                tcount: Joi.number().integer().example(123).description('How many times this route has been applied'),
+                                webhookErrorFlag: webhookErrorFlagSchema,
+                                customHeaders: webhookCustomHeadersSchema
                             }).label('WebhookRoutesListEntry')
                         )
                         .label('WebhookRoutesList')
@@ -92,7 +110,11 @@ async function init(args) {
 
         async handler(request) {
             try {
-                return await Webhooks.get(request.params.webhookRoute);
+                let webhookRouteData = await Webhooks.get(request.params.webhookRoute);
+                if (!webhookRouteData) {
+                    throwNotFound();
+                }
+                return webhookRouteData;
             } catch (err) {
                 handleError(request, err);
             }
@@ -103,7 +125,11 @@ async function init(args) {
             notes: 'Retrieve webhook route content and information',
             tags: ['api', 'Webhooks'],
 
-            plugins: {},
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
 
             auth: {
                 strategy: 'api-token',
@@ -138,9 +164,12 @@ async function init(args) {
                     enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
                     targetUrl: settingsSchema.webhooks,
                     tcount: Joi.number().integer().example(123).description('How many times this route has been applied'),
+                    v: Joi.number().integer().example(1).description('Internal version counter, increased on every update'),
+                    webhookErrorFlag: webhookErrorFlagSchema,
+                    customHeaders: webhookCustomHeadersSchema,
                     content: Joi.object({
-                        fn: Joi.string().example('return true;').description('Filter function'),
-                        map: Joi.string().example('payload.ts = Date.now(); return payload;').description('Mapping function')
+                        fn: Joi.string().allow(null).example('return true;').description('Filter function. Null if not set'),
+                        map: Joi.string().allow(null).example('payload.ts = Date.now(); return payload;').description('Mapping function. Null if not set')
                     }).label('WebhookRouteContent')
                 }).label('WebhookRouteResponse'),
                 failAction: 'log'

package/lib/autodetect-imap-settings.js

@@ -709,8 +709,6 @@ async function resolveUsingAutodiscovery(email, domain, source) {
         });
     });
 
-    resp = resp.filter(entry => entry.account);
-
     return { imap, smtp, _source: source || 'autodiscover' };
 }
 

package/lib/consts.js

@@ -107,6 +107,11 @@ module.exports = {
 
     DEFAULT_MAX_LOG_LINES: 10000,
 
+    // Shared Sentry instance run by the EmailEngine developers. Used as the error
+    // reporting target when error reporting is enabled without a custom DSN. A DSN
+    // is a write-only credential, it can only be used to submit events.
+    COMMUNITY_SENTRY_DSN: 'https://bdd958f3e813a488904b0f254e0bb8a8@sentry.emailengine.dev/3',
+
     PDKDF2_ITERATIONS: 600000,
     PDKDF2_SALT_SIZE: 16,
     PDKDF2_DIGEST: 'sha256', // 'sha512', 'sha256' or 'sha1'

package/lib/document-store.js

@@ -1,12 +1,33 @@
 'use strict';
 
+const config = require('@zone-eu/wild-config');
 const settings = require('./settings');
 const { Client: ElasticSearch } = require('@elastic/elasticsearch');
 const { ensureIndex } = require('./es');
+const { hasEnvValue, readEnvValue, getBoolean } = require('./tools');
+
+// Deployment-level gate for the deprecated Document Store feature. When this is false the
+// "documents" worker is not spawned, document-store-only endpoints are not registered, and
+// all runtime document-store code takes its existing "disabled" path. Set via the
+// --documentStore.enabled CLI flag / [documentStore] enabled config or EENGINE_DOCUMENT_STORE_ENABLED.
+const documentStoreFeatureEnabled = hasEnvValue('EENGINE_DOCUMENT_STORE_ENABLED')
+    ? getBoolean(readEnvValue('EENGINE_DOCUMENT_STORE_ENABLED'))
+    : getBoolean(config.documentStore && config.documentStore.enabled);
+
+// Effective runtime state: the feature must be available (gate) AND enabled in settings.
+// When the gate is off this is always false, so callers reuse the already-tested
+// "documentStoreEnabled is false" code paths.
+const isDocumentStoreEnabled = async () => documentStoreFeatureEnabled && !!(await settings.get('documentStoreEnabled'));
 
 const clientCache = { version: -1, config: false, client: false, index: false };
 
 const getESClient = async logger => {
+    // Feature gate is off: behave exactly as a disabled document store. Checked before any
+    // Redis access so disabled deployments do not pay a round-trip on every getESClient call.
+    if (!documentStoreFeatureEnabled) {
+        return clientCache;
+    }
+
     const documentStoreVersion = (await settings.get('documentStoreVersion')) || 0;
     if (clientCache.version === documentStoreVersion) {
         return clientCache;
@@ -51,4 +72,4 @@ const getESClient = async logger => {
     return clientCache;
 };
 
-module.exports = { getESClient };
+module.exports = { getESClient, documentStoreFeatureEnabled, isDocumentStoreEnabled };

package/lib/email-client/base-client.js

@@ -4,6 +4,7 @@ const { threadId: workerThreadId } = require('worker_threads');
 const crypto = require('crypto');
 const logger = require('../logger');
 const settings = require('../settings');
+const { isDocumentStoreEnabled } = require('../document-store');
 const msgpack = require('msgpack5')();
 const { templates } = require('../templates');
 const { Gateway } = require('../gateway');
@@ -240,6 +241,33 @@ class BaseClient {
         return rid;
     }
 
+    /**
+     * Normalizes an IMAP-style flag update request (add/delete/set) into plain add/delete
+     * flag lists. If `set` is present then it replaces the state of the flags the backend
+     * supports and add/delete are ignored, matching the IMAP backend behavior.
+     * @param {Object} [flags] - Flag update request ({ add, delete, set })
+     * @param {string[]} supportedFlags - Flags the backend can represent
+     * @returns {Object} Normalized flag lists ({ add: string[], delete: string[] })
+     */
+    normalizeFlagUpdates(flags, supportedFlags) {
+        if (flags?.set) {
+            // If set exists then ignore add/delete calls
+            let setFlags = [].concat(flags.set);
+            let addFlags = [];
+            let deleteFlags = [];
+            for (let flag of supportedFlags) {
+                if (setFlags.includes(flag)) {
+                    addFlags.push(flag);
+                } else {
+                    deleteFlags.push(flag);
+                }
+            }
+            return { add: addFlags, delete: deleteFlags };
+        }
+
+        return { add: [].concat(flags?.add || []), delete: [].concat(flags?.delete || []) };
+    }
+
     // Redis key generators for different data types
 
     getAccountKey() {
@@ -721,7 +749,7 @@ class BaseClient {
         }
 
         // use existing response
-        switch (idempotencyData.status) {
+        switch (idempotencyData?.status) {
             case 'completed':
                 // Return cached result
                 idempotencyData.returnValue = Object.assign({}, idempotencyData.result, {
@@ -1049,7 +1077,7 @@ class BaseClient {
         // Resolve reference and update reference/in-reply-to headers
         if (data.reference && data.reference.message) {
             // Try document store first if enabled
-            if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
+            if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
                 try {
                     referencedMessage = await this.accountObject.getMessage(data.reference.message, {
                         documentStore: true,
@@ -1551,7 +1579,7 @@ class BaseClient {
         // Resolve reference and update reference/in-reply-to headers
         if (data.reference && data.reference.message) {
             // Try document store first if enabled
-            if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
+            if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
                 try {
                     referencedMessage = await this.accountObject.getMessage(data.reference.message, {
                         documentStore: true,
@@ -3194,11 +3222,6 @@ class BaseClient {
     async handleSubmitError(err, context) {
         const { smtpSettings, networkRouting, gatewayData, gatewayObject, data, jobData, queueId, envelope } = context;
 
-        // Handle permanent failures
-        if (err.responseCode >= 500 && jobData.opts?.attempts <= jobData.attemptsMade) {
-            jobData.nextAttempt = false;
-        }
-
         // Build SMTP status from error
         const smtpStatus = SmtpErrorBuilder.buildStatus(err, smtpSettings, networkRouting);