emailengine-app 2.68.1 → 2.70.0

package/workers/submit.js

@@ -7,32 +7,12 @@ const config = require('@zone-eu/wild-config');
 const logger = require('../lib/logger');
 
 const { REDIS_PREFIX } = require('../lib/consts');
-const { getDuration, readEnvValue, threadStats, reloadHttpProxyAgent } = require('../lib/tools');
+const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
 const { webhooks: Webhooks } = require('../lib/webhooks');
 const settings = require('../lib/settings');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('submit');
 
 const util = require('util');
 const { redis, queueConf, submitQueue } = require('../lib/db');
@@ -222,13 +202,15 @@ const submitWorker = new Worker(
             }
 
             let backoffDelay = Number(job.opts.backoff && job.opts.backoff.delay) || 0;
-            let nextAttempt = job.attemptsMade < job.opts.attempts ? Math.round(job.processedOn + Math.pow(2, job.attemptsMade) * backoffDelay) : false;
+            // job.attemptsMade is not yet incremented for the ongoing attempt, so the
+            // next retry (if this attempt fails) is delayed by 2^attemptsMade * base
+            let nextAttempt = job.attemptsMade + 1 < job.opts.attempts ? Math.round(job.processedOn + Math.pow(2, job.attemptsMade) * backoffDelay) : false;
 
             queueEntry.job = {
                 id: job.id,
                 attemptsMade: job.attemptsMade,
                 attempts: job.opts.attempts,
-                nextAttempt: new Date(nextAttempt).toISOString()
+                nextAttempt: nextAttempt ? new Date(nextAttempt).toISOString() : false
             };
 
             let res = await accountObject.submitMessage(queueEntry);
@@ -409,12 +391,24 @@ submitWorker.on('failed', async job => {
                 logger.error({ msg: 'Failed to remove queue entry', account: job.data.account, queueId: job.data.queueId, messageId: job.data.messageId, err });
             }
             // report as failed
-            await notify(job.data.account, EMAIL_FAILED_NOTIFY, {
-                messageId: job.data.messageId,
-                queueId: job.data.queueId,
-                error: job.stacktrace && job.stacktrace[0] && job.stacktrace[0].split('\n').shift(),
-                networkRouting: job.progress?.networkRouting
-            });
+            try {
+                await notify(job.data.account, EMAIL_FAILED_NOTIFY, {
+                    messageId: job.data.messageId,
+                    queueId: job.data.queueId,
+                    error: job.stacktrace && job.stacktrace[0] && job.stacktrace[0].split('\n').shift(),
+                    networkRouting: job.progress?.networkRouting
+                });
+            } catch (notifyErr) {
+                // A failed webhook notification must not bubble out of this BullMQ
+                // event listener as an unhandled rejection and take down the worker.
+                logger.error({
+                    msg: 'Failed to deliver submission failure notification',
+                    account: job.data.account,
+                    queueId: job.data.queueId,
+                    messageId: job.data.messageId,
+                    err: notifyErr
+                });
+            }
         }
     }
 
@@ -497,10 +491,7 @@ parentPort.on('message', message => {
     }
 
     if (message && message.cmd === 'settings') {
-        let d = message.data || {};
-        if ('httpProxyEnabled' in d || 'httpProxyUrl' in d) {
-            reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
-        }
+        maybeReloadHttpProxyAgent(message.data);
     }
 });
 

package/workers/webhooks.js

@@ -10,30 +10,11 @@ const { webhooks: Webhooks } = require('../lib/webhooks');
 
 const { GooglePubSub } = require('../lib/oauth/pubsub/google');
 
-const { readEnvValue, threadStats, getDuration, httpAgent, getServiceSecret, reloadHttpProxyAgent } = require('../lib/tools');
-
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { readEnvValue, threadStats, getDuration, httpAgent, getServiceSecret, maybeReloadHttpProxyAgent } = require('../lib/tools');
+const { sendWebhookRequest } = require('../lib/webhook-request');
+
+const { initSentry } = require('../lib/sentry');
+initSentry('webhooks');
 
 const { redis, queueConf } = require('../lib/db');
 const { Worker } = require('bullmq');
@@ -199,10 +180,7 @@ parentPort.on('message', message => {
     }
 
     if (message && message.cmd === 'settings') {
-        let d = message.data || {};
-        if ('httpProxyEnabled' in d || 'httpProxyUrl' in d) {
-            reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
-        }
+        maybeReloadHttpProxyAgent(message.data);
     }
 });
 
@@ -424,9 +402,9 @@ const notifyWorker = new Worker(
         headers['X-EE-Wh-Signature'] = hmac.digest('base64url');
 
         try {
-            let res;
+            let status;
             try {
-                res = await fetchCmd(parsed.toString(), {
+                status = await sendWebhookRequest(fetchCmd, parsed.toString(), {
                     method: 'post',
                     body,
                     headers,
@@ -438,18 +416,6 @@ const notifyWorker = new Worker(
                 throw err.cause || err;
             }
 
-            if (!res.ok) {
-                // Drain response body to release connection back to pool
-                try {
-                    await res.text();
-                } catch {
-                    // ignore drain errors
-                }
-                let err = new Error(res.statusText || `Invalid response: ${res.status} ${res.statusText}`);
-                err.statusCode = res.status;
-                throw err;
-            }
-
             logger.trace({
                 msg: 'Webhook posted',
                 action: 'webhook',
@@ -460,7 +426,7 @@ const notifyWorker = new Worker(
                 requestBodySize: body.length,
                 accountWebhooks: !!accountWebhooks,
                 event: job.name,
-                status: res.status,
+                status,
                 account: job.data.account,
                 route: customRoute && customRoute.id
             });