emailengine-app 2.71.0 → 2.72.1

package/lib/oauth/gmail.js

@@ -824,6 +824,9 @@ class GmailOauth {
 }
 
 module.exports.GmailOauth = GmailOauth;
+// Exported for unit testing the auth error-to-flag mapping.
+module.exports.checkForFlags = checkForFlags;
+module.exports.checkForUserFlags = checkForUserFlags;
 module.exports.GMAIL_SCOPES = GMAIL_SCOPES;
 module.exports.GMAIL_API_SCOPES = GMAIL_API_SCOPES;
 module.exports.OPENID_SCOPES = OPENID_SCOPES;

package/lib/oauth/outlook.js

@@ -701,3 +701,6 @@ class OutlookOauth {
 module.exports.OutlookOauth = OutlookOauth;
 module.exports.outlookScopes = outlookScopes;
 module.exports.OUTLOOK_API_SCOPES = OUTLOOK_API_SCOPES;
+// Exported for unit testing the auth error-to-flag mapping.
+module.exports.checkForFlags = checkForFlags;
+module.exports.checkForUserFlags = checkForUserFlags;

package/lib/oauth2-apps.js

@@ -852,8 +852,10 @@ class OAuth2AppsHandler {
             return { id, deleted: false, accounts: 0 };
         }
 
-        // Acquire the same lock used by ensurePubsub to prevent racing with recovery
-        let needsPubSubLock = appData.pubSubTopic || appData.baseScopes === 'pubsub';
+        // Acquire the same lock used by ensurePubsub to prevent racing with recovery. Mirror the
+        // deletion gate below (pubSubTopic || pubSubSubscription) so an app that only has a
+        // subscription recorded is still locked before its resources are deleted.
+        let needsPubSubLock = appData.pubSubTopic || appData.pubSubSubscription || appData.baseScopes === 'pubsub';
         let delLock;
 
         if (needsPubSubLock) {
@@ -867,11 +869,21 @@ class OAuth2AppsHandler {
                 logger.error({ msg: 'Failed to acquire lock for pubsub app deletion', lockKey, err });
                 throw err;
             }
+
+            // Re-read under the lock so the ownership-flag deletion decision below acts on the
+            // current record, not the snapshot taken before the lock - a concurrent ensurePubsub /
+            // recovery run may have adopted or created a resource in the meantime. Keep the pre-lock
+            // snapshot if the app was deleted while waiting, so best-effort GCP cleanup still runs.
+            let lockedData = await this.get(id);
+            if (lockedData) {
+                appData = lockedData;
+            }
         }
 
         try {
-            if (appData.pubSubTopic) {
-                // try to delete topic
+            if (appData.pubSubTopic || appData.pubSubSubscription) {
+                // try to delete the Pub/Sub resources EmailEngine created (deleteTopic skips any
+                // resource marked unmanaged, i.e. adopted from a pre-existing GCP setup)
                 try {
                     await this.deleteTopic(appData);
                 } catch (err) {
@@ -1037,8 +1049,16 @@ class OAuth2AppsHandler {
             throw new Error('Failed to get access token');
         }
 
-        await this._deletePubSubResource(client, accessToken, appData, 'subscription', appData.pubSubSubscription);
-        await this._deletePubSubResource(client, accessToken, appData, 'topic', appData.pubSubTopic);
+        // Only delete resources EmailEngine provisioned. A `false` managed flag means the resource
+        // was adopted from a pre-existing GCP setup and must be left in place. Legacy apps created
+        // before ownership tracking have the flag undefined and are treated as managed (deleted),
+        // preserving the prior behavior.
+        if (appData.pubSubSubscriptionManaged !== false) {
+            await this._deletePubSubResource(client, accessToken, appData, 'subscription', appData.pubSubSubscription);
+        }
+        if (appData.pubSubTopicManaged !== false) {
+            await this._deletePubSubResource(client, accessToken, appData, 'topic', appData.pubSubTopic);
+        }
     }
 
     async ensurePubsub(appData) {
@@ -1093,6 +1113,23 @@ class OAuth2AppsHandler {
         }
 
         try {
+            // Re-read inside the lock so the "persist only if absent" guards below act on the
+            // current state, not the caller's pre-lock snapshot (which can be stale after a
+            // concurrent ensurePubsub run). Falls back to the passed-in data if the app was
+            // deleted while waiting for the lock (preserves the prior proceed-anyway behavior).
+            let current = (await this.get(appData.id)) || appData;
+
+            // Record a marker for a Pub/Sub resource that already existed in GCP (adopted, not
+            // created by us): mark it unmanaged so deletion never removes it, and surface it in
+            // results on first adoption so the create/update route restarts the pull worker. The
+            // !current guard keeps re-runs no-ops (no redundant writes or restarts).
+            const adoptExistingResource = async (key, value) => {
+                if (!current[key]) {
+                    await this.update(appData.id, { [key]: value, [`${key}Managed`]: false }, { partial: true });
+                    results[key] = value;
+                }
+            };
+
             let client = await this.getClient(appData.id);
 
             // Step 1. Get access token for service client
@@ -1109,6 +1146,9 @@ class OAuth2AppsHandler {
                 /*
                 {name: 'projects/...'}
             */
+
+                // Adopt a pre-existing topic so the Gmail watch can arm (kept unmanaged - we did not create it).
+                await adoptExistingResource('pubSubTopic', topicName);
             } catch (err) {
                 if (TRANSIENT_NETWORK_CODES.has(err.code)) {
                     logger.warn({ msg: 'Network error checking Pub/Sub topic', app: appData.id, code: err.code });
@@ -1149,7 +1189,9 @@ class OAuth2AppsHandler {
                             await this.update(
                                 appData.id,
                                 {
-                                    pubSubTopic: topicName
+                                    pubSubTopic: topicName,
+                                    // EmailEngine created this topic, so it may delete it on app removal.
+                                    pubSubTopicManaged: true
                                 },
                                 { partial: true }
                             );
@@ -1169,8 +1211,9 @@ class OAuth2AppsHandler {
                                     });
                                     throw err;
                                 case 409:
-                                    // already exists
+                                    // already exists (raced) - adopt it; mark unmanaged to be safe against deletion.
                                     logger.info({ msg: 'Topic already exists', app: appData.id, topic: topicName });
+                                    await adoptExistingResource('pubSubTopic', topicName);
                                     break;
                                 default:
                                     throw err;
@@ -1193,12 +1236,44 @@ class OAuth2AppsHandler {
                 {name: 'projects/...'}
             */
 
+                // Adopt a pre-existing subscription so the pull worker recognizes it (also stops the
+                // recovery loop in lib/oauth/pubsub/google.js); kept unmanaged - we did not create it.
+                await adoptExistingResource('pubSubSubscription', subscriptionName);
+
                 // Patch existing subscription's expiration policy if it differs from desired
                 // When no explicit TTL is configured, use Google's default (31 days) for comparison
                 let effectivePolicy = desiredExpirationPolicy ?? { ttl: GMAIL_PUBSUB_DEFAULT_EXPIRATION_TTL };
                 let currentTtl = subscriptionData?.expirationPolicy?.ttl || null;
                 let desiredTtl = effectivePolicy.ttl || null;
-                if (currentTtl !== desiredTtl) {
+
+                // Adopted (pre-existing, unmanaged) subscriptions belong to the customer - never
+                // rewrite their lifecycle policy. A subscription is ours to manage only when it is
+                // already recorded as managed (true) or predates ownership tracking (undefined);
+                // a freshly adopted one (marker absent in the pre-lock snapshot) or an explicit
+                // `false` flag is hands-off.
+                let subscriptionManagedByUs = !!current.pubSubSubscription && current.pubSubSubscriptionManaged !== false;
+
+                if (!subscriptionManagedByUs && desiredExpirationPolicy !== null) {
+                    // Do not mutate the customer's subscription, but warn when an explicitly
+                    // configured expiration is longer than theirs: the subscription may expire on
+                    // its own and silently disable Gmail push.
+                    let toSeconds = ttl => {
+                        let n = ttl ? parseInt(ttl, 10) : NaN;
+                        return Number.isFinite(n) ? n : null;
+                    };
+                    let currentSeconds = toSeconds(currentTtl);
+                    let desiredSeconds = toSeconds(desiredTtl);
+                    // desiredSeconds === null means EmailEngine wants an indefinite subscription
+                    if (currentSeconds !== null && (desiredSeconds === null || currentSeconds < desiredSeconds)) {
+                        logger.warn({
+                            msg: 'Adopted Pub/Sub subscription expires sooner than the configured expiration; leaving it unchanged',
+                            app: appData.id,
+                            subscription: subscriptionName,
+                            currentTtl,
+                            desiredTtl
+                        });
+                    }
+                } else if (subscriptionManagedByUs && currentTtl !== desiredTtl) {
                     try {
                         await client.request(accessToken, subscriptionUrl, 'PATCH', {
                             subscription: { expirationPolicy: effectivePolicy },
@@ -1271,7 +1346,9 @@ class OAuth2AppsHandler {
                             await this.update(
                                 appData.id,
                                 {
-                                    pubSubSubscription: subscriptionName
+                                    pubSubSubscription: subscriptionName,
+                                    // EmailEngine created this subscription, so it may delete it on app removal.
+                                    pubSubSubscriptionManaged: true
                                 },
                                 { partial: true }
                             );
@@ -1296,8 +1373,9 @@ class OAuth2AppsHandler {
                                     });
                                     throw err;
                                 case 409:
-                                    // already exists
+                                    // already exists (raced) - adopt it; mark unmanaged to be safe against deletion.
                                     logger.info({ msg: 'Subscription already exists', app: appData.id, subscription: subscriptionName });
+                                    await adoptExistingResource('pubSubSubscription', subscriptionName);
                                     break;
                                 default:
                                     throw err;
@@ -1350,6 +1428,17 @@ class OAuth2AppsHandler {
 
             if (existingPolicy) {
                 logger.debug({ msg: 'Gmail publisher policy already exists', app: appData.id, topic: topicName });
+
+                // The required publisher binding is already in place (granted manually or by a
+                // prior run). Record the marker so the Gmail watch can arm - it gates on this being
+                // truthy. Store the same {members, role} object shape the create path uses. No
+                // ownership flag: this marker has no deletion side effect (only read at the watch
+                // gate). Only populate results (and write) on first adoption, so a re-run is a no-op
+                // and does not trigger a redundant pull-worker restart.
+                if (!current.pubSubIamPolicy) {
+                    results.iamPolicy = { members: [member], role };
+                    await this.update(appData.id, { pubSubIamPolicy: results.iamPolicy }, { partial: true });
+                }
             } else {
                 logger.debug({ msg: 'Granting access to Gmail publisher', app: appData.id, topic: topicName });
                 let existingBindings = (getIamPolicyRes && getIamPolicyRes.bindings) || [];

package/lib/smtp-auth.js

@@ -0,0 +1,70 @@
+'use strict';
+
+// SMTP submission-server authentication handler. Extracted from workers/smtp.js
+// so the auth logic can be unit tested without booting the worker thread (which
+// connects to a parentPort and starts listening at require time).
+
+const logger = require('./logger');
+const settings = require('./settings');
+const { redis } = require('./db');
+const { Account } = require('./account');
+const getSecret = require('./get-secret');
+const { validateAuthToken, REASON_MESSAGES } = require('./auth-token');
+
+/**
+ * Builds the SMTP server onAuth handler.
+ *
+ * @param {Object} deps
+ * @param {WeakMap} deps.accountCache - session -> Account cache shared with the worker
+ * @param {Function} deps.call - RPC function passed to the Account instance
+ * @returns {Function} async onAuth(auth, session)
+ */
+function createSmtpAuthHandler({ accountCache, call }) {
+    return async function onAuth(auth, session) {
+        if (!session.eeAuthEnabled) {
+            throw new Error('Authentication not enabled');
+        }
+
+        let account = auth.username;
+
+        let smtpPassword = await settings.get('smtpServerPassword');
+        if (!smtpPassword || auth.password !== smtpPassword) {
+            // fall back to API token authentication
+            let result = await validateAuthToken({
+                password: auth.password,
+                account: auth.username,
+                requiredScope: 'smtp',
+                remoteAddress: session.remoteAddress
+            });
+
+            if (!result.authenticated) {
+                throw new Error(REASON_MESSAGES[result.reason] || 'Failed to authenticate user');
+            }
+        }
+
+        let accountObject = new Account({ account, redis, call, secret: await getSecret() });
+        let accountData;
+        try {
+            accountData = await accountObject.loadAccountData();
+        } catch (err) {
+            let respErr = new Error('Failed to authenticate user');
+
+            if (!err.output || err.output.statusCode !== 404) {
+                // only log non-obvious errors
+                logger.error({ msg: 'Failed to load account data', account: auth.username, err });
+                respErr.statusCode = 454;
+            }
+
+            throw respErr;
+        }
+
+        if (!accountData) {
+            throw new Error('Failed to authenticate user');
+        }
+
+        accountCache.set(session, accountObject);
+        return { user: accountData.account };
+    };
+}
+
+module.exports = { createSmtpAuthHandler };

package/lib/sub-script.js

@@ -67,11 +67,17 @@ class SubScript {
         }
     }
 
-    async exec(payload) {
+    async exec(payload, options) {
         if (!this.fn) {
             throw new Error('Subscript not compiled');
         }
 
+        // Allow callers (and tests) to shorten the execution timeout, but never
+        // raise it above SUBSCRIPT_RUNTIME_TIMEOUT - the sandbox cap is a hard
+        // ceiling. A falsy override (including 0, which vm treats as "no limit")
+        // falls back to the default so the cap can never be disabled.
+        let runtimeTimeout = options && options.timeout ? Math.min(options.timeout, SUBSCRIPT_RUNTIME_TIMEOUT) : SUBSCRIPT_RUNTIME_TIMEOUT;
+
         let start = Date.now();
 
         try {
@@ -93,7 +99,7 @@ class SubScript {
 
             vm.createContext(ctx);
             let result = await this.fn.runInContext(ctx, {
-                timeout: SUBSCRIPT_RUNTIME_TIMEOUT,
+                timeout: runtimeTimeout,
                 microtaskMode: 'afterEvaluate'
             });
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "emailengine-app",
-    "version": "2.71.0",
+    "version": "2.72.1",
     "private": false,
     "productTitle": "EmailEngine",
     "description": "Email Sync Engine",
@@ -13,6 +13,8 @@
         "test": "NODE_ENV=test grunt",
         "test:unit": "NODE_ENV=test grunt test-unit",
         "test:integration": "NODE_ENV=test grunt test-integration",
+        "test:e2e": "playwright test",
+        "test:e2e:install": "playwright install --with-deps chromium",
         "lint": "npx eslint 'lib/**/*.js' 'workers/**/*.js' 'test/**/*.js' server.js Gruntfile.js",
         "swagger": "./getswagger.sh",
         "build-source": "rm -rf node_modules && npm install && rm -rf node_modules && npm ci --omit=dev && rm -rf node_modules/ace-builds node_modules/@postalsys/ee-client && ./update-info.sh",
@@ -45,8 +47,8 @@
     },
     "homepage": "https://emailengine.app/",
     "dependencies": {
-        "@bull-board/api": "8.0.0",
-        "@bull-board/hapi": "8.0.0",
+        "@bull-board/api": "8.0.1",
+        "@bull-board/hapi": "8.0.1",
         "@elastic/elasticsearch": "8.15.3",
         "@hapi/accept": "6.0.3",
         "@hapi/bell": "13.1.0",
@@ -54,25 +56,25 @@
         "@hapi/cookie": "12.0.1",
         "@hapi/crumb": "9.0.1",
         "@hapi/hapi": "21.4.9",
-        "@hapi/inert": "7.1.1",
+        "@hapi/inert": "7.1.2",
         "@hapi/vision": "7.0.3",
         "@phc/pbkdf2": "1.1.14",
         "@postalsys/bounce-classifier": "3.0.0",
         "@postalsys/certs": "1.0.15",
         "@postalsys/ee-client": "1.3.0",
-        "@postalsys/email-ai-tools": "1.13.6",
-        "@postalsys/email-text-tools": "2.4.7",
+        "@postalsys/email-ai-tools": "1.13.8",
+        "@postalsys/email-text-tools": "2.4.8",
         "@postalsys/gettext": "4.1.1",
         "@postalsys/joi-messages": "1.0.5",
         "@postalsys/templates": "2.0.1",
-        "@sentry/node": "10.58.0",
+        "@sentry/node": "10.59.0",
         "@simplewebauthn/browser": "13.3.0",
         "@simplewebauthn/server": "13.3.1",
         "@zone-eu/mailsplit": "5.4.12",
         "@zone-eu/wild-config": "1.7.5",
         "ace-builds": "1.44.0",
         "base32.js": "0.1.0",
-        "bullmq": "5.78.1",
+        "bullmq": "5.79.0",
         "compare-versions": "6.1.1",
         "dotenv": "17.4.2",
         "encoding-japanese": "2.2.0",
@@ -86,7 +88,7 @@
         "html-to-text": "10.0.0",
         "ical.js": "1.5.0",
         "iconv-lite": "0.7.2",
-        "imapflow": "1.4.1",
+        "imapflow": "1.4.2",
         "ioredfour": "1.4.1",
         "ioredis": "5.11.1",
         "ipaddr.js": "2.4.0",
@@ -96,39 +98,40 @@
         "libmime": "5.3.8",
         "libqp": "2.1.1",
         "license-checker": "25.0.1",
-        "mailparser": "3.9.10",
-        "marked": "9.1.6",
+        "mailparser": "3.9.11",
+        "marked": "15.0.12",
         "minimist": "1.2.8",
         "msgpack5": "6.0.2",
         "murmurhash": "2.0.1",
-        "nanoid": "3.3.8",
-        "nodemailer": "9.0.0",
+        "nanoid": "3.3.13",
+        "nodemailer": "9.0.1",
         "pino": "10.3.1",
         "popper.js": "1.16.1",
         "prom-client": "15.1.3",
         "psl": "1.15.0",
-        "pubface": "1.1.3",
+        "pubface": "1.1.4",
         "punycode.js": "2.3.1",
         "qrcode": "1.5.4",
-        "smtp-server": "3.19.0",
+        "smtp-server": "3.19.1",
         "socks": "2.8.9",
         "speakeasy": "2.0.0",
         "startbootstrap-sb-admin-2": "3.3.7",
         "timezones-list": "3.1.0",
-        "undici": "7.25.0",
+        "undici": "7.28.0",
         "xml2js": "0.6.2"
     },
     "devDependencies": {
         "@eslint/js": "10.0.1",
+        "@playwright/test": "1.61.0",
         "acorn": "8.17.0",
         "acorn-walk": "8.3.5",
-        "chai": "4.3.10",
+        "chai": "4.5.0",
         "eerawlog": "1.5.3",
         "eslint": "10.5.0",
         "grunt": "1.6.2",
         "grunt-cli": "1.5.0",
         "grunt-shell-spawn": "0.5.0",
-        "pino-pretty": "13.0.0",
+        "pino-pretty": "13.1.3",
         "prettier": "3.8.4",
         "resedit": "3.0.2",
         "spdx-satisfies": "6.0.0",

package/playwright.config.js

@@ -0,0 +1,45 @@
+'use strict';
+
+// Playwright config for the EmailEngine happy-path e2e suite (test/e2e). Boots a fresh
+// EmailEngine via the webServer command (flush the isolated Redis db 14, then `node server.js`
+// with NODE_ENV=e2e -> config/e2e.toml) and drives it with a real browser.
+//
+// Run once:  npm run test:e2e:install   (fetch the Chromium browser)
+// Run suite: npm run test:e2e
+
+const { defineConfig, devices } = require('@playwright/test');
+
+const PORT = 7099;
+const BASE_URL = `http://127.0.0.1:${PORT}`;
+
+module.exports = defineConfig({
+    testDir: './test/e2e',
+    testMatch: '**/*.spec.js',
+    // One worker against one app + one database keeps the fresh-instance flow predictable.
+    fullyParallel: false,
+    workers: 1,
+    forbidOnly: !!process.env.CI,
+    retries: process.env.CI ? 1 : 0,
+    // The single happy-path test chains several slow external steps (Ethereal provisioning,
+    // trial activation against postalsys.com, IMAP connect, message read-back).
+    timeout: 300000,
+    reporter: process.env.CI ? [['list'], ['html', { open: 'never' }]] : 'list',
+    use: {
+        baseURL: BASE_URL,
+        trace: 'on-first-retry'
+    },
+    projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+    webServer: {
+        // Flush the isolated e2e Redis DB so every run starts from a clean, unconfigured instance.
+        command: 'node test/e2e/flush-redis.js && node server.js',
+        url: BASE_URL,
+        timeout: 120000,
+        // Always boot fresh: the suite asserts fresh-instance behaviour (enabling auth, activating
+        // the trial), so a reused server with those already set would break it. The dedicated
+        // db 14 makes the pre-boot flush safe.
+        reuseExistingServer: false,
+        stdout: 'pipe',
+        stderr: 'pipe',
+        env: Object.assign({}, process.env, { NODE_ENV: 'e2e' })
+    }
+});