emailengine-app 2.70.0 → 2.71.0

package/.github/workflows/test.yml

@@ -23,20 +23,80 @@ jobs:
               uses: actions/setup-node@v6
               with:
                   node-version: 24
+                  cache: npm
             - run: npm install
 
             - name: Run License Checks
               run: |
                   npm run licenses
 
-    test:
-        name: Test Suite
-        timeout-minutes: 15  # Increased timeout for Gmail API tests
-        strategy:
-            matrix:
-                node: [24.x]
-                os: [ubuntu-24.04]
-        runs-on: ${{ matrix.os }}
+    lint:
+        name: Lint
+        runs-on: ubuntu-24.04
+        timeout-minutes: 10
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - run: npm install
+            - name: Run ESLint
+              run: |
+                  npm run lint
+
+    unit:
+        name: Unit Tests
+        timeout-minutes: 10
+        runs-on: ubuntu-24.04
+        # Service containers to run with `container-job`
+        services:
+            # Label used to access the service container
+            redis:
+                # Docker Hub image
+                image: redis
+                # Set health checks to wait until redis has started
+                options: >-
+                    --health-cmd "redis-cli ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 6379:6379
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - name: Setup Redis CLI
+              uses: shogo82148/actions-setup-redis@v1
+              with:
+                  redis-version: '7.x'
+                  auto-start: 'false'
+            - run: npm install
+            - name: Run unit tests
+              run: |
+                  npm run test:unit
+              env:
+                  NODE_ENV: test
+                  # account-revoke-on-delete-test.js runs against live Gmail when
+                  # these are present and skips itself when they are not
+                  GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}
+                  GMAIL_API_CLIENT_ID: ${{ secrets.TEST_GMAIL_API_CLIENT_ID }}
+                  GMAIL_API_CLIENT_SECRET: ${{ secrets.TEST_GMAIL_API_CLIENT_SECRET }}
+                  GMAIL_API_SERVICE_EMAIL: ${{ secrets.TEST_GMAIL_API_SERVICE_EMAIL }}
+                  GMAIL_API_SERVICE_CLIENT: ${{ secrets.TEST_GMAIL_API_SERVICE_CLIENT }}
+                  GMAIL_API_SERVICE_KEY: ${{ secrets.TEST_GMAIL_API_SERVICE_KEY }}
+                  GMAIL_API_ACCOUNT_EMAIL_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_EMAIL_1 }}
+                  GMAIL_API_ACCOUNT_REFRESH_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_REFRESH_1 }}
+
+    integration:
+        name: Integration Tests
+        timeout-minutes: 15 # Increased timeout for Gmail API tests
+        runs-on: ubuntu-24.04
         # Service containers to run with `container-job`
         services:
             # Label used to access the service container
@@ -53,19 +113,20 @@ jobs:
                     - 6379:6379
         steps:
             - uses: actions/checkout@v6
-            - name: Use Node.js ${{ matrix.node }}
+            - name: Use Node.js 24
               uses: actions/setup-node@v6
               with:
-                  node-version: ${{ matrix.node }}
+                  node-version: 24
+                  cache: npm
             - name: Setup Redis CLI
               uses: shogo82148/actions-setup-redis@v1
               with:
                   redis-version: '7.x'
                   auto-start: 'false'
             - run: npm install
-            - name: Run tests
+            - name: Run integration tests
               run: |
-                  npm test
+                  npm run test:integration
               env:
                   NODE_ENV: test
                   GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}

package/.ncurc.js

@@ -1,5 +1,8 @@
 module.exports = {
     upgrade: true,
+    // Keep joi within the 17.x major (hapi-swagger's peer dependency requires joi 17.x).
+    // Using a target function instead of a blanket reject so joi still receives 17.x security patches.
+    target: name => (name === 'joi' ? 'minor' : 'latest'),
     reject: [
         // Block package upgrades that moved to ESM
         'nanoid',
@@ -18,9 +21,6 @@ module.exports = {
         // some kind of CVE in later versions. Only needed for license reference, so the actual version does not matter anyway
         'startbootstrap-sb-admin-2',
 
-        // Keep joi at version 17.x for hapi-swagger compatibility
-        'joi',
-
         // @asamuzakjp/css-color >=4.1.2 pulls in @csstools/* v4 which are pure ESM and break pkg bundling
         '@asamuzakjp/css-color',
 

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [2.71.0](https://github.com/postalsys/emailengine/compare/v2.70.0...v2.71.0) (2026-06-15)
+
+
+### Features
+
+* add anonymized feature beacon to license validation ([954d92a](https://github.com/postalsys/emailengine/commit/954d92a9d8e7c5fff6f1b8c711dfd228f556387a))
+* disable deprecated Document Store by default behind a feature gate ([8790f75](https://github.com/postalsys/emailengine/commit/8790f7538ba2ee8d6ec73d1f3bbc221c2f54f0fe))
+
+
+### Bug Fixes
+
+* defer export worker job consumption until startup recovery completes ([a21b0a5](https://github.com/postalsys/emailengine/commit/a21b0a50fdb98d13a2227a020069129d23fe225b))
+* fail export when a folder cannot be indexed ([36c9c9d](https://github.com/postalsys/emailengine/commit/36c9c9d536540cd5a61b16b69af5e8e84c194ee6))
+* filter transient fetch failures from Sentry and retry DNS errors ([883b9b4](https://github.com/postalsys/emailengine/commit/883b9b487e3580c5b1240d28cd519d190c402b47))
+* retry transient errors in API-account batch export path ([6cba7c7](https://github.com/postalsys/emailengine/commit/6cba7c72658a037f68867cf2464f7d3420a79507))
+* translate OAuth scope error page across 6 languages ([4ae1919](https://github.com/postalsys/emailengine/commit/4ae19199fe628f8a4f7177d53db1f05167077e3e))
+* upgrade joi to 17.13.4 and @postalsys/certs to 1.0.15 (GHSA-q7cg-457f-vx79) ([6ec77e5](https://github.com/postalsys/emailengine/commit/6ec77e50b288b14550cd09b724f5aab59d17ced4))
+
 ## [2.70.0](https://github.com/postalsys/emailengine/compare/v2.69.0...v2.70.0) (2026-06-11)
 
 

package/Gruntfile.js

@@ -5,22 +5,11 @@ const config = require('@zone-eu/wild-config');
 module.exports = function (grunt) {
     // Project configuration.
     grunt.initConfig({
-        wait: {
-            server: {
-                options: {
-                    delay: 20 * 1000 // Increased from 12s to 20s for Gmail API operations
-                }
-            }
-        },
-
         shell: {
             eslint: {
                 // Globs are quoted so eslint expands them itself - unquoted, sh (no globstar)
                 // expands lib/**/*.js to depth-2 files only and skips most of lib/
-                command: "npx eslint 'lib/**/*.js' 'workers/**/*.js' server.js Gruntfile.js",
-                options: {
-                    async: false
-                }
+                command: "npx eslint 'lib/**/*.js' 'workers/**/*.js' server.js Gruntfile.js"
             },
             server: {
                 command: 'node server.js',
@@ -29,16 +18,22 @@ module.exports = function (grunt) {
                 }
             },
             flush: {
-                command: `redis-cli -u "${config.dbs.redis}" flushdb`,
-                options: {
-                    async: false
-                }
+                command: `redis-cli -u "${config.dbs.redis}" flushdb`
             },
-            test: {
-                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/*.js', // Added 3-minute timeout for tests
-                options: {
-                    async: false
-                }
+            waitServer: {
+                // Polls /health until the server reports ready (all IMAP workers up,
+                // Redis responding) instead of sleeping for a fixed delay
+                command: 'node test/helpers/wait-for-server.js'
+            },
+            testUnit: {
+                // Self-contained tests - need Redis but not the live server.
+                // Run with default --test concurrency; the suite is verified to pass in parallel.
+                // The *-test.js pattern keeps helper modules out of the test runner
+                command: 'node --test --test-timeout=180000 test/*-test.js'
+            },
+            testIntegration: {
+                // Tests that run against the live server started by shell:server
+                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/integration/*-test.js'
             },
             options: {
                 stdout: data => console.log(data.toString().trim()),
@@ -50,10 +45,11 @@ module.exports = function (grunt) {
 
     // Load the plugin(s)
     grunt.loadNpmTasks('grunt-shell-spawn');
-    grunt.loadNpmTasks('grunt-wait');
 
     // Tasks
-    grunt.registerTask('test', ['shell:flush', 'shell:server', 'wait:server', 'shell:test', 'shell:server:kill']);
+    grunt.registerTask('test-unit', ['shell:flush', 'shell:testUnit']);
+    grunt.registerTask('test-integration', ['shell:flush', 'shell:server', 'shell:waitServer', 'shell:testIntegration', 'shell:server:kill']);
+    grunt.registerTask('test', ['test-unit', 'test-integration']);
 
     grunt.registerTask('default', ['shell:eslint', 'test']);
 };

package/bin/emailengine.js

@@ -105,7 +105,14 @@ const GLOBAL_OPTIONS = [
     { name: '--smtp.host', description: 'SMTP server bind address', type: 'string', default: '127.0.0.1', group: 'SMTP server' },
     { name: '--smtp.port', description: 'SMTP server port', type: 'number', default: 2525, group: 'SMTP server' },
     { name: '--smtp.proxy', description: 'Enable HAProxy PROXY protocol', type: 'boolean', default: false, group: 'SMTP server' },
-    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' }
+    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' },
+    {
+        name: '--documentStore.enabled',
+        description: 'Enable the deprecated Document Store (ElasticSearch) feature',
+        type: 'boolean',
+        default: false,
+        group: 'Document Store (deprecated)'
+    }
 ];
 
 // Help formatting functions

package/config/default.toml

@@ -31,6 +31,11 @@ secret = ""        # client password, if not set allows any password
 proxy = false      # Set to true if using HAProxy with send-proxy option
 #maxMessageSize = "25M" # maximum message size accepted by SMTP server (default: 25MB)
 
+[documentStore]
+# Deprecated Document Store (ElasticSearch) feature. Disabled by default; the worker and
+# all document-store API/UI endpoints are only available when this is enabled.
+enabled = false
+
 [dbs]
 # redis connection
 redis = "redis://127.0.0.1:6379/8"

package/config/test.toml

@@ -43,3 +43,8 @@ port = 7077
 
 [webhooksServer]
 port = 7078
+
+# Keep the deprecated Document Store feature available during tests so the existing
+# document-store API/UI endpoints stay covered. Disabled by default in production.
+[documentStore]
+enabled = true

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2026-06-10T14:45:48.000000",
+    "creationTime": "2026-06-15T14:45:58.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/getswagger.sh

@@ -4,6 +4,10 @@ set -e
 
 export EENGINE_PORT=5678
 
+# Keep the deprecated Document Store endpoints in the generated spec; they are only
+# registered when the feature is enabled.
+export EENGINE_DOCUMENT_STORE_ENABLED=true
+
 # refuse to run if something is already listening on the port, otherwise the
 # polling loop below would silently fetch swagger.json from a stale instance
 if (exec 3<>"/dev/tcp/127.0.0.1/${EENGINE_PORT}") 2>/dev/null; then

package/lib/account.js

@@ -20,6 +20,7 @@ const { deepStrictEqual, strictEqual } = require('assert');
 const { encrypt, decrypt } = require('./encrypt');
 const { oauth2Apps, LEGACY_KEYS, isApiBasedApp } = require('./oauth2-apps');
 const settings = require('./settings');
+const { isDocumentStoreEnabled, documentStoreFeatureEnabled } = require('./document-store');
 const redisScanDelete = require('./redis-scan-delete');
 const { customAlphabet } = require('nanoid');
 const Lock = require('ioredfour');
@@ -1048,28 +1049,33 @@ class Account {
             };
         }
 
-        try {
-            let queueKeep = (await settings.get('queueKeep')) || true;
-            let serviceUrl = (await settings.get('serviceUrl')) || null;
+        // Only notify the documents queue when the deprecated Document Store feature is enabled.
+        // When it is off the documents worker is not running, so an enqueued job would never be
+        // consumed and would pile up in Redis.
+        if (documentStoreFeatureEnabled) {
+            try {
+                let queueKeep = (await settings.get('queueKeep')) || true;
+                let serviceUrl = (await settings.get('serviceUrl')) || null;
 
-            let payload = {
-                serviceUrl,
-                account: this.account,
-                date: new Date().toISOString(),
-                event: ACCOUNT_DELETED_NOTIFY
-            };
+                let payload = {
+                    serviceUrl,
+                    account: this.account,
+                    date: new Date().toISOString(),
+                    event: ACCOUNT_DELETED_NOTIFY
+                };
 
-            await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
-                removeOnComplete: queueKeep,
-                removeOnFail: queueKeep,
-                attempts: 10,
-                backoff: {
-                    type: 'exponential',
-                    delay: 5000
-                }
-            });
-        } catch (err) {
-            this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+                await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
+                    removeOnComplete: queueKeep,
+                    removeOnFail: queueKeep,
+                    attempts: 10,
+                    backoff: {
+                        type: 'exponential',
+                        delay: 5000
+                    }
+                });
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+            }
         }
 
         await this.call({
@@ -1458,7 +1464,7 @@ class Account {
     }
 
     async getText(text, options) {
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1516,7 +1522,7 @@ class Account {
             options.preProcessHtml = true;
         }
 
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1700,7 +1706,7 @@ class Account {
     }
 
     async listMessages(query) {
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1836,7 +1842,7 @@ class Account {
 
     async searchMessages(query, searchOpts) {
         searchOpts = searchOpts || {};
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             if (!searchOpts.unified) {
                 await this.loadAccountData(this.account, false);
             }
@@ -2401,7 +2407,7 @@ class Account {
                 // scan and delete keys
                 await redisScanDelete(this.redis, this.logger, `${REDIS_PREFIX}iam:${this.account}:*`);
 
-                if (await settings.get('documentStoreEnabled')) {
+                if (await isDocumentStoreEnabled()) {
                     // Flush ElasticSearch index for this account
                     const { index, client } = this.esClient;
                     if (!client) {