emailengine-app 2.70.0 → 2.72.0

package/.github/workflows/codeql.yml

@@ -22,6 +22,9 @@ on:
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
+    # Skip release-please's auto-generated release PR: it only changes CHANGELOG/version,
+    # so there is no new code to scan. (Matches the guard in test.yml.)
+    if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
     # Runner size impacts CodeQL analysis time. To learn more, please see:
     #   - https://gh.io/recommended-hardware-resources-for-running-codeql
     #   - https://gh.io/supported-runners-and-hardware-resources

package/.github/workflows/e2e.yml

@@ -0,0 +1,56 @@
+name: E2E Tests
+
+# Browser-driven happy-path end-to-end suite (test/e2e), kept separate from the unit and
+# integration tiers so it is easy to re-run on its own and has its own requirements: it drives a
+# real browser (Playwright/Chromium) and reaches external services (Ethereal for the test mailbox,
+# postalsys.com for the 14-day trial). The trial rate limit is bypassed for the e2e serviceUrl
+# (https://e2e.emailengine.app/) - see the postalsys-web trial allowlist.
+
+on:
+    workflow_dispatch:
+    push:
+        branches: [master]
+
+permissions:
+    contents: read
+
+concurrency:
+    group: e2e-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    e2e:
+        name: E2E (Playwright)
+        timeout-minutes: 20
+        runs-on: ubuntu-24.04
+        services:
+            redis:
+                image: redis
+                options: >-
+                    --health-cmd "redis-cli ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 6379:6379
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - run: npm install
+            - name: Install Playwright browser
+              run: npx playwright install --with-deps chromium
+            - name: Run e2e tests
+              run: npm run test:e2e
+              env:
+                  NODE_ENV: e2e
+            - name: Upload Playwright report
+              uses: actions/upload-artifact@v4
+              if: ${{ !cancelled() }}
+              with:
+                  name: playwright-report
+                  path: playwright-report/
+                  retention-days: 7

package/.github/workflows/test.yml

@@ -12,8 +12,13 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
+    # Skip CI for release-please's auto-generated release PR / branch: it only bumps the
+    # version + CHANGELOG (no code change), so re-running the suite is wasted work. This is
+    # gated per job rather than with [skip ci] in the release commit, because that text would
+    # ride the squash-merge into master and skip the Manage Release run, breaking releases.
     license_check:
         name: License Compliance Check
+        if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
         runs-on: ubuntu-24.04
         # Service containers to run with `container-job`
         steps:
@@ -23,20 +28,83 @@ jobs:
               uses: actions/setup-node@v6
               with:
                   node-version: 24
+                  cache: npm
             - run: npm install
 
             - name: Run License Checks
               run: |
                   npm run licenses
 
-    test:
-        name: Test Suite
-        timeout-minutes: 15  # Increased timeout for Gmail API tests
-        strategy:
-            matrix:
-                node: [24.x]
-                os: [ubuntu-24.04]
-        runs-on: ${{ matrix.os }}
+    lint:
+        name: Lint
+        if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
+        runs-on: ubuntu-24.04
+        timeout-minutes: 10
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - run: npm install
+            - name: Run ESLint
+              run: |
+                  npm run lint
+
+    unit:
+        name: Unit Tests
+        if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
+        timeout-minutes: 10
+        runs-on: ubuntu-24.04
+        # Service containers to run with `container-job`
+        services:
+            # Label used to access the service container
+            redis:
+                # Docker Hub image
+                image: redis
+                # Set health checks to wait until redis has started
+                options: >-
+                    --health-cmd "redis-cli ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 6379:6379
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - name: Setup Redis CLI
+              uses: shogo82148/actions-setup-redis@v1
+              with:
+                  redis-version: '7.x'
+                  auto-start: 'false'
+            - run: npm install
+            - name: Run unit tests
+              run: |
+                  npm run test:unit
+              env:
+                  NODE_ENV: test
+                  # account-revoke-on-delete-test.js runs against live Gmail when
+                  # these are present and skips itself when they are not
+                  GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}
+                  GMAIL_API_CLIENT_ID: ${{ secrets.TEST_GMAIL_API_CLIENT_ID }}
+                  GMAIL_API_CLIENT_SECRET: ${{ secrets.TEST_GMAIL_API_CLIENT_SECRET }}
+                  GMAIL_API_SERVICE_EMAIL: ${{ secrets.TEST_GMAIL_API_SERVICE_EMAIL }}
+                  GMAIL_API_SERVICE_CLIENT: ${{ secrets.TEST_GMAIL_API_SERVICE_CLIENT }}
+                  GMAIL_API_SERVICE_KEY: ${{ secrets.TEST_GMAIL_API_SERVICE_KEY }}
+                  GMAIL_API_ACCOUNT_EMAIL_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_EMAIL_1 }}
+                  GMAIL_API_ACCOUNT_REFRESH_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_REFRESH_1 }}
+
+    integration:
+        name: Integration Tests
+        if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
+        timeout-minutes: 15 # Increased timeout for Gmail API tests
+        runs-on: ubuntu-24.04
         # Service containers to run with `container-job`
         services:
             # Label used to access the service container
@@ -53,19 +121,20 @@ jobs:
                     - 6379:6379
         steps:
             - uses: actions/checkout@v6
-            - name: Use Node.js ${{ matrix.node }}
+            - name: Use Node.js 24
               uses: actions/setup-node@v6
               with:
-                  node-version: ${{ matrix.node }}
+                  node-version: 24
+                  cache: npm
             - name: Setup Redis CLI
               uses: shogo82148/actions-setup-redis@v1
               with:
                   redis-version: '7.x'
                   auto-start: 'false'
             - run: npm install
-            - name: Run tests
+            - name: Run integration tests
               run: |
-                  npm test
+                  npm run test:integration
               env:
                   NODE_ENV: test
                   GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}

package/.ncurc.js

@@ -1,30 +1,30 @@
 module.exports = {
     upgrade: true,
+    // Packages capped inside their current major: the next major is ESM-only (this codebase is
+    // CommonJS and is bundled into a binary with pkg) or it breaks a peer dependency. Using 'minor'
+    // instead of a blanket reject so these still receive security/patch updates within the safe
+    // major instead of being frozen at one exact version. Verified against Node 20 (Docker) 2026-06-17.
+    target: name => (['joi', 'nanoid', 'ical.js', 'gettext-parser', 'xgettext-template', 'chai', 'undici', 'marked'].includes(name) ? 'minor' : 'latest'),
+    //   joi               - hapi-swagger (repo archived 2026-02-04) peer-requires joi 17.x; permanent ceiling
+    //   nanoid            - 4.x dropped the CommonJS require export (ESM-only)
+    //   ical.js           - 2.x is ESM-only
+    //   gettext-parser    - 8.x is ESM-only
+    //   xgettext-template - 6.x is ESM-only (translation build tool)
+    //   chai              - 5.x is ESM-only (only used by the vendored imap-core tests)
+    //   undici            - 8.x requires Node >=22.19 and crashes at require() on Node 20; EmailEngine supports Node 20+
+    //   marked            - 16.x dropped the CommonJS build (ESM-only, needs require(esm)/Node >=20.19); 15.x is the
+    //                       last require()-compatible line. 15.0.12 verified on Node 20-24 and in a yao/pkg node24 build.
     reject: [
-        // Block package upgrades that moved to ESM
-        'nanoid',
-        'gettext-parser',
-        'xgettext-template',
-        'chai',
-        'js-beautify',
-        'ical.js',
+        // 8.16+ pulls apache-arrow (ESM) into the pkg bundle; 9.x drops it but is a major bump for
+        // the deprecated, default-off Document Store. Even the 8.19 minor is unsafe to bundle.
         '@elastic/elasticsearch',
 
-        'pino-pretty',
-
-        // no support for Node 16
-        'marked',
-
-        // some kind of CVE in later versions. Only needed for license reference, so the actual version does not matter anyway
+        // v4.x adds vulnerable jquery + bootstrap runtime dependencies; v3.3.7 has none. Used only
+        // for the generated software-license listing, never executed at runtime.
         'startbootstrap-sb-admin-2',
 
-        // Keep joi at version 17.x for hapi-swagger compatibility
-        'joi',
-
         // @asamuzakjp/css-color >=4.1.2 pulls in @csstools/* v4 which are pure ESM and break pkg bundling
-        '@asamuzakjp/css-color',
-
-        // undici >=8.0.0 requires Node >=22.19.0; pin to last Node 20-compatible release
-        'undici'
+        // (transitive via @postalsys/email-text-tools -> jsdom -> cssstyle; also pinned in package.json "overrides").
+        '@asamuzakjp/css-color'
     ]
 };

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [2.72.0](https://github.com/postalsys/emailengine/compare/v2.71.0...v2.72.0) (2026-06-18)
+
+
+### Features
+
+* adopt pre-existing Gmail Pub/Sub resources with ownership tracking ([fe43fad](https://github.com/postalsys/emailengine/commit/fe43fadad624025a56139382428256eb810f9e08))
+
+## [2.71.0](https://github.com/postalsys/emailengine/compare/v2.70.0...v2.71.0) (2026-06-15)
+
+
+### Features
+
+* add anonymized feature beacon to license validation ([954d92a](https://github.com/postalsys/emailengine/commit/954d92a9d8e7c5fff6f1b8c711dfd228f556387a))
+* disable deprecated Document Store by default behind a feature gate ([8790f75](https://github.com/postalsys/emailengine/commit/8790f7538ba2ee8d6ec73d1f3bbc221c2f54f0fe))
+
+
+### Bug Fixes
+
+* defer export worker job consumption until startup recovery completes ([a21b0a5](https://github.com/postalsys/emailengine/commit/a21b0a50fdb98d13a2227a020069129d23fe225b))
+* fail export when a folder cannot be indexed ([36c9c9d](https://github.com/postalsys/emailengine/commit/36c9c9d536540cd5a61b16b69af5e8e84c194ee6))
+* filter transient fetch failures from Sentry and retry DNS errors ([883b9b4](https://github.com/postalsys/emailengine/commit/883b9b487e3580c5b1240d28cd519d190c402b47))
+* retry transient errors in API-account batch export path ([6cba7c7](https://github.com/postalsys/emailengine/commit/6cba7c72658a037f68867cf2464f7d3420a79507))
+* translate OAuth scope error page across 6 languages ([4ae1919](https://github.com/postalsys/emailengine/commit/4ae19199fe628f8a4f7177d53db1f05167077e3e))
+* upgrade joi to 17.13.4 and @postalsys/certs to 1.0.15 (GHSA-q7cg-457f-vx79) ([6ec77e5](https://github.com/postalsys/emailengine/commit/6ec77e50b288b14550cd09b724f5aab59d17ced4))
+
 ## [2.70.0](https://github.com/postalsys/emailengine/compare/v2.69.0...v2.70.0) (2026-06-11)
 
 

package/Gruntfile.js

@@ -5,22 +5,11 @@ const config = require('@zone-eu/wild-config');
 module.exports = function (grunt) {
     // Project configuration.
     grunt.initConfig({
-        wait: {
-            server: {
-                options: {
-                    delay: 20 * 1000 // Increased from 12s to 20s for Gmail API operations
-                }
-            }
-        },
-
         shell: {
             eslint: {
                 // Globs are quoted so eslint expands them itself - unquoted, sh (no globstar)
                 // expands lib/**/*.js to depth-2 files only and skips most of lib/
-                command: "npx eslint 'lib/**/*.js' 'workers/**/*.js' server.js Gruntfile.js",
-                options: {
-                    async: false
-                }
+                command: "npx eslint 'lib/**/*.js' 'workers/**/*.js' server.js Gruntfile.js"
             },
             server: {
                 command: 'node server.js',
@@ -29,16 +18,22 @@ module.exports = function (grunt) {
                 }
             },
             flush: {
-                command: `redis-cli -u "${config.dbs.redis}" flushdb`,
-                options: {
-                    async: false
-                }
+                command: `redis-cli -u "${config.dbs.redis}" flushdb`
             },
-            test: {
-                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/*.js', // Added 3-minute timeout for tests
-                options: {
-                    async: false
-                }
+            waitServer: {
+                // Polls /health until the server reports ready (all IMAP workers up,
+                // Redis responding) instead of sleeping for a fixed delay
+                command: 'node test/helpers/wait-for-server.js'
+            },
+            testUnit: {
+                // Self-contained tests - need Redis but not the live server.
+                // Run with default --test concurrency; the suite is verified to pass in parallel.
+                // The *-test.js pattern keeps helper modules out of the test runner
+                command: 'node --test --test-timeout=180000 test/*-test.js'
+            },
+            testIntegration: {
+                // Tests that run against the live server started by shell:server
+                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/integration/*-test.js'
             },
             options: {
                 stdout: data => console.log(data.toString().trim()),
@@ -50,10 +45,11 @@ module.exports = function (grunt) {
 
     // Load the plugin(s)
     grunt.loadNpmTasks('grunt-shell-spawn');
-    grunt.loadNpmTasks('grunt-wait');
 
     // Tasks
-    grunt.registerTask('test', ['shell:flush', 'shell:server', 'wait:server', 'shell:test', 'shell:server:kill']);
+    grunt.registerTask('test-unit', ['shell:flush', 'shell:testUnit']);
+    grunt.registerTask('test-integration', ['shell:flush', 'shell:server', 'shell:waitServer', 'shell:testIntegration', 'shell:server:kill']);
+    grunt.registerTask('test', ['test-unit', 'test-integration']);
 
     grunt.registerTask('default', ['shell:eslint', 'test']);
 };

package/bin/emailengine.js

@@ -105,7 +105,14 @@ const GLOBAL_OPTIONS = [
     { name: '--smtp.host', description: 'SMTP server bind address', type: 'string', default: '127.0.0.1', group: 'SMTP server' },
     { name: '--smtp.port', description: 'SMTP server port', type: 'number', default: 2525, group: 'SMTP server' },
     { name: '--smtp.proxy', description: 'Enable HAProxy PROXY protocol', type: 'boolean', default: false, group: 'SMTP server' },
-    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' }
+    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' },
+    {
+        name: '--documentStore.enabled',
+        description: 'Enable the deprecated Document Store (ElasticSearch) feature',
+        type: 'boolean',
+        default: false,
+        group: 'Document Store (deprecated)'
+    }
 ];
 
 // Help formatting functions

package/config/default.toml

@@ -31,6 +31,11 @@ secret = ""        # client password, if not set allows any password
 proxy = false      # Set to true if using HAProxy with send-proxy option
 #maxMessageSize = "25M" # maximum message size accepted by SMTP server (default: 25MB)
 
+[documentStore]
+# Deprecated Document Store (ElasticSearch) feature. Disabled by default; the worker and
+# all document-store API/UI endpoints are only available when this is enabled.
+enabled = false
+
 [dbs]
 # redis connection
 redis = "redis://127.0.0.1:6379/8"

package/config/e2e.toml

@@ -0,0 +1,35 @@
+# End-to-end overrides, merged over default.toml by @zone-eu/wild-config when NODE_ENV=e2e.
+# Boots a fresh EmailEngine for the Playwright happy-path suite (test/e2e): enable auth,
+# activate a 14-day trial (hits postalsys.com), register an Ethereal account, send + read back.
+#
+# Uses an isolated Redis DB so the suite can flush it for a clean fresh-instance run without
+# touching dev (db 9), test (db 13), or default (db 8) data. No preparedToken / preparedPassword /
+# preparedLicense here on purpose - the suite enables auth and activates the trial through the UI.
+
+# JSON formatted settings, seeded on startup.
+# serviceUrl is the recognizable e2e URL that postalsys-web allowlists to skip the trial rate
+# limit. It is only used as the trial request's `url`; the browser and API talk to 127.0.0.1:7099.
+settings = '''
+{
+  "serviceUrl": "https://e2e.emailengine.app/",
+  "ignoreMailCertErrors": true
+}
+'''
+
+[service]
+# Fixed encryption secret for the throwaway e2e instance (Redis is flushed on every run).
+secret = "e2e encryption secret"
+
+[workers]
+imap = 1
+
+[log]
+level = "warn"
+
+[dbs]
+# Isolated Redis DB for e2e; flushed before each run by test/e2e/flush-redis.js
+redis = "redis://127.0.0.1:6379/14"
+
+[api]
+host = "127.0.0.1"
+port = 7099

package/config/test.toml

@@ -43,3 +43,8 @@ port = 7077
 
 [webhooksServer]
 port = 7078
+
+# Keep the deprecated Document Store feature available during tests so the existing
+# document-store API/UI endpoints stay covered. Disabled by default in production.
+[documentStore]
+enabled = true

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2026-06-10T14:45:48.000000",
+    "creationTime": "2026-06-18T14:46:19.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/getswagger.sh

@@ -4,6 +4,10 @@ set -e
 
 export EENGINE_PORT=5678
 
+# Keep the deprecated Document Store endpoints in the generated spec; they are only
+# registered when the feature is enabled.
+export EENGINE_DOCUMENT_STORE_ENABLED=true
+
 # refuse to run if something is already listening on the port, otherwise the
 # polling loop below would silently fetch swagger.json from a stale instance
 if (exec 3<>"/dev/tcp/127.0.0.1/${EENGINE_PORT}") 2>/dev/null; then

package/lib/account.js

@@ -20,6 +20,7 @@ const { deepStrictEqual, strictEqual } = require('assert');
 const { encrypt, decrypt } = require('./encrypt');
 const { oauth2Apps, LEGACY_KEYS, isApiBasedApp } = require('./oauth2-apps');
 const settings = require('./settings');
+const { isDocumentStoreEnabled, documentStoreFeatureEnabled } = require('./document-store');
 const redisScanDelete = require('./redis-scan-delete');
 const { customAlphabet } = require('nanoid');
 const Lock = require('ioredfour');
@@ -1048,28 +1049,33 @@ class Account {
             };
         }
 
-        try {
-            let queueKeep = (await settings.get('queueKeep')) || true;
-            let serviceUrl = (await settings.get('serviceUrl')) || null;
+        // Only notify the documents queue when the deprecated Document Store feature is enabled.
+        // When it is off the documents worker is not running, so an enqueued job would never be
+        // consumed and would pile up in Redis.
+        if (documentStoreFeatureEnabled) {
+            try {
+                let queueKeep = (await settings.get('queueKeep')) || true;
+                let serviceUrl = (await settings.get('serviceUrl')) || null;
 
-            let payload = {
-                serviceUrl,
-                account: this.account,
-                date: new Date().toISOString(),
-                event: ACCOUNT_DELETED_NOTIFY
-            };
+                let payload = {
+                    serviceUrl,
+                    account: this.account,
+                    date: new Date().toISOString(),
+                    event: ACCOUNT_DELETED_NOTIFY
+                };
 
-            await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
-                removeOnComplete: queueKeep,
-                removeOnFail: queueKeep,
-                attempts: 10,
-                backoff: {
-                    type: 'exponential',
-                    delay: 5000
-                }
-            });
-        } catch (err) {
-            this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+                await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
+                    removeOnComplete: queueKeep,
+                    removeOnFail: queueKeep,
+                    attempts: 10,
+                    backoff: {
+                        type: 'exponential',
+                        delay: 5000
+                    }
+                });
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+            }
         }
 
         await this.call({
@@ -1458,7 +1464,7 @@ class Account {
     }
 
     async getText(text, options) {
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1516,7 +1522,7 @@ class Account {
             options.preProcessHtml = true;
         }
 
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1700,7 +1706,7 @@ class Account {
     }
 
     async listMessages(query) {
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1836,7 +1842,7 @@ class Account {
 
     async searchMessages(query, searchOpts) {
         searchOpts = searchOpts || {};
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             if (!searchOpts.unified) {
                 await this.loadAccountData(this.account, false);
             }
@@ -2401,7 +2407,7 @@ class Account {
                 // scan and delete keys
                 await redisScanDelete(this.redis, this.logger, `${REDIS_PREFIX}iam:${this.account}:*`);
 
-                if (await settings.get('documentStoreEnabled')) {
+                if (await isDocumentStoreEnabled()) {
                     // Flush ElasticSearch index for this account
                     const { index, client } = this.esClient;
                     if (!client) {