emailengine-app 2.69.0 → 2.71.0

package/.github/workflows/deploy.yml

@@ -41,7 +41,7 @@ jobs:
               id: deploy
               run: |
                   echo $GITHUB_SHA > commit.txt
-                  ./update-info.sh
+                  EE_COMMIT_HASH=$GITHUB_SHA ./update-info.sh
                   npm install --omit=dev
                   tar czf /tmp/${SERVICE_NAME}.tar.gz --exclude .git .
                   scp -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" /tmp/${SERVICE_NAME}.tar.gz deploy@${TARGET_HOST_02}:
@@ -55,7 +55,7 @@ jobs:
               id: deploy-demo
               run: |
                   echo $GITHUB_SHA > commit.txt
-                  ./update-info.sh
+                  EE_COMMIT_HASH=$GITHUB_SHA ./update-info.sh
                   npm install --omit=dev
                   tar czf /tmp/${SERVICE_NAME}.tar.gz --exclude .git .
                   scp -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" /tmp/${SERVICE_NAME}.tar.gz deploy@${TARGET_HOST_02}:
@@ -104,8 +104,11 @@ jobs:
                   images: |
                       ${{ github.repository }}
                       ghcr.io/${{ github.repository }}
+                  # :latest is owned by the release workflow (release.yaml) and
+                  # always points at the newest release; per-push dev builds from
+                  # master are published as :edge instead
                   tags: |
-                      type=raw,value=latest,enable=true
+                      type=raw,value=edge,enable=true
 
             - name: Build and push
               uses: docker/build-push-action@v7

package/.github/workflows/release.yaml

@@ -103,9 +103,11 @@ jobs:
                   platforms: ${{ steps.buildx.outputs.platforms }}
                   push: true
                   tags: |
+                      ${{ github.repository }}:latest
                       ${{ github.repository }}:v${{needs.release_please.outputs.major}}.${{needs.release_please.outputs.minor}}.${{needs.release_please.outputs.patch}}
                       ${{ github.repository }}:v${{needs.release_please.outputs.major}}.${{needs.release_please.outputs.minor}}
                       ${{ github.repository }}:v${{needs.release_please.outputs.major}}
+                      ghcr.io/${{ github.repository }}:latest
                       ghcr.io/${{ github.repository }}:v${{needs.release_please.outputs.major}}.${{needs.release_please.outputs.minor}}.${{needs.release_please.outputs.patch}}
                       ghcr.io/${{ github.repository }}:v${{needs.release_please.outputs.major}}.${{needs.release_please.outputs.minor}}
                       ghcr.io/${{ github.repository }}:v${{needs.release_please.outputs.major}}

package/.github/workflows/test.yml

@@ -23,20 +23,80 @@ jobs:
               uses: actions/setup-node@v6
               with:
                   node-version: 24
+                  cache: npm
             - run: npm install
 
             - name: Run License Checks
               run: |
                   npm run licenses
 
-    test:
-        name: Test Suite
-        timeout-minutes: 15  # Increased timeout for Gmail API tests
-        strategy:
-            matrix:
-                node: [24.x]
-                os: [ubuntu-24.04]
-        runs-on: ${{ matrix.os }}
+    lint:
+        name: Lint
+        runs-on: ubuntu-24.04
+        timeout-minutes: 10
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - run: npm install
+            - name: Run ESLint
+              run: |
+                  npm run lint
+
+    unit:
+        name: Unit Tests
+        timeout-minutes: 10
+        runs-on: ubuntu-24.04
+        # Service containers to run with `container-job`
+        services:
+            # Label used to access the service container
+            redis:
+                # Docker Hub image
+                image: redis
+                # Set health checks to wait until redis has started
+                options: >-
+                    --health-cmd "redis-cli ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 6379:6379
+        steps:
+            - uses: actions/checkout@v6
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  cache: npm
+            - name: Setup Redis CLI
+              uses: shogo82148/actions-setup-redis@v1
+              with:
+                  redis-version: '7.x'
+                  auto-start: 'false'
+            - run: npm install
+            - name: Run unit tests
+              run: |
+                  npm run test:unit
+              env:
+                  NODE_ENV: test
+                  # account-revoke-on-delete-test.js runs against live Gmail when
+                  # these are present and skips itself when they are not
+                  GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}
+                  GMAIL_API_CLIENT_ID: ${{ secrets.TEST_GMAIL_API_CLIENT_ID }}
+                  GMAIL_API_CLIENT_SECRET: ${{ secrets.TEST_GMAIL_API_CLIENT_SECRET }}
+                  GMAIL_API_SERVICE_EMAIL: ${{ secrets.TEST_GMAIL_API_SERVICE_EMAIL }}
+                  GMAIL_API_SERVICE_CLIENT: ${{ secrets.TEST_GMAIL_API_SERVICE_CLIENT }}
+                  GMAIL_API_SERVICE_KEY: ${{ secrets.TEST_GMAIL_API_SERVICE_KEY }}
+                  GMAIL_API_ACCOUNT_EMAIL_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_EMAIL_1 }}
+                  GMAIL_API_ACCOUNT_REFRESH_1: ${{ secrets.TEST_GMAIL_API_ACCOUNT_REFRESH_1 }}
+
+    integration:
+        name: Integration Tests
+        timeout-minutes: 15 # Increased timeout for Gmail API tests
+        runs-on: ubuntu-24.04
         # Service containers to run with `container-job`
         services:
             # Label used to access the service container
@@ -53,19 +113,20 @@ jobs:
                     - 6379:6379
         steps:
             - uses: actions/checkout@v6
-            - name: Use Node.js ${{ matrix.node }}
+            - name: Use Node.js 24
               uses: actions/setup-node@v6
               with:
-                  node-version: ${{ matrix.node }}
+                  node-version: 24
+                  cache: npm
             - name: Setup Redis CLI
               uses: shogo82148/actions-setup-redis@v1
               with:
                   redis-version: '7.x'
                   auto-start: 'false'
             - run: npm install
-            - name: Run tests
+            - name: Run integration tests
               run: |
-                  npm test
+                  npm run test:integration
               env:
                   NODE_ENV: test
                   GMAIL_API_PROJECT_ID: ${{ secrets.TEST_GMAIL_API_PROJECT_ID }}

package/.ncurc.js

@@ -1,5 +1,8 @@
 module.exports = {
     upgrade: true,
+    // Keep joi within the 17.x major (hapi-swagger's peer dependency requires joi 17.x).
+    // Using a target function instead of a blanket reject so joi still receives 17.x security patches.
+    target: name => (name === 'joi' ? 'minor' : 'latest'),
     reject: [
         // Block package upgrades that moved to ESM
         'nanoid',
@@ -18,9 +21,6 @@ module.exports = {
         // some kind of CVE in later versions. Only needed for license reference, so the actual version does not matter anyway
         'startbootstrap-sb-admin-2',
 
-        // Keep joi at version 17.x for hapi-swagger compatibility
-        'joi',
-
         // @asamuzakjp/css-color >=4.1.2 pulls in @csstools/* v4 which are pure ESM and break pkg bundling
         '@asamuzakjp/css-color',
 

package/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## [2.71.0](https://github.com/postalsys/emailengine/compare/v2.70.0...v2.71.0) (2026-06-15)
+
+
+### Features
+
+* add anonymized feature beacon to license validation ([954d92a](https://github.com/postalsys/emailengine/commit/954d92a9d8e7c5fff6f1b8c711dfd228f556387a))
+* disable deprecated Document Store by default behind a feature gate ([8790f75](https://github.com/postalsys/emailengine/commit/8790f7538ba2ee8d6ec73d1f3bbc221c2f54f0fe))
+
+
+### Bug Fixes
+
+* defer export worker job consumption until startup recovery completes ([a21b0a5](https://github.com/postalsys/emailengine/commit/a21b0a50fdb98d13a2227a020069129d23fe225b))
+* fail export when a folder cannot be indexed ([36c9c9d](https://github.com/postalsys/emailengine/commit/36c9c9d536540cd5a61b16b69af5e8e84c194ee6))
+* filter transient fetch failures from Sentry and retry DNS errors ([883b9b4](https://github.com/postalsys/emailengine/commit/883b9b487e3580c5b1240d28cd519d190c402b47))
+* retry transient errors in API-account batch export path ([6cba7c7](https://github.com/postalsys/emailengine/commit/6cba7c72658a037f68867cf2464f7d3420a79507))
+* translate OAuth scope error page across 6 languages ([4ae1919](https://github.com/postalsys/emailengine/commit/4ae19199fe628f8a4f7177d53db1f05167077e3e))
+* upgrade joi to 17.13.4 and @postalsys/certs to 1.0.15 (GHSA-q7cg-457f-vx79) ([6ec77e5](https://github.com/postalsys/emailengine/commit/6ec77e50b288b14550cd09b724f5aab59d17ced4))
+
+## [2.70.0](https://github.com/postalsys/emailengine/compare/v2.69.0...v2.70.0) (2026-06-11)
+
+
+### Features
+
+* allow enabling Sentry error reporting from the admin UI ([a4076a4](https://github.com/postalsys/emailengine/commit/a4076a4fa6658ccc998f9bfe35dddf015cd12b09))
+* replace Bugsnag with self-hosted Sentry for error tracking ([62de831](https://github.com/postalsys/emailengine/commit/62de831a2911da9b649fe693547ba09d70e2e481))
+* tag Sentry error reports with instance id and license key ([2e9683f](https://github.com/postalsys/emailengine/commit/2e9683f27d54ff8fa7ba3f6f4453866fd135f173))
+
+
+### Bug Fixes
+
+* align API response schemas with actual responses and fix uncovered API bugs ([abacbf6](https://github.com/postalsys/emailengine/commit/abacbf66f048a77d00d180a831a97594380d8ed9))
+* align remaining API response schemas with actual responses ([06136ab](https://github.com/postalsys/emailengine/commit/06136abd20a97c45b25aac7f93653387eb55928a))
+* assign unassigned accounts after license activation ([5677977](https://github.com/postalsys/emailengine/commit/56779778eba27f2d1604ff06f17e5736a5ddee61))
+* do not crash at startup when a feature flag env variable is set to a falsy value ([98ad979](https://github.com/postalsys/emailengine/commit/98ad97988e752fad9e2ae11eff63f35ffcf84976))
+* report correct nextAttempt time for queued messages ([d5ebdfb](https://github.com/postalsys/emailengine/commit/d5ebdfb029b3ed331a8239ace7426f4f0704ae2e))
+* resolve code review findings in Gmail bulk ops, exports, webhooks, and schemas ([f0ddb46](https://github.com/postalsys/emailengine/commit/f0ddb4631fb924ddb93e0a99d0e20bc9f0cc8ee2))
+
 ## [2.69.0](https://github.com/postalsys/emailengine/compare/v2.68.1...v2.69.0) (2026-06-09)
 
 

package/Gruntfile.js

@@ -5,20 +5,11 @@ const config = require('@zone-eu/wild-config');
 module.exports = function (grunt) {
     // Project configuration.
     grunt.initConfig({
-        wait: {
-            server: {
-                options: {
-                    delay: 20 * 1000 // Increased from 12s to 20s for Gmail API operations
-                }
-            }
-        },
-
         shell: {
             eslint: {
-                command: 'npx eslint lib/**/*.js workers/**/*.js server.js Gruntfile.js',
-                options: {
-                    async: false
-                }
+                // Globs are quoted so eslint expands them itself - unquoted, sh (no globstar)
+                // expands lib/**/*.js to depth-2 files only and skips most of lib/
+                command: "npx eslint 'lib/**/*.js' 'workers/**/*.js' server.js Gruntfile.js"
             },
             server: {
                 command: 'node server.js',
@@ -27,16 +18,22 @@ module.exports = function (grunt) {
                 }
             },
             flush: {
-                command: `redis-cli -u "${config.dbs.redis}" flushdb`,
-                options: {
-                    async: false
-                }
+                command: `redis-cli -u "${config.dbs.redis}" flushdb`
             },
-            test: {
-                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/*.js', // Added 3-minute timeout for tests
-                options: {
-                    async: false
-                }
+            waitServer: {
+                // Polls /health until the server reports ready (all IMAP workers up,
+                // Redis responding) instead of sleeping for a fixed delay
+                command: 'node test/helpers/wait-for-server.js'
+            },
+            testUnit: {
+                // Self-contained tests - need Redis but not the live server.
+                // Run with default --test concurrency; the suite is verified to pass in parallel.
+                // The *-test.js pattern keeps helper modules out of the test runner
+                command: 'node --test --test-timeout=180000 test/*-test.js'
+            },
+            testIntegration: {
+                // Tests that run against the live server started by shell:server
+                command: 'node --test --test-concurrency=1 --test-timeout=180000 test/integration/*-test.js'
             },
             options: {
                 stdout: data => console.log(data.toString().trim()),
@@ -48,10 +45,11 @@ module.exports = function (grunt) {
 
     // Load the plugin(s)
     grunt.loadNpmTasks('grunt-shell-spawn');
-    grunt.loadNpmTasks('grunt-wait');
 
     // Tasks
-    grunt.registerTask('test', ['shell:flush', 'shell:server', 'wait:server', 'shell:test', 'shell:server:kill']);
+    grunt.registerTask('test-unit', ['shell:flush', 'shell:testUnit']);
+    grunt.registerTask('test-integration', ['shell:flush', 'shell:server', 'shell:waitServer', 'shell:testIntegration', 'shell:server:kill']);
+    grunt.registerTask('test', ['test-unit', 'test-integration']);
 
     grunt.registerTask('default', ['shell:eslint', 'test']);
 };

package/bin/emailengine.js

@@ -105,7 +105,14 @@ const GLOBAL_OPTIONS = [
     { name: '--smtp.host', description: 'SMTP server bind address', type: 'string', default: '127.0.0.1', group: 'SMTP server' },
     { name: '--smtp.port', description: 'SMTP server port', type: 'number', default: 2525, group: 'SMTP server' },
     { name: '--smtp.proxy', description: 'Enable HAProxy PROXY protocol', type: 'boolean', default: false, group: 'SMTP server' },
-    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' }
+    { name: '--smtp.maxMessageSize', description: 'Maximum email size', type: 'number/string', default: '25M', group: 'SMTP server' },
+    {
+        name: '--documentStore.enabled',
+        description: 'Enable the deprecated Document Store (ElasticSearch) feature',
+        type: 'boolean',
+        default: false,
+        group: 'Document Store (deprecated)'
+    }
 ];
 
 // Help formatting functions

package/config/default.toml

@@ -31,6 +31,11 @@ secret = ""        # client password, if not set allows any password
 proxy = false      # Set to true if using HAProxy with send-proxy option
 #maxMessageSize = "25M" # maximum message size accepted by SMTP server (default: 25MB)
 
+[documentStore]
+# Deprecated Document Store (ElasticSearch) feature. Disabled by default; the worker and
+# all document-store API/UI endpoints are only available when this is enabled.
+enabled = false
+
 [dbs]
 # redis connection
 redis = "redis://127.0.0.1:6379/8"

package/config/test.toml

@@ -43,3 +43,8 @@ port = 7077
 
 [webhooksServer]
 port = 7078
+
+# Keep the deprecated Document Store feature available during tests so the existing
+# document-store API/UI endpoints stay covered. Disabled by default in production.
+[documentStore]
+enabled = true

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2026-06-09T14:45:50.000000",
+    "creationTime": "2026-06-15T14:45:58.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/getswagger.sh

@@ -1,8 +1,48 @@
 #!/bin/bash
 
+set -e
+
 export EENGINE_PORT=5678
 
-npm start > /dev/null 2>&1 &
-sleep 5
-curl -s "http://127.0.0.1:${EENGINE_PORT}/swagger.json" > swagger.json
-pkill emailengine
+# Keep the deprecated Document Store endpoints in the generated spec; they are only
+# registered when the feature is enabled.
+export EENGINE_DOCUMENT_STORE_ENABLED=true
+
+# refuse to run if something is already listening on the port, otherwise the
+# polling loop below would silently fetch swagger.json from a stale instance
+if (exec 3<>"/dev/tcp/127.0.0.1/${EENGINE_PORT}") 2>/dev/null; then
+    echo "Port ${EENGINE_PORT} is already in use" >&2
+    exit 1
+fi
+
+node server.js > /dev/null 2>&1 &
+SERVER_PID=$!
+
+cleanup() {
+    kill "$SERVER_PID" 2>/dev/null || true
+    wait "$SERVER_PID" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# poll until the API is up instead of relying on a fixed sleep
+rm -f swagger.json.tmp
+for i in $(seq 1 60); do
+    if curl -fs --max-time 5 "http://127.0.0.1:${EENGINE_PORT}/swagger.json" -o swagger.json.tmp; then
+        break
+    fi
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        echo "Server exited before swagger.json could be fetched" >&2
+        exit 1
+    fi
+    sleep 1
+done
+
+if [ ! -s swagger.json.tmp ]; then
+    echo "Timed out waiting for swagger.json" >&2
+    exit 1
+fi
+
+# only replace swagger.json if the download parses as JSON
+node -e 'JSON.parse(require("fs").readFileSync("swagger.json.tmp", "utf-8"))'
+
+mv swagger.json.tmp swagger.json

package/gettext-extract.js

@@ -0,0 +1,163 @@
+'use strict';
+
+// Extracts translatable strings from JS sources into translations/messages.pot.
+// Run via `npm run gettext` after xgettext-template has generated the POT from the
+// Handlebars views - this script joins the strings found in JS files into that file.
+//
+// Replaces jsxgettext, which bundled acorn 5 and crashed on post-ES2018 syntax
+// (optional chaining, nullish coalescing, etc.). Files are discovered dynamically,
+// so new modules with gettext()/ngettext() calls are picked up automatically.
+
+const { parse } = require('acorn');
+const { full: walkFull } = require('acorn-walk');
+const { po } = require('gettext-parser');
+const fs = require('fs');
+const Path = require('path');
+
+const ROOT_DIR = __dirname;
+const POT_PATH = Path.join(ROOT_DIR, 'translations', 'messages.pot');
+
+// All server-side code that may contain gettext()/ngettext() calls
+const SCAN_DIRS = ['bin', 'lib', 'workers'];
+const SCAN_FILES = ['server.js'];
+
+function listJsFiles(dir) {
+    return fs
+        .readdirSync(dir, { recursive: true })
+        .filter(entryPath => entryPath.endsWith('.js'))
+        .map(entryPath => Path.join(dir, entryPath));
+}
+
+// The full sorted scan set (sorted to keep POT reference output deterministic)
+function listScanFiles() {
+    let files = SCAN_FILES.map(file => Path.join(ROOT_DIR, file));
+    for (let dir of SCAN_DIRS) {
+        files = files.concat(listJsFiles(Path.join(ROOT_DIR, dir)));
+    }
+    return files.sort();
+}
+
+function parseFile(filePath) {
+    const source = fs.readFileSync(filePath, 'utf-8');
+    try {
+        return parse(source, { ecmaVersion: 'latest', sourceType: 'script', locations: true, allowHashBang: true });
+    } catch (err) {
+        err.message = `Failed to parse ${filePath}: ${err.message}`;
+        throw err;
+    }
+}
+
+// Resolves a call argument into a string value. Handles string literals and
+// concatenation of string literals ('foo' + 'bar'). Returns false for anything
+// dynamic (identifiers, template literals with expressions, etc.) - such calls
+// forward runtime values and do not define new translatable strings.
+function resolveString(node) {
+    switch (node.type) {
+        case 'Literal':
+            return typeof node.value === 'string' ? node.value : false;
+        case 'TemplateLiteral':
+            return node.expressions.length === 0 ? node.quasis[0].value.cooked : false;
+        case 'BinaryExpression': {
+            if (node.operator !== '+') {
+                return false;
+            }
+            let left = resolveString(node.left);
+            let right = resolveString(node.right);
+            return left !== false && right !== false ? left + right : false;
+        }
+        default:
+            return false;
+    }
+}
+
+function calleeName(callee) {
+    if (callee.type === 'Identifier') {
+        return callee.name;
+    }
+    if (callee.type === 'MemberExpression' && !callee.computed && callee.property.type === 'Identifier') {
+        return callee.property.name;
+    }
+    return false;
+}
+
+function extractFromFile(filePath) {
+    const ast = parseFile(filePath);
+
+    let entries = [];
+    let reference = node => `${Path.relative(ROOT_DIR, filePath)}:${node.loc.start.line}`;
+
+    walkFull(ast, node => {
+        if (node.type !== 'CallExpression') {
+            return;
+        }
+
+        let name = calleeName(node.callee);
+
+        // An empty msgid is skipped (like xgettext does): translations[''] is the POT header
+        // entry in gettext-parser, so an empty key would corrupt the header block
+        if (name === 'gettext' && node.arguments.length >= 1) {
+            let msgid = resolveString(node.arguments[0]);
+            if (msgid !== false && msgid !== '') {
+                entries.push({ msgid, reference: reference(node) });
+            }
+        }
+
+        if (name === 'ngettext' && node.arguments.length >= 2) {
+            let msgid = resolveString(node.arguments[0]);
+            let msgidPlural = resolveString(node.arguments[1]);
+            if (msgid !== false && msgid !== '' && msgidPlural !== false) {
+                entries.push({ msgid, msgidPlural, reference: reference(node) });
+            }
+        }
+    });
+
+    return entries;
+}
+
+function main() {
+    let files = listScanFiles();
+
+    let extracted = [];
+    for (let filePath of files) {
+        extracted = extracted.concat(extractFromFile(filePath));
+    }
+
+    let pot = po.parse(fs.readFileSync(POT_PATH));
+    let translations = (pot.translations[''] = pot.translations[''] || {});
+
+    // xgettext-template does not stamp a creation date, so set it here (jsxgettext used to)
+    pot.headers = pot.headers || {};
+    pot.headers['POT-Creation-Date'] = new Date()
+        .toISOString()
+        .replace(/T/, ' ')
+        .replace(/:\d+\.\d+Z$/, '+0000');
+
+    for (let { msgid, msgidPlural, reference } of extracted) {
+        let entry = translations[msgid];
+        if (!entry) {
+            entry = translations[msgid] = { msgid, msgstr: [''] };
+        }
+
+        if (msgidPlural && !entry.msgid_plural) {
+            entry.msgid_plural = msgidPlural;
+            entry.msgstr = ['', ''];
+        }
+
+        entry.comments = entry.comments || {};
+        let references = entry.comments.reference ? entry.comments.reference.split('\n') : [];
+        if (!references.includes(reference)) {
+            references.push(reference);
+        }
+        entry.comments.reference = references.join('\n');
+    }
+
+    fs.writeFileSync(POT_PATH, po.compile(pot));
+    console.log(`Extracted ${extracted.length} gettext strings from ${files.length} JS files into ${Path.relative(ROOT_DIR, POT_PATH)}`);
+}
+
+if (require.main === module) {
+    main();
+}
+
+// Exported for the gettext coverage test, which walks the same scan set with the same helpers
+module.exports = { listScanFiles, parseFile, calleeName, extractFromFile };