emailengine-app 2.63.4 → 2.65.0

package/.github/workflows/test.yml

@@ -80,3 +80,7 @@ jobs:
                   GMAIL_SENDONLY_CLIENT_SECRET: ${{ secrets.GMAIL_SENDONLY_CLIENT_SECRET }}
                   GMAIL_SENDONLY_ACCOUNT_EMAIL: ${{ secrets.GMAIL_SENDONLY_ACCOUNT_EMAIL }}
                   GMAIL_SENDONLY_ACCOUNT_REFRESH: ${{ secrets.GMAIL_SENDONLY_ACCOUNT_REFRESH }}
+                  OUTLOOK_SERVICE_CLIENT_ID: ${{ secrets.OUTLOOK_SERVICE_CLIENT_ID }}
+                  OUTLOOK_SERVICE_TENANT_ID: ${{ secrets.OUTLOOK_SERVICE_TENANT_ID }}
+                  OUTLOOK_SERVICE_CLIENT_SECRET: ${{ secrets.OUTLOOK_SERVICE_CLIENT_SECRET }}
+                  OUTLOOK_SERVICE_ACCOUNT_EMAIL: ${{ secrets.OUTLOOK_SERVICE_ACCOUNT_EMAIL }}

package/CHANGELOG.md

@@ -1,5 +1,75 @@
 # Changelog
 
+## [2.65.0](https://github.com/postalsys/emailengine/compare/v2.64.0...v2.65.0) (2026-03-23)
+
+
+### Features
+
+* add Outlook Service (client_credentials) provider for app-only Microsoft 365 access ([#587](https://github.com/postalsys/emailengine/issues/587)) ([5f906cd](https://github.com/postalsys/emailengine/commit/5f906cd540564afe2dc2024b4e17c2d5d9483ed2))
+
+
+### Bug Fixes
+
+* fix export TTL expiration, cancellation cleanup, and expose truncated field ([019c05c](https://github.com/postalsys/emailengine/commit/019c05c941842ae28668711e8f5c2e8a85d3b84b))
+* include failed jobs in /v1/outbox response ([b393676](https://github.com/postalsys/emailengine/commit/b393676f2f877046d27b06d518a03dd256d5b7d9))
+* switch pkg targets from node22 to node24 to fix Windows builds ([12522a7](https://github.com/postalsys/emailengine/commit/12522a7b92d72684af49cd70c1c6d4bcafe1d4af))
+
+## [2.64.0](https://github.com/postalsys/emailengine/compare/v2.63.4...v2.64.0) (2026-03-16)
+
+
+### Features
+
+* add configurable Gmail Pub/Sub subscription TTL setting ([ca33e7f](https://github.com/postalsys/emailengine/commit/ca33e7f9101b3375d7e0001cdbe1d7a41a9442d3))
+* add Gmail Subscriptions tab to OAuth config page ([3bd30bf](https://github.com/postalsys/emailengine/commit/3bd30bf0bbc26600eae40d42e95ef7976257e4f2))
+
+
+### Bug Fixes
+
+* add .catch() to fire-and-forget setMeta and track error fingerprints ([672a03e](https://github.com/postalsys/emailengine/commit/672a03e2524bcaa26c4d3c908fa828f4e636999b))
+* add lock to del() to prevent race with ensurePubsub, fix pubSubApp cleanup ([27f2bdd](https://github.com/postalsys/emailengine/commit/27f2bdd4a9e3d7e913e7f4bed96084a278bb6312))
+* always notify webhook workers when Pub/Sub app config changes ([e9f276a](https://github.com/postalsys/emailengine/commit/e9f276ab528ad384bcbf5c370a0d0cb22e084cc5))
+* auto-recover expired Gmail Pub/Sub subscriptions and expose status via API ([7961ceb](https://github.com/postalsys/emailengine/commit/7961ceb8ce81688e5c3bb435a6c33ed3882afd7b))
+* avoid redundant Redis call and concurrent backfill races in Pub/Sub setup ([05b02af](https://github.com/postalsys/emailengine/commit/05b02afc1c14de1b8fa4e272d26da2056280731a))
+* broaden Pub/Sub notification condition in oauth-routes.js ([cbe1380](https://github.com/postalsys/emailengine/commit/cbe13800ab9460c41f5dcdcd4d7ae50f8e5e75e3))
+* clean up Pub/Sub Redis keys on deletion, refresh stale instances, and localize UI labels ([3b35fa1](https://github.com/postalsys/emailengine/commit/3b35fa19a6a8024a60d2ed21e9fcd69fda349481))
+* clear stale pubSubFlag after restart and consolidate backfill push ([f67961f](https://github.com/postalsys/emailengine/commit/f67961fbc590aa586ef40b65c2189b5dab4d89c0))
+* correct pubSubApp property name in del() and add cleanup tests ([016ae0a](https://github.com/postalsys/emailengine/commit/016ae0a98bc52eeee2bd1ec4f4d0ed58e92c507d))
+* correct typo in Pub/Sub schema version log message ([2cc8e08](https://github.com/postalsys/emailengine/commit/2cc8e085463ed550e341b027e592b5351784182e))
+* eliminate redundant Redis ops in Pub/Sub pull loop and backfill ([c8ebe39](https://github.com/postalsys/emailengine/commit/c8ebe393744eee46c3d19b5be413badd8ab315d6))
+* fix lifecycle event races, lock TTL, and stale clearExisting in MS Graph subscriptions ([3f3cbac](https://github.com/postalsys/emailengine/commit/3f3cbac843c4c221644492855d7c1beb41cd1da0))
+* fix missing renewal retry, clock-skew gap, and incomplete pipeline error check ([8a8b5b6](https://github.com/postalsys/emailengine/commit/8a8b5b68feb3edd7c984707a85ddb6d146739020))
+* fix off-by-one retry cap and simplify MS Graph subscription code ([aa9afbc](https://github.com/postalsys/emailengine/commit/aa9afbc5aea20d1bf775e59dbfd82842c4f8b701))
+* fix Pub/Sub deletion race, add 429 handling, batch ACKs, and lock ensurePubsub ([744c354](https://github.com/postalsys/emailengine/commit/744c354a062f63eedcec8de1a324a5d13519b9a7))
+* fix retry boundary, lock races, and dropped lifecycle events in MS Graph subscriptions ([d62741d](https://github.com/postalsys/emailengine/commit/d62741d2a2aafc47d831482b71ff24fa4c9d1883))
+* fix silent 401/403 error suppression, floating promise, and recovery loop in Pub/Sub ([406211c](https://github.com/postalsys/emailengine/commit/406211cbd62f7d3f1b811fbec81767f319219c63))
+* fix stale subscription state blocking recovery and retry count persisting across reconnects ([60b77d5](https://github.com/postalsys/emailengine/commit/60b77d5d7574abf728c5e1f015c0cecddd5d9fdd))
+* fix subscription loss on subscriptionRemoved, lifecycle webhook timeout, and retry gaps ([62988ab](https://github.com/postalsys/emailengine/commit/62988ab3b690ed98583fa01557a9c4145427e3ae))
+* guard fire-and-forget setMeta calls and add Pub/Sub graceful shutdown ([45f7b64](https://github.com/postalsys/emailengine/commit/45f7b64d7692b220ff14ec0e1c0e9deea84731de))
+* guard releaseLock against undefined, add 429 handling to Pub/Sub deletion ([d67ebe1](https://github.com/postalsys/emailengine/commit/d67ebe138ca9663796eacc5a1c6b5b73e721e6c4))
+* handle TimeoutError from AbortSignal.timeout in Pub/Sub pull loop ([c0e3036](https://github.com/postalsys/emailengine/commit/c0e303641e4eb9f6cdca3c584861052b2bd91148))
+* harden MS Graph subscription lifecycle, locking, cleanup, and error recovery ([8210056](https://github.com/postalsys/emailengine/commit/8210056f7e7b00977fb5487323ff1b8f9f4b7a99))
+* harden Pub/Sub deletion, IAM policy, and worker timeout cleanup ([0be7c3e](https://github.com/postalsys/emailengine/commit/0be7c3eb04f3381306284e1e3790f7c3cc28cb5a))
+* harden Pub/Sub pull loop resilience and fix projectId typo ([8124328](https://github.com/postalsys/emailengine/commit/81243287f339b5fe40cddd0e1d127514d2e11f4a))
+* harden Pub/Sub pull loop, message ACK, list pairing, and shutdown cleanup ([9e9775f](https://github.com/postalsys/emailengine/commit/9e9775f6df97af42bdb8b799c2345a4564f8a77a))
+* harden Pub/Sub pull loop, worker coordination, and OAuth app lifecycle ([529b705](https://github.com/postalsys/emailengine/commit/529b7055dd4aeb0b1d80fabff3f139cf23f00252))
+* harden Pub/Sub shutdown, loop scheduling, and abort lifecycle ([71b3ab4](https://github.com/postalsys/emailengine/commit/71b3ab4625fa3304ae7a4413a3b0a64276948d7f))
+* harden Pub/Sub shutdown, transient error handling, and input validation ([cfc6e0b](https://github.com/postalsys/emailengine/commit/cfc6e0bb73902600b6eecc6bc955af85f06d9b0f))
+* localize error page strings using translation helper ([609ebb9](https://github.com/postalsys/emailengine/commit/609ebb94d1e662ac661f7efe22f6608d7823fb85))
+* move pubsub status from unauthenticated /health to GET /v1/pubsub/status ([f6597ad](https://github.com/postalsys/emailengine/commit/f6597addff4de1c23fbd88964702bc2425f14d0c))
+* prevent oscillating recovery loop for Pub/Sub apps missing googleProjectId ([8bdbc39](https://github.com/postalsys/emailengine/commit/8bdbc39c790b7097b02459b558328c18584585a2))
+* reduce Pub/Sub recovery log noise and respect backoff delay ([63cd78f](https://github.com/postalsys/emailengine/commit/63cd78fcfd0c12934e488f6936109653371f0899))
+* remove dead circuit breaker code from IMAP and webhooks workers ([da9295b](https://github.com/postalsys/emailengine/commit/da9295b951d20a5817116195689dd3c964b3d996))
+* remove Pub/Sub circuit breaker, fix 401/403 recovery handling ([b7017f3](https://github.com/postalsys/emailengine/commit/b7017f34478dbb3bcedfb9a1e098a8a45cb0f83d))
+* reorder OAuth app deletion to prevent pull-loop gap, add startLoop tests ([129f96c](https://github.com/postalsys/emailengine/commit/129f96c99ff9ff2552fa1c1962b1e0a652b8837e))
+* replace custom Redis locks with ioredfour in Outlook subscription code ([019eaa1](https://github.com/postalsys/emailengine/commit/019eaa1f26be504d127cb6c8a05dfe50d8a47b71))
+* resolve lint errors, fix TTL null guard, and extract Pub/Sub constants ([4949431](https://github.com/postalsys/emailengine/commit/4949431c8e05f06c5aafc3dfe93876ea8a2875b9))
+* resolve livelock, timeout, and state corruption in MS Graph subscription lifecycle ([745334e](https://github.com/postalsys/emailengine/commit/745334e8c55fcf4b0b3fc6adef2b37f6cf9aea61))
+* retry transient errors during Pub/Sub resource deletion and log dropped messages ([b9d9de5](https://github.com/postalsys/emailengine/commit/b9d9de56a50b37e0163ea4e9a954d65bc305004c))
+* show OAuth apps with failed Pub/Sub setup in subscriptions list ([14124cd](https://github.com/postalsys/emailengine/commit/14124cd29baf107f1f82dff48695bc51cae4fd97))
+* stop Pub/Sub instances on OAuth2 app deletion and harden lifecycle ([bc0ff75](https://github.com/postalsys/emailengine/commit/bc0ff758cea8d8713485e3a28e81b16ee56d1dc7))
+* surface TTL reconciliation failures to operators via ttlWarning meta flag ([6e5d5c5](https://github.com/postalsys/emailengine/commit/6e5d5c5a97fb9aaff2662f3fe2f90c8226359b4a))
+* update imapflow to 1.2.15 to fix unhandled rejection crashes ([494a3f8](https://github.com/postalsys/emailengine/commit/494a3f8cd9e71bc1fc1683bea2fc92474cb33206))
+
 ## [2.63.4](https://github.com/postalsys/emailengine/compare/v2.63.3...v2.63.4) (2026-03-09)
 
 

package/copy-static-files.sh

@@ -24,7 +24,7 @@ cp node_modules/ace-builds/src-min/ext-searchbox.js static/js/ace/ext-searchbox.
 
 cp node_modules/\@postalsys/ee-client/index.js static/js/ee-client.js
 
-wget https://developers.google.com/static/search/apis/ipranges/special-crawlers.json -O data/google-crawlers.json
+wget https://developers.google.com/static/crawling/ipranges/special-crawlers.json -O data/google-crawlers.json
 node -e 'console.log("Google crawlers updated: "+require("./data/google-crawlers.json").creationTime);'
 
 # brew install gh

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2026-03-06T15:46:26.000000",
+    "creationTime": "2026-03-20T15:46:03.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/eslint.config.js

@@ -27,6 +27,8 @@ module.exports = [
                 clearTimeout: 'readonly',
                 setInterval: 'readonly',
                 clearInterval: 'readonly',
+                AbortController: 'readonly',
+                AbortSignal: 'readonly',
                 URL: 'readonly',
                 URLSearchParams: 'readonly',
                 structuredClone: 'readonly',

package/lib/account.js

@@ -18,7 +18,7 @@ const { MessageChannel } = require('worker_threads');
 const { MessagePortReadable } = require('./message-port-stream');
 const { deepStrictEqual, strictEqual } = require('assert');
 const { encrypt, decrypt } = require('./encrypt');
-const { oauth2Apps, LEGACY_KEYS } = require('./oauth2-apps');
+const { oauth2Apps, LEGACY_KEYS, isApiBasedApp } = require('./oauth2-apps');
 const settings = require('./settings');
 const redisScanDelete = require('./redis-scan-delete');
 const { customAlphabet } = require('nanoid');
@@ -175,7 +175,7 @@ class Account {
                 oauthApps.set(accountData.oauth2.provider, app || null);
                 if (app) {
                     accountData.type = app.provider;
-                    if (app.baseScopes === 'api') {
+                    if (isApiBasedApp(app)) {
                         accountData.isApi = true;
                     }
 
@@ -203,7 +203,7 @@ class Account {
                                 app = await oauth2Apps.get(delegatedAccountData.oauth2.provider);
                             }
                             oauthApps.set(delegatedAccountData.oauth2.provider, app || null);
-                            if (app && app.baseScopes === 'api') {
+                            if (isApiBasedApp(app)) {
                                 accountData.isApi = true;
                             }
                         }
@@ -686,7 +686,7 @@ class Account {
         if (accountData.oauth2 && accountData.oauth2.provider) {
             let app = await oauth2Apps.get(accountData.oauth2.provider);
             if (app) {
-                if (app.baseScopes === 'api') {
+                if (isApiBasedApp(app)) {
                     accountData.isApi = true;
                 }
 
@@ -709,7 +709,7 @@ class Account {
                         let app = await oauth2Apps.get(delegatedAccountData.oauth2.provider);
                         if (app) {
                             accountData._app = app;
-                            if (app.baseScopes === 'api') {
+                            if (isApiBasedApp(app)) {
                                 accountData.isApi = true;
                             }
                         }
@@ -2374,7 +2374,9 @@ class Account {
                 });
             }
         } finally {
-            await lock.releaseLock(flushLock);
+            if (flushLock?.success) {
+                await lock.releaseLock(flushLock);
+            }
         }
     }
 
@@ -2490,7 +2492,9 @@ class Account {
 
             throw err;
         } finally {
-            await lock.releaseLock(renewLock);
+            if (renewLock?.success) {
+                await lock.releaseLock(renewLock);
+            }
         }
     }
 
@@ -2576,7 +2580,7 @@ class Account {
                     let delegatedAccountData = this.unserializeAccountData(delegatedAccountRow);
                     if (delegatedAccountData?.oauth2?.provider) {
                         let app = await oauth2Apps.get(delegatedAccountData.oauth2.provider);
-                        return app?.baseScopes === 'api';
+                        return isApiBasedApp(app);
                     } else {
                         return false;
                     }
@@ -2589,7 +2593,7 @@ class Account {
 
         if (accountData.oauth2?.provider) {
             let app = await oauth2Apps.get(accountData.oauth2.provider);
-            return app?.baseScopes === 'api';
+            return isApiBasedApp(app);
         }
         return false;
     }

package/lib/api-routes/account-routes.js

@@ -4,7 +4,7 @@ const crypto = require('crypto');
 const { redis } = require('../db');
 const { Account } = require('../account');
 const getSecret = require('../get-secret');
-const { oauth2Apps } = require('../oauth2-apps');
+const { oauth2Apps, SERVICE_ACCOUNT_PROVIDERS } = require('../oauth2-apps');
 const Boom = require('@hapi/boom');
 const Joi = require('joi');
 const { failAction } = require('../tools');
@@ -67,6 +67,12 @@ async function init(args) {
                     // redirect to OAuth2 consent screen
 
                     const oAuth2Client = await oauth2Apps.getClient(request.payload.oauth2.provider);
+
+                    // Service providers use client_credentials - no interactive authorization
+                    if (SERVICE_ACCOUNT_PROVIDERS.has(oAuth2Client.provider)) {
+                        throw Boom.badRequest('Application-only OAuth providers do not support interactive authorization');
+                    }
+
                     const nonce = crypto.randomBytes(NONCE_BYTES).toString('base64url');
 
                     const accountData = request.payload;

package/lib/consts.js

@@ -199,11 +199,27 @@ module.exports = {
     OUTLOOK_EXPIRATION_RENEW_TIME: 24 * 60 * 60 * 1000, // Renew when less than 24 hours remain
 
     // MS Graph API retry and rate limiting settings
-    OUTLOOK_SUBSCRIPTION_LOCK_TTL: 60, // seconds - lock TTL for subscription renewal
+    OUTLOOK_SUBSCRIPTION_LOCK_TTL: 4 * 60 * 1000, // ms - lock TTL for subscription operations (ioredfour uses ms)
     OUTLOOK_MAX_BATCH_SIZE: 20, // MS Graph batch request limit
     OUTLOOK_MAX_RETRY_ATTEMPTS: 3, // Maximum retry attempts for rate-limited requests
     OUTLOOK_RETRY_BASE_DELAY: 30, // seconds - base delay for exponential backoff
     OUTLOOK_RETRY_MAX_DELAY: 120, // seconds - maximum delay between retries
+    OUTLOOK_CLOCK_SKEW_BUFFER: 60 * 1000, // ms - buffer for clock skew between EmailEngine and MS Graph servers
+
+    // Google Pub/Sub subscription defaults
+    GMAIL_PUBSUB_DEFAULT_EXPIRATION_TTL: '2678400s', // 31 days in seconds (Google's default)
+    GMAIL_PUBSUB_ACK_DEADLINE_SECONDS: 30,
+    // Transient network error codes that indicate a retry-worthy connection failure
+    TRANSIENT_NETWORK_CODES: new Set([
+        'ENOTFOUND',
+        'EAI_AGAIN',
+        'ETIMEDOUT',
+        'ECONNRESET',
+        'ECONNREFUSED',
+        'UND_ERR_SOCKET',
+        'UND_ERR_CONNECT_TIMEOUT',
+        'UND_ERR_HEADERS_TIMEOUT'
+    ]),
 
     generateWebhookTable() {
         let entries = [];

package/lib/email-client/gmail/gmail-api.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { metricsMeta } = require('../base-client');
+const { TRANSIENT_NETWORK_CODES } = require('../../consts');
 
 // Gmail API configuration
 const GMAIL_API_BASE = 'https://gmail.googleapis.com';
@@ -12,18 +13,6 @@ const LIST_BATCH_SIZE = 10;
 const MAX_RETRY_ATTEMPTS = 3;
 const RETRY_BASE_DELAY = 1000; // 1 second base delay
 
-// Network-level errors that are transient and should be retried
-const TRANSIENT_NETWORK_CODES = new Set([
-    'ENOTFOUND',
-    'EAI_AGAIN',
-    'ETIMEDOUT',
-    'ECONNRESET',
-    'ECONNREFUSED',
-    'UND_ERR_SOCKET',
-    'UND_ERR_CONNECT_TIMEOUT',
-    'UND_ERR_HEADERS_TIMEOUT'
-]);
-
 // Gmail API error code mapping to internal error codes
 // https://developers.google.com/gmail/api/reference/rest#error-codes
 const GMAIL_ERROR_MAP = {

package/lib/email-client/imap-client.js

@@ -335,9 +335,11 @@ class IMAPClient extends BaseClient {
             return commandClient;
         } finally {
             // Always release the lock
-            this.logger.debug({ msg: 'Releasing connection lock', lockKey, index: connectLock.index });
-            await lock.releaseLock(connectLock);
-            this.logger.debug({ msg: 'Released connection lock', lockKey, index: connectLock.index });
+            if (connectLock?.success) {
+                this.logger.debug({ msg: 'Releasing connection lock', lockKey, index: connectLock.index });
+                await lock.releaseLock(connectLock);
+                this.logger.debug({ msg: 'Released connection lock', lockKey, index: connectLock.index });
+            }
         }
     }
 

package/lib/email-client/outlook/graph-api.js

@@ -2,23 +2,17 @@
 
 const { metricsMeta } = require('../base-client');
 
-const { OUTLOOK_MAX_BATCH_SIZE, OUTLOOK_MAX_RETRY_ATTEMPTS, OUTLOOK_RETRY_BASE_DELAY, OUTLOOK_RETRY_MAX_DELAY } = require('../../consts');
+const {
+    OUTLOOK_MAX_BATCH_SIZE,
+    OUTLOOK_MAX_RETRY_ATTEMPTS,
+    OUTLOOK_RETRY_BASE_DELAY,
+    OUTLOOK_RETRY_MAX_DELAY,
+    TRANSIENT_NETWORK_CODES
+} = require('../../consts');
 
 // Maximum number of operations in a single batch request to Microsoft Graph API
 const MAX_BATCH_SIZE = OUTLOOK_MAX_BATCH_SIZE;
 
-// Network-level errors that are transient and should be retried
-const TRANSIENT_NETWORK_CODES = new Set([
-    'ENOTFOUND',
-    'EAI_AGAIN',
-    'ETIMEDOUT',
-    'ECONNRESET',
-    'ECONNREFUSED',
-    'UND_ERR_SOCKET',
-    'UND_ERR_CONNECT_TIMEOUT',
-    'UND_ERR_HEADERS_TIMEOUT'
-]);
-
 // MS Graph API error code mapping to internal error codes
 // https://learn.microsoft.com/en-us/graph/errors
 const GRAPH_ERROR_MAP = {
@@ -100,7 +94,7 @@ async function request(context, url, method, payload, options = {}) {
         // Track successful API request
         metricsMeta({ account: context.account }, context.logger, 'oauth2ApiRequest', 'inc', {
             status: 'success',
-            provider: 'outlook',
+            provider: context.oAuth2Client?.provider || 'outlook',
             statusCode: '200'
         });
     } catch (err) {
@@ -108,7 +102,7 @@ async function request(context, url, method, payload, options = {}) {
         const statusCode = String(err.oauthRequest?.status || 0);
         metricsMeta({ account: context.account }, context.logger, 'oauth2ApiRequest', 'inc', {
             status: 'failure',
-            provider: 'outlook',
+            provider: context.oAuth2Client?.provider || 'outlook',
             statusCode
         });