emailengine-app 2.70.0 → 2.71.0

package/lib/api-routes/message-routes.js

@@ -46,7 +46,7 @@ const idMapSchema = Joi.array()
     .label('IdMapArray');
 
 async function init(args) {
-    const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT } = args;
+    const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT, documentStoreFeatureEnabled } = args;
 
     // GET /v1/account/{account}/message/{message}/source - Download raw message
     server.route({
@@ -1133,138 +1133,142 @@ async function init(args) {
     });
 
     // POST /v1/unified/search - Unified search for messages
-    server.route({
-        method: 'POST',
-        path: '/v1/unified/search',
-
-        async handler(request, h) {
-            let accountObject = new Account({
-                redis,
-                call,
-                secret: await getSecret(),
-                esClient: await h.getESClient(request.logger),
-                timeout: request.headers['x-ee-timeout']
-            });
+    // Deprecated Document Store feature: only register this endpoint when the feature is enabled,
+    // otherwise it behaves like a regular 404.
+    if (documentStoreFeatureEnabled) {
+        server.route({
+            method: 'POST',
+            path: '/v1/unified/search',
+
+            async handler(request, h) {
+                let accountObject = new Account({
+                    redis,
+                    call,
+                    secret: await getSecret(),
+                    esClient: await h.getESClient(request.logger),
+                    timeout: request.headers['x-ee-timeout']
+                });
+
+                let extraValidationErrors = [];
 
-            let extraValidationErrors = [];
-
-            for (let key of ['seq', 'modseq']) {
-                if (request.payload.search && key in request.payload.search) {
-                    extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
+                for (let key of ['seq', 'modseq']) {
+                    if (request.payload.search && key in request.payload.search) {
+                        extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
+                    }
                 }
-            }
-
-            if (extraValidationErrors.length) {
-                let error = new Error('Input validation failed');
-                error.details = extraValidationErrors;
-                return failAction(request, h, error);
-            }
 
-            let documentStoreEnabled = await settings.get('documentStoreEnabled');
-            if (!documentStoreEnabled) {
-                let error = new Error('Document store not enabled');
-                error.details = extraValidationErrors;
-                return failAction(request, h, error);
-            }
-
-            try {
-                return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
-            } catch (err) {
-                handleError(request, err);
-            }
-        },
-        options: {
-            description: 'Unified search for messages',
-            notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
-            tags: ['api', 'Deprecated endpoints (Document Store)'],
+                if (extraValidationErrors.length) {
+                    let error = new Error('Input validation failed');
+                    error.details = extraValidationErrors;
+                    return failAction(request, h, error);
+                }
 
-            plugins: {
-                'hapi-swagger': {
-                    responses: errorResponses(400, 401, 403, 429, 500)
+                let documentStoreEnabled = await settings.get('documentStoreEnabled');
+                if (!documentStoreEnabled) {
+                    let error = new Error('Document store not enabled');
+                    error.details = extraValidationErrors;
+                    return failAction(request, h, error);
                 }
-            },
 
-            auth: {
-                strategy: 'api-token',
-                mode: 'required'
+                try {
+                    return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
+                } catch (err) {
+                    handleError(request, err);
+                }
             },
-            cors: CORS_CONFIG,
-
-            validate: {
-                options: {
-                    stripUnknown: false,
-                    abortEarly: false,
-                    convert: true
+            options: {
+                description: 'Unified search for messages',
+                notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
+                tags: ['api', 'Deprecated endpoints (Document Store)'],
+
+                plugins: {
+                    'hapi-swagger': {
+                        responses: errorResponses(400, 401, 403, 429, 500)
+                    }
                 },
-                failAction,
 
-                query: Joi.object({
-                    page: Joi.number()
-                        .integer()
-                        .min(0)
-                        .max(1024 * 1024)
-                        .default(0)
-                        .example(0)
-                        .description('Page number (zero indexed, so use 0 for first page)'),
-                    pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'),
-                    exposeQuery: Joi.boolean()
-                        .truthy('Y', 'true', '1')
-                        .falsy('N', 'false', 0)
-                        .description('If enabled then includes the combined query (as the documentStoreQuery field) in the response for debugging')
-                        .label('exposeQuery')
-                        .optional()
-                        .meta({ swaggerHidden: true })
-                }),
+                auth: {
+                    strategy: 'api-token',
+                    mode: 'required'
+                },
+                cors: CORS_CONFIG,
+
+                validate: {
+                    options: {
+                        stripUnknown: false,
+                        abortEarly: false,
+                        convert: true
+                    },
+                    failAction,
+
+                    query: Joi.object({
+                        page: Joi.number()
+                            .integer()
+                            .min(0)
+                            .max(1024 * 1024)
+                            .default(0)
+                            .example(0)
+                            .description('Page number (zero indexed, so use 0 for first page)'),
+                        pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'),
+                        exposeQuery: Joi.boolean()
+                            .truthy('Y', 'true', '1')
+                            .falsy('N', 'false', 0)
+                            .description('If enabled then includes the combined query (as the documentStoreQuery field) in the response for debugging')
+                            .label('exposeQuery')
+                            .optional()
+                            .meta({ swaggerHidden: true })
+                    }),
 
-                payload: Joi.object({
-                    accounts: Joi.array()
-                        .items(Joi.string().empty('').trim().max(256).example('example'))
-                        .single()
-                        .description('Optional list of account ID values')
-                        .label('UnifiedSearchAccounts'),
-                    paths: Joi.array()
-                        .items(Joi.string().optional().example('INBOX'))
-                        .single()
-                        .description('Optional list of mailbox folder paths or specialUse flags')
-                        .label('UnifiedSearchPaths'),
-                    search: searchSchema,
-                    documentQuery: Joi.object().min(1).description('Document Store query').label('DocumentQuery').unknown().meta({ swaggerHidden: true })
-                }).label('UnifiedSearchQuery')
-            },
+                    payload: Joi.object({
+                        accounts: Joi.array()
+                            .items(Joi.string().empty('').trim().max(256).example('example'))
+                            .single()
+                            .description('Optional list of account ID values')
+                            .label('UnifiedSearchAccounts'),
+                        paths: Joi.array()
+                            .items(Joi.string().optional().example('INBOX'))
+                            .single()
+                            .description('Optional list of mailbox folder paths or specialUse flags')
+                            .label('UnifiedSearchPaths'),
+                        search: searchSchema,
+                        documentQuery: Joi.object().min(1).description('Document Store query').label('DocumentQuery').unknown().meta({ swaggerHidden: true })
+                    }).label('UnifiedSearchQuery')
+                },
 
-            response: {
-                schema: Joi.object({
-                    total: Joi.number()
-                        .integer()
-                        .example(120)
-                        .description('Total number of matching messages (capped at 10000 by the Document Store)')
-                        .label('UnifiedTotalNumber'),
-                    page: Joi.number().integer().example(0).description('Current page number (zero-based)').label('UnifiedPageNumber'),
-                    pages: Joi.number().integer().example(24).description('Total number of pages available').label('UnifiedPagesNumber'),
-                    accounts: Joi.array()
-                        .items(Joi.string().example('example'))
-                        .description('Account filter used for the search, if provided')
-                        .label('UnifiedSearchAccountsEcho'),
-                    paths: Joi.array()
-                        .items(Joi.string().example('INBOX'))
-                        .description('Path filter used for the search, if provided')
-                        .label('UnifiedSearchPathsEcho'),
-                    messages: Joi.array()
-                        .items(
-                            messageEntrySchema
-                                .keys({
-                                    account: accountIdSchema.description('Account ID this message belongs to')
-                                })
-                                .unknown()
-                                .label('UnifiedMessageListEntry')
-                        )
-                        .label('UnifiedPageMessages'),
-                    documentStoreQuery: documentStoreQuerySchema
-                }).label('UnifiedSearchResponse'),
-                failAction: 'log'
+                response: {
+                    schema: Joi.object({
+                        total: Joi.number()
+                            .integer()
+                            .example(120)
+                            .description('Total number of matching messages (capped at 10000 by the Document Store)')
+                            .label('UnifiedTotalNumber'),
+                        page: Joi.number().integer().example(0).description('Current page number (zero-based)').label('UnifiedPageNumber'),
+                        pages: Joi.number().integer().example(24).description('Total number of pages available').label('UnifiedPagesNumber'),
+                        accounts: Joi.array()
+                            .items(Joi.string().example('example'))
+                            .description('Account filter used for the search, if provided')
+                            .label('UnifiedSearchAccountsEcho'),
+                        paths: Joi.array()
+                            .items(Joi.string().example('INBOX'))
+                            .description('Path filter used for the search, if provided')
+                            .label('UnifiedSearchPathsEcho'),
+                        messages: Joi.array()
+                            .items(
+                                messageEntrySchema
+                                    .keys({
+                                        account: accountIdSchema.description('Account ID this message belongs to')
+                                    })
+                                    .unknown()
+                                    .label('UnifiedMessageListEntry')
+                            )
+                            .label('UnifiedPageMessages'),
+                        documentStoreQuery: documentStoreQuerySchema
+                    }).label('UnifiedSearchResponse'),
+                    failAction: 'log'
+                }
             }
-        }
-    });
+        });
+    }
 
     // GET /v1/account/{account}/text/{text} - Retrieve message text
     server.route({

package/lib/document-store.js

@@ -1,12 +1,33 @@
 'use strict';
 
+const config = require('@zone-eu/wild-config');
 const settings = require('./settings');
 const { Client: ElasticSearch } = require('@elastic/elasticsearch');
 const { ensureIndex } = require('./es');
+const { hasEnvValue, readEnvValue, getBoolean } = require('./tools');
+
+// Deployment-level gate for the deprecated Document Store feature. When this is false the
+// "documents" worker is not spawned, document-store-only endpoints are not registered, and
+// all runtime document-store code takes its existing "disabled" path. Set via the
+// --documentStore.enabled CLI flag / [documentStore] enabled config or EENGINE_DOCUMENT_STORE_ENABLED.
+const documentStoreFeatureEnabled = hasEnvValue('EENGINE_DOCUMENT_STORE_ENABLED')
+    ? getBoolean(readEnvValue('EENGINE_DOCUMENT_STORE_ENABLED'))
+    : getBoolean(config.documentStore && config.documentStore.enabled);
+
+// Effective runtime state: the feature must be available (gate) AND enabled in settings.
+// When the gate is off this is always false, so callers reuse the already-tested
+// "documentStoreEnabled is false" code paths.
+const isDocumentStoreEnabled = async () => documentStoreFeatureEnabled && !!(await settings.get('documentStoreEnabled'));
 
 const clientCache = { version: -1, config: false, client: false, index: false };
 
 const getESClient = async logger => {
+    // Feature gate is off: behave exactly as a disabled document store. Checked before any
+    // Redis access so disabled deployments do not pay a round-trip on every getESClient call.
+    if (!documentStoreFeatureEnabled) {
+        return clientCache;
+    }
+
     const documentStoreVersion = (await settings.get('documentStoreVersion')) || 0;
     if (clientCache.version === documentStoreVersion) {
         return clientCache;
@@ -51,4 +72,4 @@ const getESClient = async logger => {
     return clientCache;
 };
 
-module.exports = { getESClient };
+module.exports = { getESClient, documentStoreFeatureEnabled, isDocumentStoreEnabled };

package/lib/email-client/base-client.js

@@ -4,6 +4,7 @@ const { threadId: workerThreadId } = require('worker_threads');
 const crypto = require('crypto');
 const logger = require('../logger');
 const settings = require('../settings');
+const { isDocumentStoreEnabled } = require('../document-store');
 const msgpack = require('msgpack5')();
 const { templates } = require('../templates');
 const { Gateway } = require('../gateway');
@@ -1076,7 +1077,7 @@ class BaseClient {
         // Resolve reference and update reference/in-reply-to headers
         if (data.reference && data.reference.message) {
             // Try document store first if enabled
-            if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
+            if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
                 try {
                     referencedMessage = await this.accountObject.getMessage(data.reference.message, {
                         documentStore: true,
@@ -1578,7 +1579,7 @@ class BaseClient {
         // Resolve reference and update reference/in-reply-to headers
         if (data.reference && data.reference.message) {
             // Try document store first if enabled
-            if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
+            if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
                 try {
                     referencedMessage = await this.accountObject.getMessage(data.reference.message, {
                         documentStore: true,

package/lib/email-client/imap/mailbox.js

@@ -15,7 +15,7 @@ const ical = require('ical.js');
 const addressparser = require('nodemailer/lib/addressparser');
 const { llmPreProcess } = require('../../llm-pre-process');
 
-const { getESClient } = require('../../document-store');
+const { getESClient, isDocumentStoreEnabled } = require('../../document-store');
 
 const {
     MESSAGE_NEW_NOTIFY,
@@ -1958,7 +1958,7 @@ class Mailbox {
     async publishSyncedEvents(storedStatus) {
         let messageFetchOptions = {};
 
-        let documentStoreEnabled = await settings.get('documentStoreEnabled');
+        let documentStoreEnabled = await isDocumentStoreEnabled();
 
         // Configure text fetching
         let notifyText = await settings.get('notifyText');

package/lib/email-client/notification-handler.js

@@ -3,7 +3,7 @@
 const { parentPort } = require('worker_threads');
 const logger = require('../logger');
 const { webhooks: Webhooks } = require('../webhooks');
-const { getESClient } = require('../document-store');
+const { getESClient, isDocumentStoreEnabled } = require('../document-store');
 const { getThread } = require('../threads');
 const settings = require('../settings');
 
@@ -126,7 +126,7 @@ class NotificationHandler {
             return false;
         }
 
-        return await settings.get('documentStoreEnabled');
+        return await isDocumentStoreEnabled();
     }
 
     /**

package/lib/export.js

@@ -99,6 +99,17 @@ function isFolderMissingError(err) {
     return ['FolderNotFound', 'NotFound'].includes(errCode);
 }
 
+// Per-message errors worth retrying during a batch export: rate limits, a dropped batch response
+// (Outlook reports these as EMISSING_RESPONSE), and the same transient network/5xx/timeout errors
+// isTransientError already covers. Used by the export worker's API-account batch fetch path so a
+// single transient blip does not fail an entire multi-message export.
+function isRetryableError(err) {
+    if (err.statusCode === 429 || ['rateLimitExceeded', 'userRateLimitExceeded', 'EMISSING_RESPONSE'].includes(err.code)) {
+        return true;
+    }
+    return isTransientError(err);
+}
+
 async function tryAddToActiveSet(account, exportId) {
     const maxConcurrent = (await settings.get('exportMaxConcurrent')) || DEFAULT_EXPORT_MAX_CONCURRENT;
     const maxGlobalConcurrent = (await settings.get('exportMaxGlobalConcurrent')) || DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT;
@@ -618,5 +629,6 @@ module.exports = {
     isTransientError,
     isSkippableError,
     isFolderMissingError,
+    isRetryableError,
     DEFAULT_EXPORT_MAX_MESSAGE_SIZE
 };

package/lib/feature-flags.js

@@ -34,6 +34,12 @@ for (let key of Object.keys(process.env)) {
 module.exports = {
     enabled(key) {
         return FEATURES.has(formatFeatureKey(key));
+    },
+
+    // Sorted list of the currently enabled feature flag keys. Used by the license beacon
+    // to report which EENGINE_FEATURE_* flags are active on this instance.
+    listEnabled() {
+        return Array.from(FEATURES).sort();
     }
 };