emailengine-app 2.70.0 → 2.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/.github/workflows/codeql.yml +3 -0
  2. package/.github/workflows/e2e.yml +56 -0
  3. package/.github/workflows/test.yml +81 -12
  4. package/.ncurc.js +20 -20
  5. package/CHANGELOG.md +25 -0
  6. package/Gruntfile.js +19 -23
  7. package/bin/emailengine.js +8 -1
  8. package/config/default.toml +5 -0
  9. package/config/e2e.toml +35 -0
  10. package/config/test.toml +5 -0
  11. package/data/google-crawlers.json +1 -1
  12. package/getswagger.sh +4 -0
  13. package/lib/account.js +31 -25
  14. package/lib/api-routes/message-routes.js +125 -121
  15. package/lib/auth-token.js +83 -0
  16. package/lib/delivery-error.js +62 -0
  17. package/lib/document-store.js +22 -1
  18. package/lib/email-client/base-client.js +3 -2
  19. package/lib/email-client/gmail-client.js +33 -1
  20. package/lib/email-client/imap/mailbox.js +2 -2
  21. package/lib/email-client/notification-handler.js +2 -2
  22. package/lib/export.js +12 -0
  23. package/lib/feature-flags.js +6 -0
  24. package/lib/imap-proxy-auth.js +81 -0
  25. package/lib/imapproxy/imap-server.js +8 -103
  26. package/lib/license-beacon.js +367 -0
  27. package/lib/logger.js +11 -1
  28. package/lib/oauth/gmail.js +3 -0
  29. package/lib/oauth/outlook.js +3 -0
  30. package/lib/oauth2-apps.js +100 -11
  31. package/lib/routes-ui.js +2 -1
  32. package/lib/smtp-auth.js +70 -0
  33. package/lib/sub-script.js +8 -2
  34. package/lib/tools.js +26 -2
  35. package/lib/ui-routes/admin-config-routes.js +4 -3
  36. package/lib/ui-routes/document-store-routes.js +7 -1
  37. package/package.json +30 -24
  38. package/playwright.config.js +45 -0
  39. package/sbom.json +1 -1
  40. package/server.js +30 -8
  41. package/static/licenses.html +108 -128
  42. package/test-coverage-plan.md +233 -0
  43. package/translations/de.mo +0 -0
  44. package/translations/de.po +154 -142
  45. package/translations/et.mo +0 -0
  46. package/translations/et.po +129 -131
  47. package/translations/fr.mo +0 -0
  48. package/translations/fr.po +133 -136
  49. package/translations/ja.mo +0 -0
  50. package/translations/ja.po +126 -129
  51. package/translations/messages.pot +37 -37
  52. package/translations/nl.mo +0 -0
  53. package/translations/nl.po +128 -130
  54. package/translations/pl.mo +0 -0
  55. package/translations/pl.po +125 -128
  56. package/views/dashboard.hbs +22 -0
  57. package/workers/api.js +22 -5
  58. package/workers/export.js +58 -43
  59. package/workers/smtp.js +5 -85
  60. package/workers/submit.js +2 -12
@@ -46,7 +46,7 @@ const idMapSchema = Joi.array()
46
46
  .label('IdMapArray');
47
47
 
48
48
  async function init(args) {
49
- const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT } = args;
49
+ const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT, documentStoreFeatureEnabled } = args;
50
50
 
51
51
  // GET /v1/account/{account}/message/{message}/source - Download raw message
52
52
  server.route({
@@ -1133,138 +1133,142 @@ async function init(args) {
1133
1133
  });
1134
1134
 
1135
1135
  // POST /v1/unified/search - Unified search for messages
1136
- server.route({
1137
- method: 'POST',
1138
- path: '/v1/unified/search',
1139
-
1140
- async handler(request, h) {
1141
- let accountObject = new Account({
1142
- redis,
1143
- call,
1144
- secret: await getSecret(),
1145
- esClient: await h.getESClient(request.logger),
1146
- timeout: request.headers['x-ee-timeout']
1147
- });
1136
+ // Deprecated Document Store feature: only register this endpoint when the feature is enabled,
1137
+ // otherwise it behaves like a regular 404.
1138
+ if (documentStoreFeatureEnabled) {
1139
+ server.route({
1140
+ method: 'POST',
1141
+ path: '/v1/unified/search',
1142
+
1143
+ async handler(request, h) {
1144
+ let accountObject = new Account({
1145
+ redis,
1146
+ call,
1147
+ secret: await getSecret(),
1148
+ esClient: await h.getESClient(request.logger),
1149
+ timeout: request.headers['x-ee-timeout']
1150
+ });
1151
+
1152
+ let extraValidationErrors = [];
1148
1153
 
1149
- let extraValidationErrors = [];
1150
-
1151
- for (let key of ['seq', 'modseq']) {
1152
- if (request.payload.search && key in request.payload.search) {
1153
- extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
1154
+ for (let key of ['seq', 'modseq']) {
1155
+ if (request.payload.search && key in request.payload.search) {
1156
+ extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
1157
+ }
1154
1158
  }
1155
- }
1156
-
1157
- if (extraValidationErrors.length) {
1158
- let error = new Error('Input validation failed');
1159
- error.details = extraValidationErrors;
1160
- return failAction(request, h, error);
1161
- }
1162
1159
 
1163
- let documentStoreEnabled = await settings.get('documentStoreEnabled');
1164
- if (!documentStoreEnabled) {
1165
- let error = new Error('Document store not enabled');
1166
- error.details = extraValidationErrors;
1167
- return failAction(request, h, error);
1168
- }
1169
-
1170
- try {
1171
- return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
1172
- } catch (err) {
1173
- handleError(request, err);
1174
- }
1175
- },
1176
- options: {
1177
- description: 'Unified search for messages',
1178
- notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
1179
- tags: ['api', 'Deprecated endpoints (Document Store)'],
1160
+ if (extraValidationErrors.length) {
1161
+ let error = new Error('Input validation failed');
1162
+ error.details = extraValidationErrors;
1163
+ return failAction(request, h, error);
1164
+ }
1180
1165
 
1181
- plugins: {
1182
- 'hapi-swagger': {
1183
- responses: errorResponses(400, 401, 403, 429, 500)
1166
+ let documentStoreEnabled = await settings.get('documentStoreEnabled');
1167
+ if (!documentStoreEnabled) {
1168
+ let error = new Error('Document store not enabled');
1169
+ error.details = extraValidationErrors;
1170
+ return failAction(request, h, error);
1184
1171
  }
1185
- },
1186
1172
 
1187
- auth: {
1188
- strategy: 'api-token',
1189
- mode: 'required'
1173
+ try {
1174
+ return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
1175
+ } catch (err) {
1176
+ handleError(request, err);
1177
+ }
1190
1178
  },
1191
- cors: CORS_CONFIG,
1192
-
1193
- validate: {
1194
- options: {
1195
- stripUnknown: false,
1196
- abortEarly: false,
1197
- convert: true
1179
+ options: {
1180
+ description: 'Unified search for messages',
1181
+ notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
1182
+ tags: ['api', 'Deprecated endpoints (Document Store)'],
1183
+
1184
+ plugins: {
1185
+ 'hapi-swagger': {
1186
+ responses: errorResponses(400, 401, 403, 429, 500)
1187
+ }
1198
1188
  },
1199
- failAction,
1200
1189
 
1201
- query: Joi.object({
1202
- page: Joi.number()
1203
- .integer()
1204
- .min(0)
1205
- .max(1024 * 1024)
1206
- .default(0)
1207
- .example(0)
1208
- .description('Page number (zero indexed, so use 0 for first page)'),
1209
- pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'),
1210
- exposeQuery: Joi.boolean()
1211
- .truthy('Y', 'true', '1')
1212
- .falsy('N', 'false', 0)
1213
- .description('If enabled then includes the combined query (as the documentStoreQuery field) in the response for debugging')
1214
- .label('exposeQuery')
1215
- .optional()
1216
- .meta({ swaggerHidden: true })
1217
- }),
1190
+ auth: {
1191
+ strategy: 'api-token',
1192
+ mode: 'required'
1193
+ },
1194
+ cors: CORS_CONFIG,
1195
+
1196
+ validate: {
1197
+ options: {
1198
+ stripUnknown: false,
1199
+ abortEarly: false,
1200
+ convert: true
1201
+ },
1202
+ failAction,
1203
+
1204
+ query: Joi.object({
1205
+ page: Joi.number()
1206
+ .integer()
1207
+ .min(0)
1208
+ .max(1024 * 1024)
1209
+ .default(0)
1210
+ .example(0)
1211
+ .description('Page number (zero indexed, so use 0 for first page)'),
1212
+ pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'),
1213
+ exposeQuery: Joi.boolean()
1214
+ .truthy('Y', 'true', '1')
1215
+ .falsy('N', 'false', 0)
1216
+ .description('If enabled then includes the combined query (as the documentStoreQuery field) in the response for debugging')
1217
+ .label('exposeQuery')
1218
+ .optional()
1219
+ .meta({ swaggerHidden: true })
1220
+ }),
1218
1221
 
1219
- payload: Joi.object({
1220
- accounts: Joi.array()
1221
- .items(Joi.string().empty('').trim().max(256).example('example'))
1222
- .single()
1223
- .description('Optional list of account ID values')
1224
- .label('UnifiedSearchAccounts'),
1225
- paths: Joi.array()
1226
- .items(Joi.string().optional().example('INBOX'))
1227
- .single()
1228
- .description('Optional list of mailbox folder paths or specialUse flags')
1229
- .label('UnifiedSearchPaths'),
1230
- search: searchSchema,
1231
- documentQuery: Joi.object().min(1).description('Document Store query').label('DocumentQuery').unknown().meta({ swaggerHidden: true })
1232
- }).label('UnifiedSearchQuery')
1233
- },
1222
+ payload: Joi.object({
1223
+ accounts: Joi.array()
1224
+ .items(Joi.string().empty('').trim().max(256).example('example'))
1225
+ .single()
1226
+ .description('Optional list of account ID values')
1227
+ .label('UnifiedSearchAccounts'),
1228
+ paths: Joi.array()
1229
+ .items(Joi.string().optional().example('INBOX'))
1230
+ .single()
1231
+ .description('Optional list of mailbox folder paths or specialUse flags')
1232
+ .label('UnifiedSearchPaths'),
1233
+ search: searchSchema,
1234
+ documentQuery: Joi.object().min(1).description('Document Store query').label('DocumentQuery').unknown().meta({ swaggerHidden: true })
1235
+ }).label('UnifiedSearchQuery')
1236
+ },
1234
1237
 
1235
- response: {
1236
- schema: Joi.object({
1237
- total: Joi.number()
1238
- .integer()
1239
- .example(120)
1240
- .description('Total number of matching messages (capped at 10000 by the Document Store)')
1241
- .label('UnifiedTotalNumber'),
1242
- page: Joi.number().integer().example(0).description('Current page number (zero-based)').label('UnifiedPageNumber'),
1243
- pages: Joi.number().integer().example(24).description('Total number of pages available').label('UnifiedPagesNumber'),
1244
- accounts: Joi.array()
1245
- .items(Joi.string().example('example'))
1246
- .description('Account filter used for the search, if provided')
1247
- .label('UnifiedSearchAccountsEcho'),
1248
- paths: Joi.array()
1249
- .items(Joi.string().example('INBOX'))
1250
- .description('Path filter used for the search, if provided')
1251
- .label('UnifiedSearchPathsEcho'),
1252
- messages: Joi.array()
1253
- .items(
1254
- messageEntrySchema
1255
- .keys({
1256
- account: accountIdSchema.description('Account ID this message belongs to')
1257
- })
1258
- .unknown()
1259
- .label('UnifiedMessageListEntry')
1260
- )
1261
- .label('UnifiedPageMessages'),
1262
- documentStoreQuery: documentStoreQuerySchema
1263
- }).label('UnifiedSearchResponse'),
1264
- failAction: 'log'
1238
+ response: {
1239
+ schema: Joi.object({
1240
+ total: Joi.number()
1241
+ .integer()
1242
+ .example(120)
1243
+ .description('Total number of matching messages (capped at 10000 by the Document Store)')
1244
+ .label('UnifiedTotalNumber'),
1245
+ page: Joi.number().integer().example(0).description('Current page number (zero-based)').label('UnifiedPageNumber'),
1246
+ pages: Joi.number().integer().example(24).description('Total number of pages available').label('UnifiedPagesNumber'),
1247
+ accounts: Joi.array()
1248
+ .items(Joi.string().example('example'))
1249
+ .description('Account filter used for the search, if provided')
1250
+ .label('UnifiedSearchAccountsEcho'),
1251
+ paths: Joi.array()
1252
+ .items(Joi.string().example('INBOX'))
1253
+ .description('Path filter used for the search, if provided')
1254
+ .label('UnifiedSearchPathsEcho'),
1255
+ messages: Joi.array()
1256
+ .items(
1257
+ messageEntrySchema
1258
+ .keys({
1259
+ account: accountIdSchema.description('Account ID this message belongs to')
1260
+ })
1261
+ .unknown()
1262
+ .label('UnifiedMessageListEntry')
1263
+ )
1264
+ .label('UnifiedPageMessages'),
1265
+ documentStoreQuery: documentStoreQuerySchema
1266
+ }).label('UnifiedSearchResponse'),
1267
+ failAction: 'log'
1268
+ }
1265
1269
  }
1266
- }
1267
- });
1270
+ });
1271
+ }
1268
1272
 
1269
1273
  // GET /v1/account/{account}/text/{text} - Retrieve message text
1270
1274
  server.route({
@@ -0,0 +1,83 @@
1
+ 'use strict';
2
+
3
+ // Shared token validation for the SMTP and IMAP-proxy submission servers. Both
4
+ // servers accept a 64-char hex API token as the password and apply the same
5
+ // checks (account binding, scope, IP allowlist). The logic lives here so the two
6
+ // auth handlers (lib/smtp-auth.js, lib/imap-proxy-auth.js) cannot drift apart -
7
+ // a token security-policy change is made once and applies to both surfaces.
8
+
9
+ const logger = require('./logger');
10
+ const tokens = require('./tokens');
11
+ const { matchIp } = require('./utils/network');
12
+
13
+ // Reason-specific denial messages, shared verbatim by both auth handlers. The
14
+ // generic "failed to authenticate" fallback and any protocol-specific error
15
+ // decoration are left to each caller.
16
+ const REASON_MESSAGES = {
17
+ username: 'Access denied, invalid username',
18
+ scope: 'Access denied, invalid scope',
19
+ ip: 'Access denied, traffic not accepted from this IP'
20
+ };
21
+
22
+ /**
23
+ * Validates a 64-char hex API token supplied as a server password.
24
+ *
25
+ * Performs the token lookup plus the account-binding, scope and IP-allowlist
26
+ * checks. Does NOT throw - the caller maps the returned reason to its own
27
+ * protocol-specific error (SMTP and IMAP use different response shapes).
28
+ *
29
+ * @param {Object} opts
30
+ * @param {String} opts.password - supplied password (candidate token)
31
+ * @param {String} opts.account - username the client authenticated as
32
+ * @param {String} opts.requiredScope - scope the token must hold ('smtp' | 'imap-proxy')
33
+ * @param {String} opts.remoteAddress - client IP, checked against token restrictions
34
+ * @returns {Promise<{authenticated: Boolean, reason: (null|'username'|'scope'|'ip')}>}
35
+ */
36
+ async function validateAuthToken({ password, account, requiredScope, remoteAddress }) {
37
+ if (!/^[0-9a-f]{64}$/i.test(password)) {
38
+ return { authenticated: false, reason: null };
39
+ }
40
+
41
+ let tokenData;
42
+ try {
43
+ tokenData = await tokens.get(password, false, { log: true, remoteAddress });
44
+ } catch (err) {
45
+ logger.error({ msg: 'Failed to fetch token', err });
46
+ }
47
+
48
+ if (!tokenData) {
49
+ return { authenticated: false, reason: null };
50
+ }
51
+
52
+ if (tokenData.account && tokenData.account !== account) {
53
+ return { authenticated: false, reason: 'username' };
54
+ }
55
+
56
+ if (tokenData.scopes && !tokenData.scopes.includes(requiredScope) && !tokenData.scopes.includes('*')) {
57
+ logger.error({
58
+ msg: 'Trying to use invalid scope for a token',
59
+ tokenAccount: tokenData.account,
60
+ tokenId: tokenData.id,
61
+ account,
62
+ requestedScope: requiredScope,
63
+ scopes: tokenData.scopes
64
+ });
65
+ return { authenticated: false, reason: 'scope' };
66
+ }
67
+
68
+ if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(remoteAddress, tokenData.restrictions.addresses)) {
69
+ logger.error({
70
+ msg: 'Trying to use invalid IP for a token',
71
+ tokenAccount: tokenData.account,
72
+ tokenId: tokenData.id,
73
+ account,
74
+ remoteAddress,
75
+ addressAllowlist: tokenData.restrictions.addresses
76
+ });
77
+ return { authenticated: false, reason: 'ip' };
78
+ }
79
+
80
+ return { authenticated: true, reason: null };
81
+ }
82
+
83
+ module.exports = { validateAuthToken, REASON_MESSAGES };
@@ -0,0 +1,62 @@
1
+ 'use strict';
2
+
3
+ // Classification of outbound delivery errors for the submit worker.
4
+ //
5
+ // This logic lives in its own module so it can be unit tested without booting
6
+ // the BullMQ submit worker (workers/submit.js connects to Redis and expects a
7
+ // worker-thread parentPort at require time).
8
+
9
+ // Nodemailer/SMTP error codes that represent permanent failures. A message that
10
+ // fails with one of these will fail again on every retry, so the job is
11
+ // discarded instead of being retried.
12
+ const NON_RETRYABLE_CODES = new Set([
13
+ 'EAUTH', // authentication failed
14
+ 'ENOAUTH', // no credentials provided
15
+ 'EOAUTH2', // OAuth2 token failure
16
+ 'ETLS', // TLS handshake failed
17
+ 'EENVELOPE', // invalid sender/recipients
18
+ 'EMESSAGE', // message content error
19
+ 'EPROTOCOL' // SMTP protocol mismatch
20
+ ]);
21
+
22
+ /**
23
+ * Determines whether a delivery error is permanent (must not be retried).
24
+ *
25
+ * An error is permanent when either:
26
+ * - the SMTP status code is a 5xx other than 503 (503 is treated as a
27
+ * transient "try again later" response), or
28
+ * - the error carries one of the NON_RETRYABLE_CODES.
29
+ *
30
+ * @param {Object} err - Error thrown during submission
31
+ * @param {Number} [err.statusCode] - SMTP/HTTP status code
32
+ * @param {String} [err.code] - Nodemailer error code
33
+ * @returns {Boolean} True if the error should never be retried
34
+ */
35
+ function isPermanentDeliveryError(err) {
36
+ if (!err) {
37
+ return false;
38
+ }
39
+ const isPermanentSmtp = err.statusCode >= 500 && err.statusCode !== 503;
40
+ const isPermanentCode = NON_RETRYABLE_CODES.has(err.code);
41
+ return isPermanentSmtp || isPermanentCode;
42
+ }
43
+
44
+ /**
45
+ * Determines whether the submit worker should discard a job (stop retrying).
46
+ *
47
+ * A job is discarded only when the error is permanent AND there are still
48
+ * attempts remaining. When attempts are already exhausted, BullMQ will fail the
49
+ * job naturally, so there is nothing to discard.
50
+ *
51
+ * @param {Object} err - Error thrown during submission
52
+ * @param {Object} job - BullMQ job
53
+ * @param {Number} job.attemptsMade - Attempts already made
54
+ * @param {Object} job.opts - Job options
55
+ * @param {Number} job.opts.attempts - Configured max attempts
56
+ * @returns {Boolean} True if the job should be discarded
57
+ */
58
+ function shouldDiscardJob(err, job) {
59
+ return isPermanentDeliveryError(err) && job.attemptsMade < job.opts.attempts;
60
+ }
61
+
62
+ module.exports = { NON_RETRYABLE_CODES, isPermanentDeliveryError, shouldDiscardJob };
@@ -1,12 +1,33 @@
1
1
  'use strict';
2
2
 
3
+ const config = require('@zone-eu/wild-config');
3
4
  const settings = require('./settings');
4
5
  const { Client: ElasticSearch } = require('@elastic/elasticsearch');
5
6
  const { ensureIndex } = require('./es');
7
+ const { hasEnvValue, readEnvValue, getBoolean } = require('./tools');
8
+
9
+ // Deployment-level gate for the deprecated Document Store feature. When this is false the
10
+ // "documents" worker is not spawned, document-store-only endpoints are not registered, and
11
+ // all runtime document-store code takes its existing "disabled" path. Set via the
12
+ // --documentStore.enabled CLI flag / [documentStore] enabled config or EENGINE_DOCUMENT_STORE_ENABLED.
13
+ const documentStoreFeatureEnabled = hasEnvValue('EENGINE_DOCUMENT_STORE_ENABLED')
14
+ ? getBoolean(readEnvValue('EENGINE_DOCUMENT_STORE_ENABLED'))
15
+ : getBoolean(config.documentStore && config.documentStore.enabled);
16
+
17
+ // Effective runtime state: the feature must be available (gate) AND enabled in settings.
18
+ // When the gate is off this is always false, so callers reuse the already-tested
19
+ // "documentStoreEnabled is false" code paths.
20
+ const isDocumentStoreEnabled = async () => documentStoreFeatureEnabled && !!(await settings.get('documentStoreEnabled'));
6
21
 
7
22
  const clientCache = { version: -1, config: false, client: false, index: false };
8
23
 
9
24
  const getESClient = async logger => {
25
+ // Feature gate is off: behave exactly as a disabled document store. Checked before any
26
+ // Redis access so disabled deployments do not pay a round-trip on every getESClient call.
27
+ if (!documentStoreFeatureEnabled) {
28
+ return clientCache;
29
+ }
30
+
10
31
  const documentStoreVersion = (await settings.get('documentStoreVersion')) || 0;
11
32
  if (clientCache.version === documentStoreVersion) {
12
33
  return clientCache;
@@ -51,4 +72,4 @@ const getESClient = async logger => {
51
72
  return clientCache;
52
73
  };
53
74
 
54
- module.exports = { getESClient };
75
+ module.exports = { getESClient, documentStoreFeatureEnabled, isDocumentStoreEnabled };
@@ -4,6 +4,7 @@ const { threadId: workerThreadId } = require('worker_threads');
4
4
  const crypto = require('crypto');
5
5
  const logger = require('../logger');
6
6
  const settings = require('../settings');
7
+ const { isDocumentStoreEnabled } = require('../document-store');
7
8
  const msgpack = require('msgpack5')();
8
9
  const { templates } = require('../templates');
9
10
  const { Gateway } = require('../gateway');
@@ -1076,7 +1077,7 @@ class BaseClient {
1076
1077
  // Resolve reference and update reference/in-reply-to headers
1077
1078
  if (data.reference && data.reference.message) {
1078
1079
  // Try document store first if enabled
1079
- if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
1080
+ if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
1080
1081
  try {
1081
1082
  referencedMessage = await this.accountObject.getMessage(data.reference.message, {
1082
1083
  documentStore: true,
@@ -1578,7 +1579,7 @@ class BaseClient {
1578
1579
  // Resolve reference and update reference/in-reply-to headers
1579
1580
  if (data.reference && data.reference.message) {
1580
1581
  // Try document store first if enabled
1581
- if (data.reference.documentStore && (await settings.get('documentStoreEnabled'))) {
1582
+ if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
1582
1583
  try {
1583
1584
  referencedMessage = await this.accountObject.getMessage(data.reference.message, {
1584
1585
  documentStore: true,
@@ -257,6 +257,9 @@ class GmailClient extends BaseClient {
257
257
 
258
258
  this.processingHistory = null;
259
259
  this.renewWatchTimer = null;
260
+ // Set once renewWatch() has logged that the linked Pub/Sub app is missing its topic/IAM
261
+ // markers, so the warning is emitted once per connection instead of every renewal cycle.
262
+ this._loggedMissingWatchMarkers = false;
260
263
 
261
264
  this.cachedLabels = null;
262
265
  this.cachedDetailedLabels = null;
@@ -2047,6 +2050,9 @@ class GmailClient extends BaseClient {
2047
2050
  watchExpiration,
2048
2051
  watchFailure: null
2049
2052
  });
2053
+ // Markers are present and the watch armed - allow the missing-markers warning
2054
+ // to fire again if the app's markers ever disappear.
2055
+ this._loggedMissingWatchMarkers = false;
2050
2056
  this.logger.info({
2051
2057
  msg: 'Renewed Gmail pubsub watch',
2052
2058
  account: this.account,
@@ -2067,6 +2073,23 @@ class GmailClient extends BaseClient {
2067
2073
  err
2068
2074
  });
2069
2075
  }
2076
+ } else {
2077
+ // pubSubApp is linked and a renewal is due, but the topic/IAM markers required to
2078
+ // arm the watch are not recorded on the linked Pub/Sub app. This is the silent
2079
+ // failure that leaves Gmail push disabled (e.g. Pub/Sub resources pre-provisioned in
2080
+ // GCP, so ensurePubsub never persisted the markers). Surface it once per connection -
2081
+ // the renewal timer re-fires ~hourly and this branch never updates lastWatch, so an
2082
+ // unguarded warning would otherwise repeat every cycle.
2083
+ if (!this._loggedMissingWatchMarkers) {
2084
+ this._loggedMissingWatchMarkers = true;
2085
+ this.logger.warn({
2086
+ msg: 'Skipping Gmail watch renewal: Pub/Sub app linked but topic/IAM markers are not recorded',
2087
+ account: this.account,
2088
+ pubSubApp: accountData._app?.pubSubApp,
2089
+ hasTopic: !!appData?.pubSubTopic,
2090
+ hasIamPolicy: !!appData?.pubSubIamPolicy
2091
+ });
2092
+ }
2070
2093
  }
2071
2094
  }
2072
2095
  }
@@ -3076,4 +3099,13 @@ class GmailClient extends BaseClient {
3076
3099
  }
3077
3100
  }
3078
3101
 
3079
- module.exports = { GmailClient };
3102
+ // PageCursor and the label maps are exported for unit testing. The maps are
3103
+ // exported as frozen copies so importers (or tests) cannot mutate the live
3104
+ // objects used by the running Gmail client.
3105
+ module.exports = {
3106
+ GmailClient,
3107
+ PageCursor,
3108
+ SKIP_LABELS: Object.freeze([...SKIP_LABELS]),
3109
+ SYSTEM_LABELS: Object.freeze({ ...SYSTEM_LABELS }),
3110
+ SYSTEM_NAMES: Object.freeze({ ...SYSTEM_NAMES })
3111
+ };
@@ -15,7 +15,7 @@ const ical = require('ical.js');
15
15
  const addressparser = require('nodemailer/lib/addressparser');
16
16
  const { llmPreProcess } = require('../../llm-pre-process');
17
17
 
18
- const { getESClient } = require('../../document-store');
18
+ const { getESClient, isDocumentStoreEnabled } = require('../../document-store');
19
19
 
20
20
  const {
21
21
  MESSAGE_NEW_NOTIFY,
@@ -1958,7 +1958,7 @@ class Mailbox {
1958
1958
  async publishSyncedEvents(storedStatus) {
1959
1959
  let messageFetchOptions = {};
1960
1960
 
1961
- let documentStoreEnabled = await settings.get('documentStoreEnabled');
1961
+ let documentStoreEnabled = await isDocumentStoreEnabled();
1962
1962
 
1963
1963
  // Configure text fetching
1964
1964
  let notifyText = await settings.get('notifyText');
@@ -3,7 +3,7 @@
3
3
  const { parentPort } = require('worker_threads');
4
4
  const logger = require('../logger');
5
5
  const { webhooks: Webhooks } = require('../webhooks');
6
- const { getESClient } = require('../document-store');
6
+ const { getESClient, isDocumentStoreEnabled } = require('../document-store');
7
7
  const { getThread } = require('../threads');
8
8
  const settings = require('../settings');
9
9
 
@@ -126,7 +126,7 @@ class NotificationHandler {
126
126
  return false;
127
127
  }
128
128
 
129
- return await settings.get('documentStoreEnabled');
129
+ return await isDocumentStoreEnabled();
130
130
  }
131
131
 
132
132
  /**
package/lib/export.js CHANGED
@@ -99,6 +99,17 @@ function isFolderMissingError(err) {
99
99
  return ['FolderNotFound', 'NotFound'].includes(errCode);
100
100
  }
101
101
 
102
+ // Per-message errors worth retrying during a batch export: rate limits, a dropped batch response
103
+ // (Outlook reports these as EMISSING_RESPONSE), and the same transient network/5xx/timeout errors
104
+ // isTransientError already covers. Used by the export worker's API-account batch fetch path so a
105
+ // single transient blip does not fail an entire multi-message export.
106
+ function isRetryableError(err) {
107
+ if (err.statusCode === 429 || ['rateLimitExceeded', 'userRateLimitExceeded', 'EMISSING_RESPONSE'].includes(err.code)) {
108
+ return true;
109
+ }
110
+ return isTransientError(err);
111
+ }
112
+
102
113
  async function tryAddToActiveSet(account, exportId) {
103
114
  const maxConcurrent = (await settings.get('exportMaxConcurrent')) || DEFAULT_EXPORT_MAX_CONCURRENT;
104
115
  const maxGlobalConcurrent = (await settings.get('exportMaxGlobalConcurrent')) || DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT;
@@ -618,5 +629,6 @@ module.exports = {
618
629
  isTransientError,
619
630
  isSkippableError,
620
631
  isFolderMissingError,
632
+ isRetryableError,
621
633
  DEFAULT_EXPORT_MAX_MESSAGE_SIZE
622
634
  };
@@ -34,6 +34,12 @@ for (let key of Object.keys(process.env)) {
34
34
  module.exports = {
35
35
  enabled(key) {
36
36
  return FEATURES.has(formatFeatureKey(key));
37
+ },
38
+
39
+ // Sorted list of the currently enabled feature flag keys. Used by the license beacon
40
+ // to report which EENGINE_FEATURE_* flags are active on this instance.
41
+ listEnabled() {
42
+ return Array.from(FEATURES).sort();
37
43
  }
38
44
  };
39
45