emailengine-app 2.69.0 → 2.71.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.github/workflows/deploy.yml +6 -3
  2. package/.github/workflows/release.yaml +2 -0
  3. package/.github/workflows/test.yml +73 -12
  4. package/.ncurc.js +3 -3
  5. package/CHANGELOG.md +37 -0
  6. package/Gruntfile.js +21 -23
  7. package/bin/emailengine.js +8 -1
  8. package/config/default.toml +5 -0
  9. package/config/test.toml +5 -0
  10. package/data/google-crawlers.json +1 -1
  11. package/getswagger.sh +44 -4
  12. package/gettext-extract.js +163 -0
  13. package/lib/account.js +104 -72
  14. package/lib/api-routes/account-routes.js +231 -71
  15. package/lib/api-routes/blocklist-routes.js +25 -18
  16. package/lib/api-routes/chat-routes.js +32 -14
  17. package/lib/api-routes/delivery-test-routes.js +30 -5
  18. package/lib/api-routes/export-routes.js +27 -2
  19. package/lib/api-routes/gateway-routes.js +63 -12
  20. package/lib/api-routes/license-routes.js +18 -4
  21. package/lib/api-routes/mailbox-routes.js +33 -7
  22. package/lib/api-routes/message-routes.js +291 -145
  23. package/lib/api-routes/oauth2-app-routes.js +90 -24
  24. package/lib/api-routes/outbox-routes.js +16 -4
  25. package/lib/api-routes/pubsub-routes.js +8 -4
  26. package/lib/api-routes/route-helpers.js +14 -1
  27. package/lib/api-routes/settings-routes.js +51 -25
  28. package/lib/api-routes/stats-routes.js +37 -3
  29. package/lib/api-routes/submit-routes.js +31 -42
  30. package/lib/api-routes/template-routes.js +54 -21
  31. package/lib/api-routes/token-routes.js +67 -67
  32. package/lib/api-routes/webhook-route-routes.js +37 -8
  33. package/lib/autodetect-imap-settings.js +0 -2
  34. package/lib/consts.js +5 -0
  35. package/lib/document-store.js +22 -1
  36. package/lib/email-client/base-client.js +31 -8
  37. package/lib/email-client/gmail-client.js +119 -112
  38. package/lib/email-client/imap/mailbox.js +2 -2
  39. package/lib/email-client/imap/subconnection.js +0 -1
  40. package/lib/email-client/imap/sync-operations.js +1 -1
  41. package/lib/email-client/imap-client.js +36 -17
  42. package/lib/email-client/notification-handler.js +3 -6
  43. package/lib/email-client/outlook-client.js +49 -62
  44. package/lib/export.js +49 -1
  45. package/lib/feature-flags.js +8 -2
  46. package/lib/gateway.js +4 -9
  47. package/lib/get-raw-email.js +5 -5
  48. package/lib/imapproxy/imap-core/lib/imap-connection.js +0 -1
  49. package/lib/license-beacon.js +367 -0
  50. package/lib/logger.js +35 -22
  51. package/lib/metrics-collector.js +0 -2
  52. package/lib/oauth2-apps.js +13 -4
  53. package/lib/outbox.js +24 -40
  54. package/lib/redis-operations.js +1 -1
  55. package/lib/routes-ui.js +2 -1
  56. package/lib/schemas.js +403 -83
  57. package/lib/sentry.js +139 -0
  58. package/lib/settings.js +9 -3
  59. package/lib/stream-encrypt.js +1 -1
  60. package/lib/templates.js +1 -1
  61. package/lib/tokens.js +5 -3
  62. package/lib/tools.js +28 -6
  63. package/lib/ui-routes/account-routes.js +7 -4
  64. package/lib/ui-routes/admin-config-routes.js +20 -6
  65. package/lib/ui-routes/document-store-routes.js +7 -1
  66. package/lib/ui-routes/oauth-config-routes.js +0 -2
  67. package/lib/ui-routes/route-helpers.js +0 -2
  68. package/lib/ui-routes/unsubscribe-routes.js +0 -2
  69. package/lib/webhooks.js +8 -4
  70. package/package.json +23 -19
  71. package/sbom.json +1 -1
  72. package/server.js +38 -31
  73. package/static/licenses.html +171 -391
  74. package/translations/de.mo +0 -0
  75. package/translations/de.po +154 -142
  76. package/translations/et.mo +0 -0
  77. package/translations/et.po +129 -131
  78. package/translations/fr.mo +0 -0
  79. package/translations/fr.po +133 -136
  80. package/translations/ja.mo +0 -0
  81. package/translations/ja.po +126 -129
  82. package/translations/messages.pot +107 -107
  83. package/translations/nl.mo +0 -0
  84. package/translations/nl.po +128 -130
  85. package/translations/pl.mo +0 -0
  86. package/translations/pl.po +125 -128
  87. package/update-info.sh +19 -1
  88. package/views/config/logging.hbs +48 -0
  89. package/views/dashboard.hbs +22 -0
  90. package/workers/api.js +33 -37
  91. package/workers/documents.js +2 -22
  92. package/workers/export.js +73 -92
  93. package/workers/imap-proxy.js +3 -23
  94. package/workers/imap.js +2 -22
  95. package/workers/smtp.js +2 -22
  96. package/workers/submit.js +6 -24
  97. package/workers/webhooks.js +2 -22
package/server.js CHANGED
@@ -107,29 +107,9 @@ const bounceClassifier = require('@postalsys/bounce-classifier');
107
107
 
108
108
  const v8 = require('node:v8');
109
109
 
110
- // Initialize Bugsnag error tracking if API key is provided
111
- const Bugsnag = require('@bugsnag/js');
112
- if (readEnvValue('BUGSNAG_API_KEY')) {
113
- Bugsnag.start({
114
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
115
- appVersion: packageData.version,
116
- logger: {
117
- debug(...args) {
118
- logger.debug({ msg: args.shift(), worker: 'main', source: 'bugsnag', args: args.length ? args : undefined });
119
- },
120
- info(...args) {
121
- logger.debug({ msg: args.shift(), worker: 'main', source: 'bugsnag', args: args.length ? args : undefined });
122
- },
123
- warn(...args) {
124
- logger.warn({ msg: args.shift(), worker: 'main', source: 'bugsnag', args: args.length ? args : undefined });
125
- },
126
- error(...args) {
127
- logger.error({ msg: args.shift(), worker: 'main', source: 'bugsnag', args: args.length ? args : undefined });
128
- }
129
- }
130
- });
131
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
132
- }
110
+ // Initialize Sentry error tracking if a DSN is provided
111
+ const { initSentry } = require('./lib/sentry');
112
+ initSentry('main');
133
113
 
134
114
  // Import additional dependencies
135
115
  const pathlib = require('path');
@@ -141,6 +121,8 @@ const { compare: cv } = require('compare-versions');
141
121
  const Joi = require('joi');
142
122
  const { settingsSchema } = require('./lib/schemas');
143
123
  const settings = require('./lib/settings');
124
+ const { documentStoreFeatureEnabled } = require('./lib/document-store');
125
+ const { attachBeacon, persistBeaconMarkers } = require('./lib/license-beacon');
144
126
  const tokens = require('./lib/tokens');
145
127
 
146
128
  const { checkRateLimit } = require('./lib/rate-limit');
@@ -263,6 +245,9 @@ const API_PROXY = hasEnvValue('EENGINE_API_PROXY') ? getBoolean(readEnvValue('EE
263
245
  // API authentication requirement configuration (default: true)
264
246
  const REQUIRE_API_AUTH = hasEnvValue('EENGINE_REQUIRE_API_AUTH') ? getBoolean(readEnvValue('EENGINE_REQUIRE_API_AUTH')) : null;
265
247
 
248
+ // Opt-out for the license-validation feature beacon (telemetry rides on the existing license call)
249
+ const BEACON_DISABLED = hasEnvValue('EENGINE_BEACON_DISABLED') ? getBoolean(readEnvValue('EENGINE_BEACON_DISABLED')) : false;
250
+
266
251
  // OAuth2 token access configuration
267
252
  const ENABLE_OAUTH_TOKENS_API = hasEnvValue('EENGINE_ENABLE_OAUTH_TOKENS_API') ? getBoolean(readEnvValue('EENGINE_ENABLE_OAUTH_TOKENS_API')) : null;
268
253
 
@@ -1877,6 +1862,21 @@ let licenseCheckHandler = async opts => {
1877
1862
  (await redis.hUpdateBigger(`${REDIS_PREFIX}settings`, 'subcheck', now - subscriptionCheckTimeout, now))
1878
1863
  ) {
1879
1864
  try {
1865
+ let body = {
1866
+ key: licenseInfo.details.key,
1867
+ version: packageData.version,
1868
+ app: '@postalsys/emailengine-app',
1869
+ instance: (await settings.get('serviceId')) || ''
1870
+ };
1871
+
1872
+ // Best-effort feature beacon: enrich the existing call with a compact, anonymized
1873
+ // feature snapshot. Time-boxed and fully isolated (attachBeacon never throws) so it
1874
+ // can never block, delay, or alter license validation. The full snapshot is only sent
1875
+ // when its digest changes or once every 30 days; otherwise just the digest rides along.
1876
+ if (!BEACON_DISABLED) {
1877
+ await attachBeacon(body, { redis, logger, now });
1878
+ }
1879
+
1880
1880
  // Call license validation API
1881
1881
  let res = await fetchCmd(`https://postalsys.com/licenses/validate`, {
1882
1882
  method: 'post',
@@ -1884,12 +1884,7 @@ let licenseCheckHandler = async opts => {
1884
1884
  'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`,
1885
1885
  'Content-Type': 'application/json'
1886
1886
  },
1887
- body: JSON.stringify({
1888
- key: licenseInfo.details.key,
1889
- version: packageData.version,
1890
- app: '@postalsys/emailengine-app',
1891
- instance: (await settings.get('serviceId')) || ''
1892
- }),
1887
+ body: JSON.stringify(body),
1893
1888
  dispatcher: httpAgent.retry
1894
1889
  });
1895
1890
 
@@ -1921,6 +1916,11 @@ let licenseCheckHandler = async opts => {
1921
1916
  let nextCheck = Math.min(now + MAX_LICENSE_CHECK_DELAY, validatedUntil.getTime());
1922
1917
  await redis.hset(`${REDIS_PREFIX}settings`, 'ks', new Date(nextCheck).getTime().toString(16));
1923
1918
  }
1919
+
1920
+ // Persist beacon markers so the full snapshot is only resent when it changes.
1921
+ if (!BEACON_DISABLED) {
1922
+ await persistBeaconMarkers({ redis, logger, body, now, needFull: data.needFull });
1923
+ }
1924
1924
  }
1925
1925
  } catch (err) {
1926
1926
  logger.error({ msg: 'License validation error', err });
@@ -1970,6 +1970,11 @@ let licenseCheckHandler = async opts => {
1970
1970
  }
1971
1971
  }
1972
1972
  }
1973
+
1974
+ // Workers were respawned after license activation. Assign any accounts that
1975
+ // accumulated in the unassigned set while workers were suspended, as the
1976
+ // worker ready handler only reassigns after crashes (reassignmentPending)
1977
+ assignAccounts().catch(err => logger.error({ msg: 'Unable to assign accounts after license activation', err }));
1973
1978
  }
1974
1979
  } finally {
1975
1980
  checkingLicense = false;
@@ -3433,8 +3438,10 @@ const startApplication = async () => {
3433
3438
  await spawnWorker('export');
3434
3439
  }
3435
3440
 
3436
- // Start document processing worker
3437
- await spawnWorker('documents');
3441
+ // Start document processing worker (deprecated Document Store feature; only when enabled)
3442
+ if (documentStoreFeatureEnabled) {
3443
+ await spawnWorker('documents');
3444
+ }
3438
3445
 
3439
3446
  // Start SMTP proxy if enabled
3440
3447
  if (await settings.get('smtpServerEnabled')) {