emailengine-app 2.70.0 → 2.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/.github/workflows/codeql.yml +3 -0
  2. package/.github/workflows/e2e.yml +56 -0
  3. package/.github/workflows/test.yml +81 -12
  4. package/.ncurc.js +20 -20
  5. package/CHANGELOG.md +25 -0
  6. package/Gruntfile.js +19 -23
  7. package/bin/emailengine.js +8 -1
  8. package/config/default.toml +5 -0
  9. package/config/e2e.toml +35 -0
  10. package/config/test.toml +5 -0
  11. package/data/google-crawlers.json +1 -1
  12. package/getswagger.sh +4 -0
  13. package/lib/account.js +31 -25
  14. package/lib/api-routes/message-routes.js +125 -121
  15. package/lib/auth-token.js +83 -0
  16. package/lib/delivery-error.js +62 -0
  17. package/lib/document-store.js +22 -1
  18. package/lib/email-client/base-client.js +3 -2
  19. package/lib/email-client/gmail-client.js +33 -1
  20. package/lib/email-client/imap/mailbox.js +2 -2
  21. package/lib/email-client/notification-handler.js +2 -2
  22. package/lib/export.js +12 -0
  23. package/lib/feature-flags.js +6 -0
  24. package/lib/imap-proxy-auth.js +81 -0
  25. package/lib/imapproxy/imap-server.js +8 -103
  26. package/lib/license-beacon.js +367 -0
  27. package/lib/logger.js +11 -1
  28. package/lib/oauth/gmail.js +3 -0
  29. package/lib/oauth/outlook.js +3 -0
  30. package/lib/oauth2-apps.js +100 -11
  31. package/lib/routes-ui.js +2 -1
  32. package/lib/smtp-auth.js +70 -0
  33. package/lib/sub-script.js +8 -2
  34. package/lib/tools.js +26 -2
  35. package/lib/ui-routes/admin-config-routes.js +4 -3
  36. package/lib/ui-routes/document-store-routes.js +7 -1
  37. package/package.json +30 -24
  38. package/playwright.config.js +45 -0
  39. package/sbom.json +1 -1
  40. package/server.js +30 -8
  41. package/static/licenses.html +108 -128
  42. package/test-coverage-plan.md +233 -0
  43. package/translations/de.mo +0 -0
  44. package/translations/de.po +154 -142
  45. package/translations/et.mo +0 -0
  46. package/translations/et.po +129 -131
  47. package/translations/fr.mo +0 -0
  48. package/translations/fr.po +133 -136
  49. package/translations/ja.mo +0 -0
  50. package/translations/ja.po +126 -129
  51. package/translations/messages.pot +37 -37
  52. package/translations/nl.mo +0 -0
  53. package/translations/nl.po +128 -130
  54. package/translations/pl.mo +0 -0
  55. package/translations/pl.po +125 -128
  56. package/views/dashboard.hbs +22 -0
  57. package/workers/api.js +22 -5
  58. package/workers/export.js +58 -43
  59. package/workers/smtp.js +5 -85
  60. package/workers/submit.js +2 -12
@@ -0,0 +1,81 @@
1
+ 'use strict';
2
+
3
+ // IMAP proxy authentication. Extracted from lib/imapproxy/imap-server.js so the
4
+ // auth decision can be unit tested without booting the proxy worker (which uses
5
+ // a parentPort at require time). Only the authentication portion is extracted;
6
+ // the backend IMAP connection config is still built by the server.
7
+
8
+ const logger = require('./logger');
9
+ const settings = require('./settings');
10
+ const { redis } = require('./db');
11
+ const { Account } = require('./account');
12
+ const getSecret = require('./get-secret');
13
+ const { isApiBasedApp } = require('./oauth2-apps');
14
+ const { validateAuthToken, REASON_MESSAGES } = require('./auth-token');
15
+
16
+ /**
17
+ * Builds the IMAP proxy authentication handler.
18
+ *
19
+ * @param {Object} deps
20
+ * @param {Function} deps.call - RPC function passed to the Account instance
21
+ * @returns {Function} async authenticate(auth, session) -> { accountObject, accountData }
22
+ */
23
+ function createImapProxyAuthHandler({ call }) {
24
+ return async function authenticate(auth, session) {
25
+ let account = auth.username;
26
+
27
+ let imapPassword = await settings.get('imapProxyServerPassword');
28
+ if (!imapPassword || auth.password !== imapPassword) {
29
+ // fall back to API token authentication
30
+ let result = await validateAuthToken({
31
+ password: auth.password,
32
+ account: auth.username,
33
+ requiredScope: 'imap-proxy',
34
+ remoteAddress: session.remoteAddress
35
+ });
36
+
37
+ if (!result.authenticated) {
38
+ let err = new Error(REASON_MESSAGES[result.reason] || 'Access denied, failed to authenticate user');
39
+ err.serverResponseCode = 'AUTHENTICATIONFAILED';
40
+ err.responseStatus = 'NO';
41
+ throw err;
42
+ }
43
+ }
44
+
45
+ let accountObject = new Account({ account, redis, call, secret: await getSecret() });
46
+ let accountData;
47
+ try {
48
+ accountData = await accountObject.loadAccountData();
49
+ } catch (err) {
50
+ let respErr = new Error('Failed to authenticate user');
51
+ respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
52
+ respErr.responseStatus = 'NO';
53
+
54
+ if (!err.output || err.output.statusCode !== 404) {
55
+ // only log non-obvious errors
56
+ logger.error({ msg: 'Failed to load account data', account: auth.username, err });
57
+ }
58
+
59
+ throw respErr;
60
+ }
61
+
62
+ if (isApiBasedApp(accountData?._app)) {
63
+ let respErr = new Error('IMAP is not supported for API-based accounts');
64
+ respErr.authenticationFailed = true;
65
+ respErr.serverResponseCode = 'ACCOUNTDISABLED';
66
+ respErr.responseStatus = 'NO';
67
+ throw respErr;
68
+ }
69
+
70
+ if (!accountData) {
71
+ let err = new Error('Access denied, failed to authenticate user');
72
+ err.serverResponseCode = 'AUTHENTICATIONFAILED';
73
+ err.responseStatus = 'NO';
74
+ throw err;
75
+ }
76
+
77
+ return { accountObject, accountData };
78
+ };
79
+ }
80
+
81
+ module.exports = { createImapProxyAuthHandler };
@@ -4,16 +4,15 @@ const { parentPort } = require('worker_threads');
4
4
 
5
5
  const config = require('@zone-eu/wild-config');
6
6
  const logger = require('../logger');
7
- const { oauth2Apps, oauth2ProviderData, isApiBasedApp } = require('../oauth2-apps');
7
+ const { oauth2Apps, oauth2ProviderData } = require('../oauth2-apps');
8
8
 
9
9
  const { getDuration, getBoolean, resolveCredentials, hasEnvValue, readEnvValue, emitChangeEvent, loadTlsConfig } = require('../tools');
10
- const { matchIp, getLocalAddress } = require('../utils/network');
10
+ const { getLocalAddress } = require('../utils/network');
11
11
 
12
12
  const { redis } = require('../db');
13
- const { Account } = require('../account');
13
+ const { createImapProxyAuthHandler } = require('../imap-proxy-auth');
14
14
  const getSecret = require('../get-secret');
15
15
  const settings = require('../settings');
16
- const tokens = require('../tokens');
17
16
 
18
17
  const { encrypt, decrypt } = require('../encrypt');
19
18
  const { Certs } = require('@postalsys/certs');
@@ -150,108 +149,14 @@ class PassThroughLogger extends PassThrough {
150
149
  }
151
150
  }
152
151
 
152
+ // Authentication logic lives in lib/imap-proxy-auth.js so it can be unit tested
153
+ // without booting this worker. call() is injected for the Account instance.
154
+ const authenticateImapProxy = createImapProxyAuthHandler({ call });
155
+
153
156
  async function onAuth(auth, session) {
154
157
  let account = auth.username;
155
158
 
156
- let imapPassword = await settings.get('imapProxyServerPassword');
157
- let authPass = false;
158
-
159
- if (!imapPassword || auth.password !== imapPassword) {
160
- if (/^[0-9a-f]{64}$/i.test(auth.password)) {
161
- // fallback to tokens
162
- let tokenData;
163
- try {
164
- tokenData = await tokens.get(auth.password, false, { log: true, remoteAddress: session.remoteAddress });
165
- } catch (err) {
166
- logger.error({
167
- msg: 'Failed to fetch token',
168
- err
169
- });
170
- }
171
-
172
- if (tokenData) {
173
- if (tokenData.account && tokenData.account !== auth.username) {
174
- let err = new Error('Access denied, invalid username');
175
- err.serverResponseCode = 'AUTHENTICATIONFAILED';
176
- err.responseStatus = 'NO';
177
- throw err;
178
- }
179
-
180
- if (tokenData.scopes && !tokenData.scopes.includes('imap-proxy') && !tokenData.scopes.includes('*')) {
181
- logger.error({
182
- msg: 'Trying to use invalid scope for a token',
183
- tokenAccount: tokenData.account,
184
- tokenId: tokenData.id,
185
- account,
186
- requestedScope: 'imap-proxy',
187
- scopes: tokenData.scopes
188
- });
189
-
190
- let err = new Error('Access denied, invalid scope');
191
- err.serverResponseCode = 'AUTHENTICATIONFAILED';
192
- err.responseStatus = 'NO';
193
- throw err;
194
- }
195
-
196
- if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(session.remoteAddress, tokenData.restrictions.addresses)) {
197
- logger.error({
198
- msg: 'Trying to use invalid IP for a token',
199
- tokenAccount: tokenData.account,
200
- tokenId: tokenData.id,
201
- account,
202
- remoteAddress: session.remoteAddress,
203
- addressAllowlist: tokenData.restrictions.addresses
204
- });
205
-
206
- let err = new Error('Access denied, traffic not accepted from this IP');
207
- err.serverResponseCode = 'AUTHENTICATIONFAILED';
208
- err.responseStatus = 'NO';
209
- throw err;
210
- }
211
-
212
- authPass = true;
213
- }
214
- }
215
-
216
- if (!authPass) {
217
- let err = new Error('Access denied, failed to authenticate user');
218
- err.serverResponseCode = 'AUTHENTICATIONFAILED';
219
- err.responseStatus = 'NO';
220
- throw err;
221
- }
222
- }
223
-
224
- let accountObject = new Account({ account, redis, call, secret: await getSecret() });
225
- let accountData;
226
- try {
227
- accountData = await accountObject.loadAccountData();
228
- } catch (err) {
229
- let respErr = new Error('Failed to authenticate user');
230
- respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
231
- respErr.responseStatus = 'NO';
232
-
233
- if (!err.output || err.output.statusCode !== 404) {
234
- // only log non-obvious errors
235
- logger.error({ msg: 'Failed to load account data', account: auth.username, err });
236
- }
237
-
238
- throw respErr;
239
- }
240
-
241
- if (isApiBasedApp(accountData?._app)) {
242
- let respErr = new Error('IMAP is not supported for API-based accounts');
243
- respErr.authenticationFailed = true;
244
- respErr.serverResponseCode = 'ACCOUNTDISABLED';
245
- respErr.responseStatus = 'NO';
246
- throw respErr;
247
- }
248
-
249
- if (!accountData) {
250
- let err = new Error('Access denied, failed to authenticate user');
251
- err.serverResponseCode = 'AUTHENTICATIONFAILED';
252
- err.responseStatus = 'NO';
253
- throw err;
254
- }
159
+ let { accountObject, accountData } = await authenticateImapProxy(auth, session);
255
160
 
256
161
  if (!accountData.imap && !accountData.oauth2) {
257
162
  // can not make connection
@@ -0,0 +1,367 @@
1
+ 'use strict';
2
+
3
+ // License-validation feature beacon.
4
+ //
5
+ // Collects a compact, anonymized snapshot of which features are enabled and exercised on this
6
+ // instance, to be piggybacked onto the existing daily license-validation POST. The intent is to
7
+ // learn whether deprecation-candidate features are still in use in the field.
8
+ //
9
+ // Privacy: the snapshot contains only enable-flags, provider type names, coarse magnitude tiers
10
+ // (NOT raw counts), exercised-usage booleans, and runtime context. It never includes account
11
+ // addresses, URLs, credentials, or any other PII/secrets.
12
+ //
13
+ // Reliability: collection is strictly best-effort. Every field is isolated so one failing Redis
14
+ // read degrades that field rather than the whole snapshot, and collectBeacon never throws - on a
15
+ // catastrophic failure it returns null and the license call proceeds with its original fields.
16
+ // The caller is expected to also time-box this with withTimeout().
17
+
18
+ const crypto = require('crypto');
19
+ const msgpack = require('msgpack5')();
20
+
21
+ const settings = require('./settings');
22
+ const { getCounterValues, hasEnvValue, readEnvValue, getBoolean } = require('./tools');
23
+ const { REDIS_PREFIX, EE_DOCKER_LEGACY } = require('./consts');
24
+ const { oauth2Apps } = require('./oauth2-apps');
25
+ const passkeys = require('./passkeys');
26
+ const featureFlags = require('./feature-flags');
27
+ const { documentStoreFeatureEnabled } = require('./document-store');
28
+
29
+ // Beacon schema version. Bump when the meaning of codes changes so the license server can adapt.
30
+ const SCHEMA_VERSION = 1;
31
+
32
+ // Window for "exercised recently" usage signals. A week smooths over quiet days so the digest
33
+ // (and therefore the full-payload sends) does not churn day to day.
34
+ const USE_WINDOW_SECONDS = 7 * 24 * 3600;
35
+
36
+ // Skip the per-route webhook content scan above this many routes (keeps the collector cheap).
37
+ const WH_SCAN_LIMIT = 250;
38
+
39
+ // Time-box for a single collection so a slow Redis can never delay license validation.
40
+ const COLLECT_TIMEOUT_MS = 2000;
41
+
42
+ // Resend the full snapshot at least this often even when its digest has not changed.
43
+ const FULL_RESEND_INTERVAL_MS = 30 * 24 * 3600 * 1000;
44
+
45
+ // Map a raw count to a coarse magnitude tier (powers of ten). Values are buckets, never counts.
46
+ function tier(n) {
47
+ n = Number(n) || 0;
48
+ if (n <= 0) return 0;
49
+ if (n === 1) return 1;
50
+ if (n < 10) return 2;
51
+ if (n < 100) return 3;
52
+ if (n < 1000) return 4;
53
+ if (n < 10000) return 5;
54
+ return 6;
55
+ }
56
+
57
+ // Truthiness for boolean-ish settings (schema booleans arrive as real booleans; legacy/raw values
58
+ // may be strings or arrays).
59
+ function truthy(value) {
60
+ if (value === true) {
61
+ return true;
62
+ }
63
+ if (typeof value === 'number') {
64
+ return value !== 0;
65
+ }
66
+ if (Array.isArray(value)) {
67
+ return value.length > 0;
68
+ }
69
+ if (typeof value === 'string') {
70
+ return /^(y|yes|true|t|1)$/i.test(value.trim());
71
+ }
72
+ return false;
73
+ }
74
+
75
+ // Non-empty check for string-valued settings (URLs, keys, scripts) without inspecting the value.
76
+ function nonEmpty(value) {
77
+ if (typeof value === 'string') {
78
+ return value.trim().length > 0;
79
+ }
80
+ return truthy(value);
81
+ }
82
+
83
+ // Deterministic serialization: object keys sorted recursively. Arrays are pre-sorted at build time.
84
+ // Produces a stable string so the digest only changes when the snapshot meaningfully changes.
85
+ function stableStringify(value) {
86
+ if (Array.isArray(value)) {
87
+ return '[' + value.map(stableStringify).join(',') + ']';
88
+ }
89
+ if (value && typeof value === 'object') {
90
+ return (
91
+ '{' +
92
+ Object.keys(value)
93
+ .sort()
94
+ .map(key => JSON.stringify(key) + ':' + stableStringify(value[key]))
95
+ .join(',') +
96
+ '}'
97
+ );
98
+ }
99
+ return JSON.stringify(value);
100
+ }
101
+
102
+ // Resolve the install channel, mirroring lib/ui-routes/dashboard-routes.js.
103
+ function installChannel() {
104
+ if (getBoolean(readEnvValue('EENGINE_DOCEAN'))) {
105
+ return 'docean';
106
+ }
107
+ if (typeof readEnvValue('RENDER_SERVICE_SLUG') === 'string' && readEnvValue('RENDER_SERVICE_SLUG')) {
108
+ return 'render';
109
+ }
110
+ if (getBoolean(readEnvValue('EENGINE_INSTALL_SCRIPT'))) {
111
+ return 'script';
112
+ }
113
+ if (EE_DOCKER_LEGACY) {
114
+ return 'docker-legacy';
115
+ }
116
+ return 'general';
117
+ }
118
+
119
+ // Race a promise against a timeout so a slow Redis can never delay license validation.
120
+ function withTimeout(promise, ms) {
121
+ return Promise.race([
122
+ promise,
123
+ new Promise((resolve, reject) => {
124
+ setTimeout(() => reject(new Error('Beacon collection timed out')), ms).unref();
125
+ })
126
+ ]);
127
+ }
128
+
129
+ // Build the diagnostic snapshot and its digest. Returns { fh, diag } or null on failure.
130
+ async function collectBeacon({ redis, logger }) {
131
+ // Isolate a single field: log and swallow so one failure does not abort the whole snapshot.
132
+ const safe = async fn => {
133
+ try {
134
+ return await fn();
135
+ } catch (err) {
136
+ if (logger) {
137
+ logger.error({ msg: 'Beacon field collection failed', err });
138
+ }
139
+ return undefined;
140
+ }
141
+ };
142
+
143
+ try {
144
+ const diag = { v: SCHEMA_VERSION };
145
+
146
+ const s =
147
+ (await safe(() =>
148
+ settings.getMulti(
149
+ 'smtpServerEnabled',
150
+ 'imapProxyServerEnabled',
151
+ 'enableApiProxy',
152
+ 'trackOpens',
153
+ 'trackClicks',
154
+ 'webhooksEnabled',
155
+ 'openAiAPIKey',
156
+ 'generateEmailSummary',
157
+ 'openAiGenerateEmbeddings',
158
+ 'openAiAPIUrl',
159
+ 'openAiPreProcessingFn',
160
+ 'proxyEnabled',
161
+ 'httpProxyEnabled',
162
+ 'localAddresses',
163
+ 'sentryEnabled',
164
+ 'authServer',
165
+ 'imapIndexer',
166
+ 'totpEnabled',
167
+ 'documentStoreEnabled',
168
+ 'documentStoreGenerateEmbeddings',
169
+ 'documentStorePreProcessingEnabled',
170
+ 'gmailEnabled',
171
+ 'outlookEnabled',
172
+ 'mailRuEnabled',
173
+ 'trackSentMessages'
174
+ )
175
+ )) || {};
176
+
177
+ const on = key => truthy(s[key]);
178
+
179
+ // Enabled-feature codes (presence = on; codes are omitted when off).
180
+ const feat = [];
181
+ if (on('smtpServerEnabled')) feat.push('smtp');
182
+ if (on('imapProxyServerEnabled')) feat.push('imapproxy');
183
+ if (on('enableApiProxy')) feat.push('apiproxy');
184
+ if (on('trackOpens')) feat.push('track_o');
185
+ if (on('trackClicks')) feat.push('track_c');
186
+ if (on('webhooksEnabled')) feat.push('webhooks');
187
+ if (nonEmpty(s.openAiAPIKey)) feat.push('ai');
188
+ if (on('generateEmailSummary')) feat.push('ai_sum');
189
+ if (on('openAiGenerateEmbeddings')) feat.push('ai_embed');
190
+ if (nonEmpty(s.openAiAPIUrl)) feat.push('ai_url');
191
+ if (nonEmpty(s.openAiPreProcessingFn)) feat.push('ai_prefn');
192
+ if (on('proxyEnabled')) feat.push('proxy');
193
+ if (on('httpProxyEnabled')) feat.push('httpproxy');
194
+ if (truthy(s.localAddresses)) feat.push('localaddr');
195
+ if (on('sentryEnabled')) feat.push('sentry');
196
+ if (nonEmpty(s.authServer)) feat.push('authsrv');
197
+ if (s.imapIndexer === 'fast') feat.push('idx_fast');
198
+ if (on('totpEnabled')) feat.push('totp');
199
+ if (hasEnvValue('OKTA_OAUTH2_ISSUER') && hasEnvValue('OKTA_OAUTH2_CLIENT_ID') && hasEnvValue('OKTA_OAUTH2_CLIENT_SECRET')) {
200
+ feat.push('okta');
201
+ }
202
+ if (await safe(() => passkeys.hasPasskeys())) {
203
+ feat.push('passkey');
204
+ }
205
+ diag.feat = feat.sort();
206
+
207
+ // Entity magnitude tiers (buckets, not counts).
208
+ const scard = key => safe(() => redis.scard(`${REDIS_PREFIX}${key}`));
209
+ const rawAccounts = Number(await scard('ia:accounts')) || 0;
210
+ diag.tiers = {
211
+ acct: tier(rawAccounts),
212
+ oapp: tier(await scard('oapp:i')),
213
+ gw: tier(await scard('gateways')),
214
+ wh: tier(await scard('wh:i')),
215
+ tpl: tier(await scard('tpl::i')),
216
+ bl: tier(await safe(() => redis.hlen(`${REDIS_PREFIX}lists:unsub:lists`)))
217
+ };
218
+
219
+ // Provider mix. `oapp` = provider types of configured OAuth apps; `prov` = provider types
220
+ // that actually have accounts (plus `imap` for any non-OAuth accounts). Only the app id and
221
+ // provider type are read from the sanitized app listing - no secrets are inspected.
222
+ await safe(async () => {
223
+ const res = await oauth2Apps.list(0, 100000);
224
+ const apps = (res && res.apps) || [];
225
+
226
+ const configured = new Set();
227
+ const appProviders = [];
228
+ for (const app of apps) {
229
+ if (app && app.provider) {
230
+ configured.add(app.provider);
231
+ appProviders.push([app.id, app.provider]);
232
+ }
233
+ }
234
+ diag.oapp = Array.from(configured).sort();
235
+
236
+ const inUse = new Set();
237
+ let oauthAccounts = 0;
238
+ if (appProviders.length) {
239
+ const multi = redis.multi();
240
+ for (const [id] of appProviders) {
241
+ multi.scard(`${REDIS_PREFIX}oapp:a:${id}`);
242
+ }
243
+ const counts = await multi.exec();
244
+ for (let i = 0; i < appProviders.length; i++) {
245
+ const entry = counts[i];
246
+ const count = (entry && !entry[0] && Number(entry[1])) || 0;
247
+ oauthAccounts += count;
248
+ if (count > 0) {
249
+ inUse.add(appProviders[i][1]);
250
+ }
251
+ }
252
+ }
253
+ if (rawAccounts > oauthAccounts) {
254
+ inUse.add('imap');
255
+ }
256
+ diag.prov = Array.from(inUse).sort();
257
+ });
258
+
259
+ // Exercised-usage signals from the existing event counters.
260
+ await safe(async () => {
261
+ const counters = (await getCounterValues(redis, USE_WINDOW_SECONDS)) || {};
262
+ const use = [];
263
+ if (counters['events:messageNew'] > 0) use.push('recv');
264
+ if (counters['submit:success'] > 0) use.push('send');
265
+ if (counters['webhooks:success'] > 0) use.push('wh');
266
+ if (counters['apiCall:success'] > 0) use.push('api');
267
+ diag.use = use.sort();
268
+ });
269
+
270
+ // Deprecation watchlist (presence of legacy/candidate-for-removal features).
271
+ const dep = [];
272
+ if (on('documentStoreEnabled')) dep.push('documentStore');
273
+ if (documentStoreFeatureEnabled) dep.push('documentStoreGate');
274
+ if (on('documentStoreGenerateEmbeddings')) dep.push('ds_embed');
275
+ if (on('documentStorePreProcessingEnabled')) dep.push('ds_preproc');
276
+ if (on('gmailEnabled') || on('outlookEnabled') || on('mailRuEnabled')) dep.push('legacyOauth');
277
+ if (on('trackSentMessages')) dep.push('trackSent');
278
+ if (EE_DOCKER_LEGACY) dep.push('dockerLegacy');
279
+ await safe(async () => {
280
+ const ids = await redis.smembers(`${REDIS_PREFIX}wh:i`);
281
+ if (ids && ids.length && ids.length <= WH_SCAN_LIMIT) {
282
+ const bufs = await redis.hmgetBuffer(
283
+ `${REDIS_PREFIX}wh:c`,
284
+ ids.map(id => `${id}:content`)
285
+ );
286
+ for (const buf of bufs || []) {
287
+ if (!buf || !buf.length) {
288
+ continue;
289
+ }
290
+ try {
291
+ const content = msgpack.decode(buf);
292
+ if (content && (content.fn || content.map)) {
293
+ dep.push('whSubscript');
294
+ break;
295
+ }
296
+ } catch (err) {
297
+ // undecodable entry, skip
298
+ }
299
+ }
300
+ }
301
+ });
302
+ diag.dep = dep.sort();
303
+
304
+ // Enabled EENGINE_FEATURE_* flags (already sorted).
305
+ diag.flags = (await safe(() => featureFlags.listEnabled())) || [];
306
+
307
+ // Runtime context.
308
+ diag.dist = installChannel();
309
+ diag.node = process.versions.node;
310
+ diag.arch = process.arch;
311
+
312
+ const fh = crypto.createHash('sha256').update(stableStringify(diag)).digest('hex').slice(0, 12);
313
+
314
+ return { fh, diag };
315
+ } catch (err) {
316
+ if (logger) {
317
+ logger.error({ msg: 'Beacon collection failed', err });
318
+ }
319
+ return null;
320
+ }
321
+ }
322
+
323
+ // Collect the snapshot (time-boxed) and decide what to attach to the license request body.
324
+ // Always attaches the digest `fh`; attaches the full `diag` only when the digest changed since the
325
+ // last accepted send or the 30-day heartbeat is due. Best-effort: never throws.
326
+ async function attachBeacon(body, { redis, logger, now }) {
327
+ try {
328
+ const beacon = await withTimeout(collectBeacon({ redis, logger }), COLLECT_TIMEOUT_MS);
329
+ if (!beacon || !beacon.fh) {
330
+ return;
331
+ }
332
+ body.fh = beacon.fh;
333
+
334
+ const [storedHash, bft] = await redis.hmget(`${REDIS_PREFIX}settings`, ['bfh', 'bft']);
335
+ const lastFull = parseInt(bft || '0', 16) || 0;
336
+ if (beacon.fh !== storedHash || now - lastFull > FULL_RESEND_INTERVAL_MS) {
337
+ body.diag = beacon.diag;
338
+ }
339
+ } catch (err) {
340
+ if (logger) {
341
+ logger.error({ msg: 'License beacon collection failed', err });
342
+ }
343
+ }
344
+ }
345
+
346
+ // Persist the send-on-change markers after a successful validation. `needFull` (from the server)
347
+ // forces a full resend on the next cycle when the server has the digest but not the snapshot.
348
+ // Best-effort: never throws.
349
+ async function persistBeaconMarkers({ redis, logger, body, now, needFull }) {
350
+ try {
351
+ if (body.fh) {
352
+ await redis.hset(`${REDIS_PREFIX}settings`, 'bfh', body.fh);
353
+ if (body.diag) {
354
+ await redis.hset(`${REDIS_PREFIX}settings`, 'bft', now.toString(16));
355
+ }
356
+ }
357
+ if (needFull) {
358
+ await redis.hdel(`${REDIS_PREFIX}settings`, 'bfh');
359
+ }
360
+ } catch (err) {
361
+ if (logger) {
362
+ logger.error({ msg: 'Failed to persist license beacon markers', err });
363
+ }
364
+ }
365
+ }
366
+
367
+ module.exports = { collectBeacon, attachBeacon, persistBeaconMarkers, withTimeout, tier, stableStringify };
package/lib/logger.js CHANGED
@@ -7,6 +7,7 @@ if (!process.env.EE_ENV_LOADED) {
7
7
 
8
8
  const config = require('@zone-eu/wild-config');
9
9
  const pino = require('pino');
10
+ const { TRANSIENT_NETWORK_CODES } = require('./consts');
10
11
 
11
12
  config.log = config.log || {
12
13
  level: 'trace'
@@ -14,10 +15,19 @@ config.log = config.log || {
14
15
 
15
16
  config.log.level = config.log.level || 'trace';
16
17
 
18
+ // undici raises every connection failure as a generic `TypeError` (e.g. "fetch failed"
19
+ // or "terminated") with the real DNS/socket error attached as err.cause. Those are
20
+ // transient, environmental blips - not code bugs - so they must not be forwarded to
21
+ // error tracking, where they pile up as useless "fetch failed" reports. Genuine
22
+ // TypeError/RangeError bugs have no network errno cause and still get reported.
23
+ function isTransientFetchError(err) {
24
+ return err && err.name === 'TypeError' && err.cause && TRANSIENT_NETWORK_CODES.has(err.cause.code);
25
+ }
26
+
17
27
  let logger = pino({
18
28
  formatters: {
19
29
  log(object) {
20
- if (object.err && ['TypeError', 'RangeError'].includes(object.err.name)) {
30
+ if (object.err && ['TypeError', 'RangeError'].includes(object.err.name) && !isTransientFetchError(object.err)) {
21
31
  if (logger.notifyError) {
22
32
  let meta = {};
23
33
  for (let key of ['msg', 'path', 'cid']) {
@@ -824,6 +824,9 @@ class GmailOauth {
824
824
  }
825
825
 
826
826
  module.exports.GmailOauth = GmailOauth;
827
+ // Exported for unit testing the auth error-to-flag mapping.
828
+ module.exports.checkForFlags = checkForFlags;
829
+ module.exports.checkForUserFlags = checkForUserFlags;
827
830
  module.exports.GMAIL_SCOPES = GMAIL_SCOPES;
828
831
  module.exports.GMAIL_API_SCOPES = GMAIL_API_SCOPES;
829
832
  module.exports.OPENID_SCOPES = OPENID_SCOPES;
@@ -701,3 +701,6 @@ class OutlookOauth {
701
701
  module.exports.OutlookOauth = OutlookOauth;
702
702
  module.exports.outlookScopes = outlookScopes;
703
703
  module.exports.OUTLOOK_API_SCOPES = OUTLOOK_API_SCOPES;
704
+ // Exported for unit testing the auth error-to-flag mapping.
705
+ module.exports.checkForFlags = checkForFlags;
706
+ module.exports.checkForUserFlags = checkForUserFlags;