emailengine-app 2.70.0 → 2.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/codeql.yml +3 -0
- package/.github/workflows/e2e.yml +56 -0
- package/.github/workflows/test.yml +81 -12
- package/.ncurc.js +20 -20
- package/CHANGELOG.md +25 -0
- package/Gruntfile.js +19 -23
- package/bin/emailengine.js +8 -1
- package/config/default.toml +5 -0
- package/config/e2e.toml +35 -0
- package/config/test.toml +5 -0
- package/data/google-crawlers.json +1 -1
- package/getswagger.sh +4 -0
- package/lib/account.js +31 -25
- package/lib/api-routes/message-routes.js +125 -121
- package/lib/auth-token.js +83 -0
- package/lib/delivery-error.js +62 -0
- package/lib/document-store.js +22 -1
- package/lib/email-client/base-client.js +3 -2
- package/lib/email-client/gmail-client.js +33 -1
- package/lib/email-client/imap/mailbox.js +2 -2
- package/lib/email-client/notification-handler.js +2 -2
- package/lib/export.js +12 -0
- package/lib/feature-flags.js +6 -0
- package/lib/imap-proxy-auth.js +81 -0
- package/lib/imapproxy/imap-server.js +8 -103
- package/lib/license-beacon.js +367 -0
- package/lib/logger.js +11 -1
- package/lib/oauth/gmail.js +3 -0
- package/lib/oauth/outlook.js +3 -0
- package/lib/oauth2-apps.js +100 -11
- package/lib/routes-ui.js +2 -1
- package/lib/smtp-auth.js +70 -0
- package/lib/sub-script.js +8 -2
- package/lib/tools.js +26 -2
- package/lib/ui-routes/admin-config-routes.js +4 -3
- package/lib/ui-routes/document-store-routes.js +7 -1
- package/package.json +30 -24
- package/playwright.config.js +45 -0
- package/sbom.json +1 -1
- package/server.js +30 -8
- package/static/licenses.html +108 -128
- package/test-coverage-plan.md +233 -0
- package/translations/de.mo +0 -0
- package/translations/de.po +154 -142
- package/translations/et.mo +0 -0
- package/translations/et.po +129 -131
- package/translations/fr.mo +0 -0
- package/translations/fr.po +133 -136
- package/translations/ja.mo +0 -0
- package/translations/ja.po +126 -129
- package/translations/messages.pot +37 -37
- package/translations/nl.mo +0 -0
- package/translations/nl.po +128 -130
- package/translations/pl.mo +0 -0
- package/translations/pl.po +125 -128
- package/views/dashboard.hbs +22 -0
- package/workers/api.js +22 -5
- package/workers/export.js +58 -43
- package/workers/smtp.js +5 -85
- package/workers/submit.js +2 -12
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// IMAP proxy authentication. Extracted from lib/imapproxy/imap-server.js so the
|
|
4
|
+
// auth decision can be unit tested without booting the proxy worker (which uses
|
|
5
|
+
// a parentPort at require time). Only the authentication portion is extracted;
|
|
6
|
+
// the backend IMAP connection config is still built by the server.
|
|
7
|
+
|
|
8
|
+
const logger = require('./logger');
|
|
9
|
+
const settings = require('./settings');
|
|
10
|
+
const { redis } = require('./db');
|
|
11
|
+
const { Account } = require('./account');
|
|
12
|
+
const getSecret = require('./get-secret');
|
|
13
|
+
const { isApiBasedApp } = require('./oauth2-apps');
|
|
14
|
+
const { validateAuthToken, REASON_MESSAGES } = require('./auth-token');
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Builds the IMAP proxy authentication handler.
|
|
18
|
+
*
|
|
19
|
+
* @param {Object} deps
|
|
20
|
+
* @param {Function} deps.call - RPC function passed to the Account instance
|
|
21
|
+
* @returns {Function} async authenticate(auth, session) -> { accountObject, accountData }
|
|
22
|
+
*/
|
|
23
|
+
function createImapProxyAuthHandler({ call }) {
|
|
24
|
+
return async function authenticate(auth, session) {
|
|
25
|
+
let account = auth.username;
|
|
26
|
+
|
|
27
|
+
let imapPassword = await settings.get('imapProxyServerPassword');
|
|
28
|
+
if (!imapPassword || auth.password !== imapPassword) {
|
|
29
|
+
// fall back to API token authentication
|
|
30
|
+
let result = await validateAuthToken({
|
|
31
|
+
password: auth.password,
|
|
32
|
+
account: auth.username,
|
|
33
|
+
requiredScope: 'imap-proxy',
|
|
34
|
+
remoteAddress: session.remoteAddress
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
if (!result.authenticated) {
|
|
38
|
+
let err = new Error(REASON_MESSAGES[result.reason] || 'Access denied, failed to authenticate user');
|
|
39
|
+
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
40
|
+
err.responseStatus = 'NO';
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
let accountObject = new Account({ account, redis, call, secret: await getSecret() });
|
|
46
|
+
let accountData;
|
|
47
|
+
try {
|
|
48
|
+
accountData = await accountObject.loadAccountData();
|
|
49
|
+
} catch (err) {
|
|
50
|
+
let respErr = new Error('Failed to authenticate user');
|
|
51
|
+
respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
52
|
+
respErr.responseStatus = 'NO';
|
|
53
|
+
|
|
54
|
+
if (!err.output || err.output.statusCode !== 404) {
|
|
55
|
+
// only log non-obvious errors
|
|
56
|
+
logger.error({ msg: 'Failed to load account data', account: auth.username, err });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
throw respErr;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (isApiBasedApp(accountData?._app)) {
|
|
63
|
+
let respErr = new Error('IMAP is not supported for API-based accounts');
|
|
64
|
+
respErr.authenticationFailed = true;
|
|
65
|
+
respErr.serverResponseCode = 'ACCOUNTDISABLED';
|
|
66
|
+
respErr.responseStatus = 'NO';
|
|
67
|
+
throw respErr;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!accountData) {
|
|
71
|
+
let err = new Error('Access denied, failed to authenticate user');
|
|
72
|
+
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
73
|
+
err.responseStatus = 'NO';
|
|
74
|
+
throw err;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return { accountObject, accountData };
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { createImapProxyAuthHandler };
|
|
@@ -4,16 +4,15 @@ const { parentPort } = require('worker_threads');
|
|
|
4
4
|
|
|
5
5
|
const config = require('@zone-eu/wild-config');
|
|
6
6
|
const logger = require('../logger');
|
|
7
|
-
const { oauth2Apps, oauth2ProviderData
|
|
7
|
+
const { oauth2Apps, oauth2ProviderData } = require('../oauth2-apps');
|
|
8
8
|
|
|
9
9
|
const { getDuration, getBoolean, resolveCredentials, hasEnvValue, readEnvValue, emitChangeEvent, loadTlsConfig } = require('../tools');
|
|
10
|
-
const {
|
|
10
|
+
const { getLocalAddress } = require('../utils/network');
|
|
11
11
|
|
|
12
12
|
const { redis } = require('../db');
|
|
13
|
-
const {
|
|
13
|
+
const { createImapProxyAuthHandler } = require('../imap-proxy-auth');
|
|
14
14
|
const getSecret = require('../get-secret');
|
|
15
15
|
const settings = require('../settings');
|
|
16
|
-
const tokens = require('../tokens');
|
|
17
16
|
|
|
18
17
|
const { encrypt, decrypt } = require('../encrypt');
|
|
19
18
|
const { Certs } = require('@postalsys/certs');
|
|
@@ -150,108 +149,14 @@ class PassThroughLogger extends PassThrough {
|
|
|
150
149
|
}
|
|
151
150
|
}
|
|
152
151
|
|
|
152
|
+
// Authentication logic lives in lib/imap-proxy-auth.js so it can be unit tested
|
|
153
|
+
// without booting this worker. call() is injected for the Account instance.
|
|
154
|
+
const authenticateImapProxy = createImapProxyAuthHandler({ call });
|
|
155
|
+
|
|
153
156
|
async function onAuth(auth, session) {
|
|
154
157
|
let account = auth.username;
|
|
155
158
|
|
|
156
|
-
let
|
|
157
|
-
let authPass = false;
|
|
158
|
-
|
|
159
|
-
if (!imapPassword || auth.password !== imapPassword) {
|
|
160
|
-
if (/^[0-9a-f]{64}$/i.test(auth.password)) {
|
|
161
|
-
// fallback to tokens
|
|
162
|
-
let tokenData;
|
|
163
|
-
try {
|
|
164
|
-
tokenData = await tokens.get(auth.password, false, { log: true, remoteAddress: session.remoteAddress });
|
|
165
|
-
} catch (err) {
|
|
166
|
-
logger.error({
|
|
167
|
-
msg: 'Failed to fetch token',
|
|
168
|
-
err
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
if (tokenData) {
|
|
173
|
-
if (tokenData.account && tokenData.account !== auth.username) {
|
|
174
|
-
let err = new Error('Access denied, invalid username');
|
|
175
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
176
|
-
err.responseStatus = 'NO';
|
|
177
|
-
throw err;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
if (tokenData.scopes && !tokenData.scopes.includes('imap-proxy') && !tokenData.scopes.includes('*')) {
|
|
181
|
-
logger.error({
|
|
182
|
-
msg: 'Trying to use invalid scope for a token',
|
|
183
|
-
tokenAccount: tokenData.account,
|
|
184
|
-
tokenId: tokenData.id,
|
|
185
|
-
account,
|
|
186
|
-
requestedScope: 'imap-proxy',
|
|
187
|
-
scopes: tokenData.scopes
|
|
188
|
-
});
|
|
189
|
-
|
|
190
|
-
let err = new Error('Access denied, invalid scope');
|
|
191
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
192
|
-
err.responseStatus = 'NO';
|
|
193
|
-
throw err;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(session.remoteAddress, tokenData.restrictions.addresses)) {
|
|
197
|
-
logger.error({
|
|
198
|
-
msg: 'Trying to use invalid IP for a token',
|
|
199
|
-
tokenAccount: tokenData.account,
|
|
200
|
-
tokenId: tokenData.id,
|
|
201
|
-
account,
|
|
202
|
-
remoteAddress: session.remoteAddress,
|
|
203
|
-
addressAllowlist: tokenData.restrictions.addresses
|
|
204
|
-
});
|
|
205
|
-
|
|
206
|
-
let err = new Error('Access denied, traffic not accepted from this IP');
|
|
207
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
208
|
-
err.responseStatus = 'NO';
|
|
209
|
-
throw err;
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
authPass = true;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
if (!authPass) {
|
|
217
|
-
let err = new Error('Access denied, failed to authenticate user');
|
|
218
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
219
|
-
err.responseStatus = 'NO';
|
|
220
|
-
throw err;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
let accountObject = new Account({ account, redis, call, secret: await getSecret() });
|
|
225
|
-
let accountData;
|
|
226
|
-
try {
|
|
227
|
-
accountData = await accountObject.loadAccountData();
|
|
228
|
-
} catch (err) {
|
|
229
|
-
let respErr = new Error('Failed to authenticate user');
|
|
230
|
-
respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
231
|
-
respErr.responseStatus = 'NO';
|
|
232
|
-
|
|
233
|
-
if (!err.output || err.output.statusCode !== 404) {
|
|
234
|
-
// only log non-obvious errors
|
|
235
|
-
logger.error({ msg: 'Failed to load account data', account: auth.username, err });
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
throw respErr;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
if (isApiBasedApp(accountData?._app)) {
|
|
242
|
-
let respErr = new Error('IMAP is not supported for API-based accounts');
|
|
243
|
-
respErr.authenticationFailed = true;
|
|
244
|
-
respErr.serverResponseCode = 'ACCOUNTDISABLED';
|
|
245
|
-
respErr.responseStatus = 'NO';
|
|
246
|
-
throw respErr;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
if (!accountData) {
|
|
250
|
-
let err = new Error('Access denied, failed to authenticate user');
|
|
251
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
252
|
-
err.responseStatus = 'NO';
|
|
253
|
-
throw err;
|
|
254
|
-
}
|
|
159
|
+
let { accountObject, accountData } = await authenticateImapProxy(auth, session);
|
|
255
160
|
|
|
256
161
|
if (!accountData.imap && !accountData.oauth2) {
|
|
257
162
|
// can not make connection
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// License-validation feature beacon.
|
|
4
|
+
//
|
|
5
|
+
// Collects a compact, anonymized snapshot of which features are enabled and exercised on this
|
|
6
|
+
// instance, to be piggybacked onto the existing daily license-validation POST. The intent is to
|
|
7
|
+
// learn whether deprecation-candidate features are still in use in the field.
|
|
8
|
+
//
|
|
9
|
+
// Privacy: the snapshot contains only enable-flags, provider type names, coarse magnitude tiers
|
|
10
|
+
// (NOT raw counts), exercised-usage booleans, and runtime context. It never includes account
|
|
11
|
+
// addresses, URLs, credentials, or any other PII/secrets.
|
|
12
|
+
//
|
|
13
|
+
// Reliability: collection is strictly best-effort. Every field is isolated so one failing Redis
|
|
14
|
+
// read degrades that field rather than the whole snapshot, and collectBeacon never throws - on a
|
|
15
|
+
// catastrophic failure it returns null and the license call proceeds with its original fields.
|
|
16
|
+
// The caller is expected to also time-box this with withTimeout().
|
|
17
|
+
|
|
18
|
+
const crypto = require('crypto');
|
|
19
|
+
const msgpack = require('msgpack5')();
|
|
20
|
+
|
|
21
|
+
const settings = require('./settings');
|
|
22
|
+
const { getCounterValues, hasEnvValue, readEnvValue, getBoolean } = require('./tools');
|
|
23
|
+
const { REDIS_PREFIX, EE_DOCKER_LEGACY } = require('./consts');
|
|
24
|
+
const { oauth2Apps } = require('./oauth2-apps');
|
|
25
|
+
const passkeys = require('./passkeys');
|
|
26
|
+
const featureFlags = require('./feature-flags');
|
|
27
|
+
const { documentStoreFeatureEnabled } = require('./document-store');
|
|
28
|
+
|
|
29
|
+
// Beacon schema version. Bump when the meaning of codes changes so the license server can adapt.
|
|
30
|
+
const SCHEMA_VERSION = 1;
|
|
31
|
+
|
|
32
|
+
// Window for "exercised recently" usage signals. A week smooths over quiet days so the digest
|
|
33
|
+
// (and therefore the full-payload sends) does not churn day to day.
|
|
34
|
+
const USE_WINDOW_SECONDS = 7 * 24 * 3600;
|
|
35
|
+
|
|
36
|
+
// Skip the per-route webhook content scan above this many routes (keeps the collector cheap).
|
|
37
|
+
const WH_SCAN_LIMIT = 250;
|
|
38
|
+
|
|
39
|
+
// Time-box for a single collection so a slow Redis can never delay license validation.
|
|
40
|
+
const COLLECT_TIMEOUT_MS = 2000;
|
|
41
|
+
|
|
42
|
+
// Resend the full snapshot at least this often even when its digest has not changed.
|
|
43
|
+
const FULL_RESEND_INTERVAL_MS = 30 * 24 * 3600 * 1000;
|
|
44
|
+
|
|
45
|
+
// Map a raw count to a coarse magnitude tier (powers of ten). Values are buckets, never counts.
|
|
46
|
+
function tier(n) {
|
|
47
|
+
n = Number(n) || 0;
|
|
48
|
+
if (n <= 0) return 0;
|
|
49
|
+
if (n === 1) return 1;
|
|
50
|
+
if (n < 10) return 2;
|
|
51
|
+
if (n < 100) return 3;
|
|
52
|
+
if (n < 1000) return 4;
|
|
53
|
+
if (n < 10000) return 5;
|
|
54
|
+
return 6;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Truthiness for boolean-ish settings (schema booleans arrive as real booleans; legacy/raw values
|
|
58
|
+
// may be strings or arrays).
|
|
59
|
+
function truthy(value) {
|
|
60
|
+
if (value === true) {
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
if (typeof value === 'number') {
|
|
64
|
+
return value !== 0;
|
|
65
|
+
}
|
|
66
|
+
if (Array.isArray(value)) {
|
|
67
|
+
return value.length > 0;
|
|
68
|
+
}
|
|
69
|
+
if (typeof value === 'string') {
|
|
70
|
+
return /^(y|yes|true|t|1)$/i.test(value.trim());
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Non-empty check for string-valued settings (URLs, keys, scripts) without inspecting the value.
|
|
76
|
+
function nonEmpty(value) {
|
|
77
|
+
if (typeof value === 'string') {
|
|
78
|
+
return value.trim().length > 0;
|
|
79
|
+
}
|
|
80
|
+
return truthy(value);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Deterministic serialization: object keys sorted recursively. Arrays are pre-sorted at build time.
|
|
84
|
+
// Produces a stable string so the digest only changes when the snapshot meaningfully changes.
|
|
85
|
+
function stableStringify(value) {
|
|
86
|
+
if (Array.isArray(value)) {
|
|
87
|
+
return '[' + value.map(stableStringify).join(',') + ']';
|
|
88
|
+
}
|
|
89
|
+
if (value && typeof value === 'object') {
|
|
90
|
+
return (
|
|
91
|
+
'{' +
|
|
92
|
+
Object.keys(value)
|
|
93
|
+
.sort()
|
|
94
|
+
.map(key => JSON.stringify(key) + ':' + stableStringify(value[key]))
|
|
95
|
+
.join(',') +
|
|
96
|
+
'}'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
return JSON.stringify(value);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Resolve the install channel, mirroring lib/ui-routes/dashboard-routes.js.
|
|
103
|
+
function installChannel() {
|
|
104
|
+
if (getBoolean(readEnvValue('EENGINE_DOCEAN'))) {
|
|
105
|
+
return 'docean';
|
|
106
|
+
}
|
|
107
|
+
if (typeof readEnvValue('RENDER_SERVICE_SLUG') === 'string' && readEnvValue('RENDER_SERVICE_SLUG')) {
|
|
108
|
+
return 'render';
|
|
109
|
+
}
|
|
110
|
+
if (getBoolean(readEnvValue('EENGINE_INSTALL_SCRIPT'))) {
|
|
111
|
+
return 'script';
|
|
112
|
+
}
|
|
113
|
+
if (EE_DOCKER_LEGACY) {
|
|
114
|
+
return 'docker-legacy';
|
|
115
|
+
}
|
|
116
|
+
return 'general';
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Race a promise against a timeout so a slow Redis can never delay license validation.
|
|
120
|
+
function withTimeout(promise, ms) {
|
|
121
|
+
return Promise.race([
|
|
122
|
+
promise,
|
|
123
|
+
new Promise((resolve, reject) => {
|
|
124
|
+
setTimeout(() => reject(new Error('Beacon collection timed out')), ms).unref();
|
|
125
|
+
})
|
|
126
|
+
]);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Build the diagnostic snapshot and its digest. Returns { fh, diag } or null on failure.
|
|
130
|
+
async function collectBeacon({ redis, logger }) {
|
|
131
|
+
// Isolate a single field: log and swallow so one failure does not abort the whole snapshot.
|
|
132
|
+
const safe = async fn => {
|
|
133
|
+
try {
|
|
134
|
+
return await fn();
|
|
135
|
+
} catch (err) {
|
|
136
|
+
if (logger) {
|
|
137
|
+
logger.error({ msg: 'Beacon field collection failed', err });
|
|
138
|
+
}
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
try {
|
|
144
|
+
const diag = { v: SCHEMA_VERSION };
|
|
145
|
+
|
|
146
|
+
const s =
|
|
147
|
+
(await safe(() =>
|
|
148
|
+
settings.getMulti(
|
|
149
|
+
'smtpServerEnabled',
|
|
150
|
+
'imapProxyServerEnabled',
|
|
151
|
+
'enableApiProxy',
|
|
152
|
+
'trackOpens',
|
|
153
|
+
'trackClicks',
|
|
154
|
+
'webhooksEnabled',
|
|
155
|
+
'openAiAPIKey',
|
|
156
|
+
'generateEmailSummary',
|
|
157
|
+
'openAiGenerateEmbeddings',
|
|
158
|
+
'openAiAPIUrl',
|
|
159
|
+
'openAiPreProcessingFn',
|
|
160
|
+
'proxyEnabled',
|
|
161
|
+
'httpProxyEnabled',
|
|
162
|
+
'localAddresses',
|
|
163
|
+
'sentryEnabled',
|
|
164
|
+
'authServer',
|
|
165
|
+
'imapIndexer',
|
|
166
|
+
'totpEnabled',
|
|
167
|
+
'documentStoreEnabled',
|
|
168
|
+
'documentStoreGenerateEmbeddings',
|
|
169
|
+
'documentStorePreProcessingEnabled',
|
|
170
|
+
'gmailEnabled',
|
|
171
|
+
'outlookEnabled',
|
|
172
|
+
'mailRuEnabled',
|
|
173
|
+
'trackSentMessages'
|
|
174
|
+
)
|
|
175
|
+
)) || {};
|
|
176
|
+
|
|
177
|
+
const on = key => truthy(s[key]);
|
|
178
|
+
|
|
179
|
+
// Enabled-feature codes (presence = on; codes are omitted when off).
|
|
180
|
+
const feat = [];
|
|
181
|
+
if (on('smtpServerEnabled')) feat.push('smtp');
|
|
182
|
+
if (on('imapProxyServerEnabled')) feat.push('imapproxy');
|
|
183
|
+
if (on('enableApiProxy')) feat.push('apiproxy');
|
|
184
|
+
if (on('trackOpens')) feat.push('track_o');
|
|
185
|
+
if (on('trackClicks')) feat.push('track_c');
|
|
186
|
+
if (on('webhooksEnabled')) feat.push('webhooks');
|
|
187
|
+
if (nonEmpty(s.openAiAPIKey)) feat.push('ai');
|
|
188
|
+
if (on('generateEmailSummary')) feat.push('ai_sum');
|
|
189
|
+
if (on('openAiGenerateEmbeddings')) feat.push('ai_embed');
|
|
190
|
+
if (nonEmpty(s.openAiAPIUrl)) feat.push('ai_url');
|
|
191
|
+
if (nonEmpty(s.openAiPreProcessingFn)) feat.push('ai_prefn');
|
|
192
|
+
if (on('proxyEnabled')) feat.push('proxy');
|
|
193
|
+
if (on('httpProxyEnabled')) feat.push('httpproxy');
|
|
194
|
+
if (truthy(s.localAddresses)) feat.push('localaddr');
|
|
195
|
+
if (on('sentryEnabled')) feat.push('sentry');
|
|
196
|
+
if (nonEmpty(s.authServer)) feat.push('authsrv');
|
|
197
|
+
if (s.imapIndexer === 'fast') feat.push('idx_fast');
|
|
198
|
+
if (on('totpEnabled')) feat.push('totp');
|
|
199
|
+
if (hasEnvValue('OKTA_OAUTH2_ISSUER') && hasEnvValue('OKTA_OAUTH2_CLIENT_ID') && hasEnvValue('OKTA_OAUTH2_CLIENT_SECRET')) {
|
|
200
|
+
feat.push('okta');
|
|
201
|
+
}
|
|
202
|
+
if (await safe(() => passkeys.hasPasskeys())) {
|
|
203
|
+
feat.push('passkey');
|
|
204
|
+
}
|
|
205
|
+
diag.feat = feat.sort();
|
|
206
|
+
|
|
207
|
+
// Entity magnitude tiers (buckets, not counts).
|
|
208
|
+
const scard = key => safe(() => redis.scard(`${REDIS_PREFIX}${key}`));
|
|
209
|
+
const rawAccounts = Number(await scard('ia:accounts')) || 0;
|
|
210
|
+
diag.tiers = {
|
|
211
|
+
acct: tier(rawAccounts),
|
|
212
|
+
oapp: tier(await scard('oapp:i')),
|
|
213
|
+
gw: tier(await scard('gateways')),
|
|
214
|
+
wh: tier(await scard('wh:i')),
|
|
215
|
+
tpl: tier(await scard('tpl::i')),
|
|
216
|
+
bl: tier(await safe(() => redis.hlen(`${REDIS_PREFIX}lists:unsub:lists`)))
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
// Provider mix. `oapp` = provider types of configured OAuth apps; `prov` = provider types
|
|
220
|
+
// that actually have accounts (plus `imap` for any non-OAuth accounts). Only the app id and
|
|
221
|
+
// provider type are read from the sanitized app listing - no secrets are inspected.
|
|
222
|
+
await safe(async () => {
|
|
223
|
+
const res = await oauth2Apps.list(0, 100000);
|
|
224
|
+
const apps = (res && res.apps) || [];
|
|
225
|
+
|
|
226
|
+
const configured = new Set();
|
|
227
|
+
const appProviders = [];
|
|
228
|
+
for (const app of apps) {
|
|
229
|
+
if (app && app.provider) {
|
|
230
|
+
configured.add(app.provider);
|
|
231
|
+
appProviders.push([app.id, app.provider]);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
diag.oapp = Array.from(configured).sort();
|
|
235
|
+
|
|
236
|
+
const inUse = new Set();
|
|
237
|
+
let oauthAccounts = 0;
|
|
238
|
+
if (appProviders.length) {
|
|
239
|
+
const multi = redis.multi();
|
|
240
|
+
for (const [id] of appProviders) {
|
|
241
|
+
multi.scard(`${REDIS_PREFIX}oapp:a:${id}`);
|
|
242
|
+
}
|
|
243
|
+
const counts = await multi.exec();
|
|
244
|
+
for (let i = 0; i < appProviders.length; i++) {
|
|
245
|
+
const entry = counts[i];
|
|
246
|
+
const count = (entry && !entry[0] && Number(entry[1])) || 0;
|
|
247
|
+
oauthAccounts += count;
|
|
248
|
+
if (count > 0) {
|
|
249
|
+
inUse.add(appProviders[i][1]);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (rawAccounts > oauthAccounts) {
|
|
254
|
+
inUse.add('imap');
|
|
255
|
+
}
|
|
256
|
+
diag.prov = Array.from(inUse).sort();
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
// Exercised-usage signals from the existing event counters.
|
|
260
|
+
await safe(async () => {
|
|
261
|
+
const counters = (await getCounterValues(redis, USE_WINDOW_SECONDS)) || {};
|
|
262
|
+
const use = [];
|
|
263
|
+
if (counters['events:messageNew'] > 0) use.push('recv');
|
|
264
|
+
if (counters['submit:success'] > 0) use.push('send');
|
|
265
|
+
if (counters['webhooks:success'] > 0) use.push('wh');
|
|
266
|
+
if (counters['apiCall:success'] > 0) use.push('api');
|
|
267
|
+
diag.use = use.sort();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
// Deprecation watchlist (presence of legacy/candidate-for-removal features).
|
|
271
|
+
const dep = [];
|
|
272
|
+
if (on('documentStoreEnabled')) dep.push('documentStore');
|
|
273
|
+
if (documentStoreFeatureEnabled) dep.push('documentStoreGate');
|
|
274
|
+
if (on('documentStoreGenerateEmbeddings')) dep.push('ds_embed');
|
|
275
|
+
if (on('documentStorePreProcessingEnabled')) dep.push('ds_preproc');
|
|
276
|
+
if (on('gmailEnabled') || on('outlookEnabled') || on('mailRuEnabled')) dep.push('legacyOauth');
|
|
277
|
+
if (on('trackSentMessages')) dep.push('trackSent');
|
|
278
|
+
if (EE_DOCKER_LEGACY) dep.push('dockerLegacy');
|
|
279
|
+
await safe(async () => {
|
|
280
|
+
const ids = await redis.smembers(`${REDIS_PREFIX}wh:i`);
|
|
281
|
+
if (ids && ids.length && ids.length <= WH_SCAN_LIMIT) {
|
|
282
|
+
const bufs = await redis.hmgetBuffer(
|
|
283
|
+
`${REDIS_PREFIX}wh:c`,
|
|
284
|
+
ids.map(id => `${id}:content`)
|
|
285
|
+
);
|
|
286
|
+
for (const buf of bufs || []) {
|
|
287
|
+
if (!buf || !buf.length) {
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
try {
|
|
291
|
+
const content = msgpack.decode(buf);
|
|
292
|
+
if (content && (content.fn || content.map)) {
|
|
293
|
+
dep.push('whSubscript');
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
} catch (err) {
|
|
297
|
+
// undecodable entry, skip
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
diag.dep = dep.sort();
|
|
303
|
+
|
|
304
|
+
// Enabled EENGINE_FEATURE_* flags (already sorted).
|
|
305
|
+
diag.flags = (await safe(() => featureFlags.listEnabled())) || [];
|
|
306
|
+
|
|
307
|
+
// Runtime context.
|
|
308
|
+
diag.dist = installChannel();
|
|
309
|
+
diag.node = process.versions.node;
|
|
310
|
+
diag.arch = process.arch;
|
|
311
|
+
|
|
312
|
+
const fh = crypto.createHash('sha256').update(stableStringify(diag)).digest('hex').slice(0, 12);
|
|
313
|
+
|
|
314
|
+
return { fh, diag };
|
|
315
|
+
} catch (err) {
|
|
316
|
+
if (logger) {
|
|
317
|
+
logger.error({ msg: 'Beacon collection failed', err });
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Collect the snapshot (time-boxed) and decide what to attach to the license request body.
|
|
324
|
+
// Always attaches the digest `fh`; attaches the full `diag` only when the digest changed since the
|
|
325
|
+
// last accepted send or the 30-day heartbeat is due. Best-effort: never throws.
|
|
326
|
+
async function attachBeacon(body, { redis, logger, now }) {
|
|
327
|
+
try {
|
|
328
|
+
const beacon = await withTimeout(collectBeacon({ redis, logger }), COLLECT_TIMEOUT_MS);
|
|
329
|
+
if (!beacon || !beacon.fh) {
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
body.fh = beacon.fh;
|
|
333
|
+
|
|
334
|
+
const [storedHash, bft] = await redis.hmget(`${REDIS_PREFIX}settings`, ['bfh', 'bft']);
|
|
335
|
+
const lastFull = parseInt(bft || '0', 16) || 0;
|
|
336
|
+
if (beacon.fh !== storedHash || now - lastFull > FULL_RESEND_INTERVAL_MS) {
|
|
337
|
+
body.diag = beacon.diag;
|
|
338
|
+
}
|
|
339
|
+
} catch (err) {
|
|
340
|
+
if (logger) {
|
|
341
|
+
logger.error({ msg: 'License beacon collection failed', err });
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Persist the send-on-change markers after a successful validation. `needFull` (from the server)
|
|
347
|
+
// forces a full resend on the next cycle when the server has the digest but not the snapshot.
|
|
348
|
+
// Best-effort: never throws.
|
|
349
|
+
async function persistBeaconMarkers({ redis, logger, body, now, needFull }) {
|
|
350
|
+
try {
|
|
351
|
+
if (body.fh) {
|
|
352
|
+
await redis.hset(`${REDIS_PREFIX}settings`, 'bfh', body.fh);
|
|
353
|
+
if (body.diag) {
|
|
354
|
+
await redis.hset(`${REDIS_PREFIX}settings`, 'bft', now.toString(16));
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
if (needFull) {
|
|
358
|
+
await redis.hdel(`${REDIS_PREFIX}settings`, 'bfh');
|
|
359
|
+
}
|
|
360
|
+
} catch (err) {
|
|
361
|
+
if (logger) {
|
|
362
|
+
logger.error({ msg: 'Failed to persist license beacon markers', err });
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
module.exports = { collectBeacon, attachBeacon, persistBeaconMarkers, withTimeout, tier, stableStringify };
|
package/lib/logger.js
CHANGED
|
@@ -7,6 +7,7 @@ if (!process.env.EE_ENV_LOADED) {
|
|
|
7
7
|
|
|
8
8
|
const config = require('@zone-eu/wild-config');
|
|
9
9
|
const pino = require('pino');
|
|
10
|
+
const { TRANSIENT_NETWORK_CODES } = require('./consts');
|
|
10
11
|
|
|
11
12
|
config.log = config.log || {
|
|
12
13
|
level: 'trace'
|
|
@@ -14,10 +15,19 @@ config.log = config.log || {
|
|
|
14
15
|
|
|
15
16
|
config.log.level = config.log.level || 'trace';
|
|
16
17
|
|
|
18
|
+
// undici raises every connection failure as a generic `TypeError` (e.g. "fetch failed"
|
|
19
|
+
// or "terminated") with the real DNS/socket error attached as err.cause. Those are
|
|
20
|
+
// transient, environmental blips - not code bugs - so they must not be forwarded to
|
|
21
|
+
// error tracking, where they pile up as useless "fetch failed" reports. Genuine
|
|
22
|
+
// TypeError/RangeError bugs have no network errno cause and still get reported.
|
|
23
|
+
function isTransientFetchError(err) {
|
|
24
|
+
return err && err.name === 'TypeError' && err.cause && TRANSIENT_NETWORK_CODES.has(err.cause.code);
|
|
25
|
+
}
|
|
26
|
+
|
|
17
27
|
let logger = pino({
|
|
18
28
|
formatters: {
|
|
19
29
|
log(object) {
|
|
20
|
-
if (object.err && ['TypeError', 'RangeError'].includes(object.err.name)) {
|
|
30
|
+
if (object.err && ['TypeError', 'RangeError'].includes(object.err.name) && !isTransientFetchError(object.err)) {
|
|
21
31
|
if (logger.notifyError) {
|
|
22
32
|
let meta = {};
|
|
23
33
|
for (let key of ['msg', 'path', 'cid']) {
|
package/lib/oauth/gmail.js
CHANGED
|
@@ -824,6 +824,9 @@ class GmailOauth {
|
|
|
824
824
|
}
|
|
825
825
|
|
|
826
826
|
module.exports.GmailOauth = GmailOauth;
|
|
827
|
+
// Exported for unit testing the auth error-to-flag mapping.
|
|
828
|
+
module.exports.checkForFlags = checkForFlags;
|
|
829
|
+
module.exports.checkForUserFlags = checkForUserFlags;
|
|
827
830
|
module.exports.GMAIL_SCOPES = GMAIL_SCOPES;
|
|
828
831
|
module.exports.GMAIL_API_SCOPES = GMAIL_API_SCOPES;
|
|
829
832
|
module.exports.OPENID_SCOPES = OPENID_SCOPES;
|
package/lib/oauth/outlook.js
CHANGED
|
@@ -701,3 +701,6 @@ class OutlookOauth {
|
|
|
701
701
|
module.exports.OutlookOauth = OutlookOauth;
|
|
702
702
|
module.exports.outlookScopes = outlookScopes;
|
|
703
703
|
module.exports.OUTLOOK_API_SCOPES = OUTLOOK_API_SCOPES;
|
|
704
|
+
// Exported for unit testing the auth error-to-flag mapping.
|
|
705
|
+
module.exports.checkForFlags = checkForFlags;
|
|
706
|
+
module.exports.checkForUserFlags = checkForUserFlags;
|