emailengine-app 2.70.0 → 2.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/test.yml +73 -12
- package/.ncurc.js +3 -3
- package/CHANGELOG.md +18 -0
- package/Gruntfile.js +19 -23
- package/bin/emailengine.js +8 -1
- package/config/default.toml +5 -0
- package/config/test.toml +5 -0
- package/data/google-crawlers.json +1 -1
- package/getswagger.sh +4 -0
- package/lib/account.js +31 -25
- package/lib/api-routes/message-routes.js +125 -121
- package/lib/document-store.js +22 -1
- package/lib/email-client/base-client.js +3 -2
- package/lib/email-client/imap/mailbox.js +2 -2
- package/lib/email-client/notification-handler.js +2 -2
- package/lib/export.js +12 -0
- package/lib/feature-flags.js +6 -0
- package/lib/license-beacon.js +367 -0
- package/lib/logger.js +11 -1
- package/lib/routes-ui.js +2 -1
- package/lib/tools.js +26 -2
- package/lib/ui-routes/admin-config-routes.js +4 -3
- package/lib/ui-routes/document-store-routes.js +7 -1
- package/package.json +19 -16
- package/sbom.json +1 -1
- package/server.js +30 -8
- package/static/licenses.html +43 -123
- package/translations/de.mo +0 -0
- package/translations/de.po +154 -142
- package/translations/et.mo +0 -0
- package/translations/et.po +129 -131
- package/translations/fr.mo +0 -0
- package/translations/fr.po +133 -136
- package/translations/ja.mo +0 -0
- package/translations/ja.po +126 -129
- package/translations/messages.pot +37 -37
- package/translations/nl.mo +0 -0
- package/translations/nl.po +128 -130
- package/translations/pl.mo +0 -0
- package/translations/pl.po +125 -128
- package/views/dashboard.hbs +22 -0
- package/workers/api.js +22 -5
- package/workers/export.js +58 -43
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// License-validation feature beacon.
|
|
4
|
+
//
|
|
5
|
+
// Collects a compact, anonymized snapshot of which features are enabled and exercised on this
|
|
6
|
+
// instance, to be piggybacked onto the existing daily license-validation POST. The intent is to
|
|
7
|
+
// learn whether deprecation-candidate features are still in use in the field.
|
|
8
|
+
//
|
|
9
|
+
// Privacy: the snapshot contains only enable-flags, provider type names, coarse magnitude tiers
|
|
10
|
+
// (NOT raw counts), exercised-usage booleans, and runtime context. It never includes account
|
|
11
|
+
// addresses, URLs, credentials, or any other PII/secrets.
|
|
12
|
+
//
|
|
13
|
+
// Reliability: collection is strictly best-effort. Every field is isolated so one failing Redis
|
|
14
|
+
// read degrades that field rather than the whole snapshot, and collectBeacon never throws - on a
|
|
15
|
+
// catastrophic failure it returns null and the license call proceeds with its original fields.
|
|
16
|
+
// The caller is expected to also time-box this with withTimeout().
|
|
17
|
+
|
|
18
|
+
const crypto = require('crypto');
|
|
19
|
+
const msgpack = require('msgpack5')();
|
|
20
|
+
|
|
21
|
+
const settings = require('./settings');
|
|
22
|
+
const { getCounterValues, hasEnvValue, readEnvValue, getBoolean } = require('./tools');
|
|
23
|
+
const { REDIS_PREFIX, EE_DOCKER_LEGACY } = require('./consts');
|
|
24
|
+
const { oauth2Apps } = require('./oauth2-apps');
|
|
25
|
+
const passkeys = require('./passkeys');
|
|
26
|
+
const featureFlags = require('./feature-flags');
|
|
27
|
+
const { documentStoreFeatureEnabled } = require('./document-store');
|
|
28
|
+
|
|
29
|
+
// Beacon schema version. Bump when the meaning of codes changes so the license server can adapt.
|
|
30
|
+
const SCHEMA_VERSION = 1;
|
|
31
|
+
|
|
32
|
+
// Window for "exercised recently" usage signals. A week smooths over quiet days so the digest
|
|
33
|
+
// (and therefore the full-payload sends) does not churn day to day.
|
|
34
|
+
const USE_WINDOW_SECONDS = 7 * 24 * 3600;
|
|
35
|
+
|
|
36
|
+
// Skip the per-route webhook content scan above this many routes (keeps the collector cheap).
|
|
37
|
+
const WH_SCAN_LIMIT = 250;
|
|
38
|
+
|
|
39
|
+
// Time-box for a single collection so a slow Redis can never delay license validation.
|
|
40
|
+
const COLLECT_TIMEOUT_MS = 2000;
|
|
41
|
+
|
|
42
|
+
// Resend the full snapshot at least this often even when its digest has not changed.
|
|
43
|
+
const FULL_RESEND_INTERVAL_MS = 30 * 24 * 3600 * 1000;
|
|
44
|
+
|
|
45
|
+
// Map a raw count to a coarse magnitude tier (powers of ten). Values are buckets, never counts.
|
|
46
|
+
function tier(n) {
|
|
47
|
+
n = Number(n) || 0;
|
|
48
|
+
if (n <= 0) return 0;
|
|
49
|
+
if (n === 1) return 1;
|
|
50
|
+
if (n < 10) return 2;
|
|
51
|
+
if (n < 100) return 3;
|
|
52
|
+
if (n < 1000) return 4;
|
|
53
|
+
if (n < 10000) return 5;
|
|
54
|
+
return 6;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Truthiness for boolean-ish settings (schema booleans arrive as real booleans; legacy/raw values
|
|
58
|
+
// may be strings or arrays).
|
|
59
|
+
function truthy(value) {
|
|
60
|
+
if (value === true) {
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
if (typeof value === 'number') {
|
|
64
|
+
return value !== 0;
|
|
65
|
+
}
|
|
66
|
+
if (Array.isArray(value)) {
|
|
67
|
+
return value.length > 0;
|
|
68
|
+
}
|
|
69
|
+
if (typeof value === 'string') {
|
|
70
|
+
return /^(y|yes|true|t|1)$/i.test(value.trim());
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Non-empty check for string-valued settings (URLs, keys, scripts) without inspecting the value.
|
|
76
|
+
function nonEmpty(value) {
|
|
77
|
+
if (typeof value === 'string') {
|
|
78
|
+
return value.trim().length > 0;
|
|
79
|
+
}
|
|
80
|
+
return truthy(value);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Deterministic serialization: object keys sorted recursively. Arrays are pre-sorted at build time.
|
|
84
|
+
// Produces a stable string so the digest only changes when the snapshot meaningfully changes.
|
|
85
|
+
function stableStringify(value) {
|
|
86
|
+
if (Array.isArray(value)) {
|
|
87
|
+
return '[' + value.map(stableStringify).join(',') + ']';
|
|
88
|
+
}
|
|
89
|
+
if (value && typeof value === 'object') {
|
|
90
|
+
return (
|
|
91
|
+
'{' +
|
|
92
|
+
Object.keys(value)
|
|
93
|
+
.sort()
|
|
94
|
+
.map(key => JSON.stringify(key) + ':' + stableStringify(value[key]))
|
|
95
|
+
.join(',') +
|
|
96
|
+
'}'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
return JSON.stringify(value);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Resolve the install channel, mirroring lib/ui-routes/dashboard-routes.js.
|
|
103
|
+
function installChannel() {
|
|
104
|
+
if (getBoolean(readEnvValue('EENGINE_DOCEAN'))) {
|
|
105
|
+
return 'docean';
|
|
106
|
+
}
|
|
107
|
+
if (typeof readEnvValue('RENDER_SERVICE_SLUG') === 'string' && readEnvValue('RENDER_SERVICE_SLUG')) {
|
|
108
|
+
return 'render';
|
|
109
|
+
}
|
|
110
|
+
if (getBoolean(readEnvValue('EENGINE_INSTALL_SCRIPT'))) {
|
|
111
|
+
return 'script';
|
|
112
|
+
}
|
|
113
|
+
if (EE_DOCKER_LEGACY) {
|
|
114
|
+
return 'docker-legacy';
|
|
115
|
+
}
|
|
116
|
+
return 'general';
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Race a promise against a timeout so a slow Redis can never delay license validation.
|
|
120
|
+
function withTimeout(promise, ms) {
|
|
121
|
+
return Promise.race([
|
|
122
|
+
promise,
|
|
123
|
+
new Promise((resolve, reject) => {
|
|
124
|
+
setTimeout(() => reject(new Error('Beacon collection timed out')), ms).unref();
|
|
125
|
+
})
|
|
126
|
+
]);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Build the diagnostic snapshot and its digest. Returns { fh, diag } or null on failure.
|
|
130
|
+
async function collectBeacon({ redis, logger }) {
|
|
131
|
+
// Isolate a single field: log and swallow so one failure does not abort the whole snapshot.
|
|
132
|
+
const safe = async fn => {
|
|
133
|
+
try {
|
|
134
|
+
return await fn();
|
|
135
|
+
} catch (err) {
|
|
136
|
+
if (logger) {
|
|
137
|
+
logger.error({ msg: 'Beacon field collection failed', err });
|
|
138
|
+
}
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
try {
|
|
144
|
+
const diag = { v: SCHEMA_VERSION };
|
|
145
|
+
|
|
146
|
+
const s =
|
|
147
|
+
(await safe(() =>
|
|
148
|
+
settings.getMulti(
|
|
149
|
+
'smtpServerEnabled',
|
|
150
|
+
'imapProxyServerEnabled',
|
|
151
|
+
'enableApiProxy',
|
|
152
|
+
'trackOpens',
|
|
153
|
+
'trackClicks',
|
|
154
|
+
'webhooksEnabled',
|
|
155
|
+
'openAiAPIKey',
|
|
156
|
+
'generateEmailSummary',
|
|
157
|
+
'openAiGenerateEmbeddings',
|
|
158
|
+
'openAiAPIUrl',
|
|
159
|
+
'openAiPreProcessingFn',
|
|
160
|
+
'proxyEnabled',
|
|
161
|
+
'httpProxyEnabled',
|
|
162
|
+
'localAddresses',
|
|
163
|
+
'sentryEnabled',
|
|
164
|
+
'authServer',
|
|
165
|
+
'imapIndexer',
|
|
166
|
+
'totpEnabled',
|
|
167
|
+
'documentStoreEnabled',
|
|
168
|
+
'documentStoreGenerateEmbeddings',
|
|
169
|
+
'documentStorePreProcessingEnabled',
|
|
170
|
+
'gmailEnabled',
|
|
171
|
+
'outlookEnabled',
|
|
172
|
+
'mailRuEnabled',
|
|
173
|
+
'trackSentMessages'
|
|
174
|
+
)
|
|
175
|
+
)) || {};
|
|
176
|
+
|
|
177
|
+
const on = key => truthy(s[key]);
|
|
178
|
+
|
|
179
|
+
// Enabled-feature codes (presence = on; codes are omitted when off).
|
|
180
|
+
const feat = [];
|
|
181
|
+
if (on('smtpServerEnabled')) feat.push('smtp');
|
|
182
|
+
if (on('imapProxyServerEnabled')) feat.push('imapproxy');
|
|
183
|
+
if (on('enableApiProxy')) feat.push('apiproxy');
|
|
184
|
+
if (on('trackOpens')) feat.push('track_o');
|
|
185
|
+
if (on('trackClicks')) feat.push('track_c');
|
|
186
|
+
if (on('webhooksEnabled')) feat.push('webhooks');
|
|
187
|
+
if (nonEmpty(s.openAiAPIKey)) feat.push('ai');
|
|
188
|
+
if (on('generateEmailSummary')) feat.push('ai_sum');
|
|
189
|
+
if (on('openAiGenerateEmbeddings')) feat.push('ai_embed');
|
|
190
|
+
if (nonEmpty(s.openAiAPIUrl)) feat.push('ai_url');
|
|
191
|
+
if (nonEmpty(s.openAiPreProcessingFn)) feat.push('ai_prefn');
|
|
192
|
+
if (on('proxyEnabled')) feat.push('proxy');
|
|
193
|
+
if (on('httpProxyEnabled')) feat.push('httpproxy');
|
|
194
|
+
if (truthy(s.localAddresses)) feat.push('localaddr');
|
|
195
|
+
if (on('sentryEnabled')) feat.push('sentry');
|
|
196
|
+
if (nonEmpty(s.authServer)) feat.push('authsrv');
|
|
197
|
+
if (s.imapIndexer === 'fast') feat.push('idx_fast');
|
|
198
|
+
if (on('totpEnabled')) feat.push('totp');
|
|
199
|
+
if (hasEnvValue('OKTA_OAUTH2_ISSUER') && hasEnvValue('OKTA_OAUTH2_CLIENT_ID') && hasEnvValue('OKTA_OAUTH2_CLIENT_SECRET')) {
|
|
200
|
+
feat.push('okta');
|
|
201
|
+
}
|
|
202
|
+
if (await safe(() => passkeys.hasPasskeys())) {
|
|
203
|
+
feat.push('passkey');
|
|
204
|
+
}
|
|
205
|
+
diag.feat = feat.sort();
|
|
206
|
+
|
|
207
|
+
// Entity magnitude tiers (buckets, not counts).
|
|
208
|
+
const scard = key => safe(() => redis.scard(`${REDIS_PREFIX}${key}`));
|
|
209
|
+
const rawAccounts = Number(await scard('ia:accounts')) || 0;
|
|
210
|
+
diag.tiers = {
|
|
211
|
+
acct: tier(rawAccounts),
|
|
212
|
+
oapp: tier(await scard('oapp:i')),
|
|
213
|
+
gw: tier(await scard('gateways')),
|
|
214
|
+
wh: tier(await scard('wh:i')),
|
|
215
|
+
tpl: tier(await scard('tpl::i')),
|
|
216
|
+
bl: tier(await safe(() => redis.hlen(`${REDIS_PREFIX}lists:unsub:lists`)))
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
// Provider mix. `oapp` = provider types of configured OAuth apps; `prov` = provider types
|
|
220
|
+
// that actually have accounts (plus `imap` for any non-OAuth accounts). Only the app id and
|
|
221
|
+
// provider type are read from the sanitized app listing - no secrets are inspected.
|
|
222
|
+
await safe(async () => {
|
|
223
|
+
const res = await oauth2Apps.list(0, 100000);
|
|
224
|
+
const apps = (res && res.apps) || [];
|
|
225
|
+
|
|
226
|
+
const configured = new Set();
|
|
227
|
+
const appProviders = [];
|
|
228
|
+
for (const app of apps) {
|
|
229
|
+
if (app && app.provider) {
|
|
230
|
+
configured.add(app.provider);
|
|
231
|
+
appProviders.push([app.id, app.provider]);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
diag.oapp = Array.from(configured).sort();
|
|
235
|
+
|
|
236
|
+
const inUse = new Set();
|
|
237
|
+
let oauthAccounts = 0;
|
|
238
|
+
if (appProviders.length) {
|
|
239
|
+
const multi = redis.multi();
|
|
240
|
+
for (const [id] of appProviders) {
|
|
241
|
+
multi.scard(`${REDIS_PREFIX}oapp:a:${id}`);
|
|
242
|
+
}
|
|
243
|
+
const counts = await multi.exec();
|
|
244
|
+
for (let i = 0; i < appProviders.length; i++) {
|
|
245
|
+
const entry = counts[i];
|
|
246
|
+
const count = (entry && !entry[0] && Number(entry[1])) || 0;
|
|
247
|
+
oauthAccounts += count;
|
|
248
|
+
if (count > 0) {
|
|
249
|
+
inUse.add(appProviders[i][1]);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (rawAccounts > oauthAccounts) {
|
|
254
|
+
inUse.add('imap');
|
|
255
|
+
}
|
|
256
|
+
diag.prov = Array.from(inUse).sort();
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
// Exercised-usage signals from the existing event counters.
|
|
260
|
+
await safe(async () => {
|
|
261
|
+
const counters = (await getCounterValues(redis, USE_WINDOW_SECONDS)) || {};
|
|
262
|
+
const use = [];
|
|
263
|
+
if (counters['events:messageNew'] > 0) use.push('recv');
|
|
264
|
+
if (counters['submit:success'] > 0) use.push('send');
|
|
265
|
+
if (counters['webhooks:success'] > 0) use.push('wh');
|
|
266
|
+
if (counters['apiCall:success'] > 0) use.push('api');
|
|
267
|
+
diag.use = use.sort();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
// Deprecation watchlist (presence of legacy/candidate-for-removal features).
|
|
271
|
+
const dep = [];
|
|
272
|
+
if (on('documentStoreEnabled')) dep.push('documentStore');
|
|
273
|
+
if (documentStoreFeatureEnabled) dep.push('documentStoreGate');
|
|
274
|
+
if (on('documentStoreGenerateEmbeddings')) dep.push('ds_embed');
|
|
275
|
+
if (on('documentStorePreProcessingEnabled')) dep.push('ds_preproc');
|
|
276
|
+
if (on('gmailEnabled') || on('outlookEnabled') || on('mailRuEnabled')) dep.push('legacyOauth');
|
|
277
|
+
if (on('trackSentMessages')) dep.push('trackSent');
|
|
278
|
+
if (EE_DOCKER_LEGACY) dep.push('dockerLegacy');
|
|
279
|
+
await safe(async () => {
|
|
280
|
+
const ids = await redis.smembers(`${REDIS_PREFIX}wh:i`);
|
|
281
|
+
if (ids && ids.length && ids.length <= WH_SCAN_LIMIT) {
|
|
282
|
+
const bufs = await redis.hmgetBuffer(
|
|
283
|
+
`${REDIS_PREFIX}wh:c`,
|
|
284
|
+
ids.map(id => `${id}:content`)
|
|
285
|
+
);
|
|
286
|
+
for (const buf of bufs || []) {
|
|
287
|
+
if (!buf || !buf.length) {
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
try {
|
|
291
|
+
const content = msgpack.decode(buf);
|
|
292
|
+
if (content && (content.fn || content.map)) {
|
|
293
|
+
dep.push('whSubscript');
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
} catch (err) {
|
|
297
|
+
// undecodable entry, skip
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
diag.dep = dep.sort();
|
|
303
|
+
|
|
304
|
+
// Enabled EENGINE_FEATURE_* flags (already sorted).
|
|
305
|
+
diag.flags = (await safe(() => featureFlags.listEnabled())) || [];
|
|
306
|
+
|
|
307
|
+
// Runtime context.
|
|
308
|
+
diag.dist = installChannel();
|
|
309
|
+
diag.node = process.versions.node;
|
|
310
|
+
diag.arch = process.arch;
|
|
311
|
+
|
|
312
|
+
const fh = crypto.createHash('sha256').update(stableStringify(diag)).digest('hex').slice(0, 12);
|
|
313
|
+
|
|
314
|
+
return { fh, diag };
|
|
315
|
+
} catch (err) {
|
|
316
|
+
if (logger) {
|
|
317
|
+
logger.error({ msg: 'Beacon collection failed', err });
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Collect the snapshot (time-boxed) and decide what to attach to the license request body.
|
|
324
|
+
// Always attaches the digest `fh`; attaches the full `diag` only when the digest changed since the
|
|
325
|
+
// last accepted send or the 30-day heartbeat is due. Best-effort: never throws.
|
|
326
|
+
async function attachBeacon(body, { redis, logger, now }) {
|
|
327
|
+
try {
|
|
328
|
+
const beacon = await withTimeout(collectBeacon({ redis, logger }), COLLECT_TIMEOUT_MS);
|
|
329
|
+
if (!beacon || !beacon.fh) {
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
body.fh = beacon.fh;
|
|
333
|
+
|
|
334
|
+
const [storedHash, bft] = await redis.hmget(`${REDIS_PREFIX}settings`, ['bfh', 'bft']);
|
|
335
|
+
const lastFull = parseInt(bft || '0', 16) || 0;
|
|
336
|
+
if (beacon.fh !== storedHash || now - lastFull > FULL_RESEND_INTERVAL_MS) {
|
|
337
|
+
body.diag = beacon.diag;
|
|
338
|
+
}
|
|
339
|
+
} catch (err) {
|
|
340
|
+
if (logger) {
|
|
341
|
+
logger.error({ msg: 'License beacon collection failed', err });
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Persist the send-on-change markers after a successful validation. `needFull` (from the server)
|
|
347
|
+
// forces a full resend on the next cycle when the server has the digest but not the snapshot.
|
|
348
|
+
// Best-effort: never throws.
|
|
349
|
+
async function persistBeaconMarkers({ redis, logger, body, now, needFull }) {
|
|
350
|
+
try {
|
|
351
|
+
if (body.fh) {
|
|
352
|
+
await redis.hset(`${REDIS_PREFIX}settings`, 'bfh', body.fh);
|
|
353
|
+
if (body.diag) {
|
|
354
|
+
await redis.hset(`${REDIS_PREFIX}settings`, 'bft', now.toString(16));
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
if (needFull) {
|
|
358
|
+
await redis.hdel(`${REDIS_PREFIX}settings`, 'bfh');
|
|
359
|
+
}
|
|
360
|
+
} catch (err) {
|
|
361
|
+
if (logger) {
|
|
362
|
+
logger.error({ msg: 'Failed to persist license beacon markers', err });
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
module.exports = { collectBeacon, attachBeacon, persistBeaconMarkers, withTimeout, tier, stableStringify };
|
package/lib/logger.js
CHANGED
|
@@ -7,6 +7,7 @@ if (!process.env.EE_ENV_LOADED) {
|
|
|
7
7
|
|
|
8
8
|
const config = require('@zone-eu/wild-config');
|
|
9
9
|
const pino = require('pino');
|
|
10
|
+
const { TRANSIENT_NETWORK_CODES } = require('./consts');
|
|
10
11
|
|
|
11
12
|
config.log = config.log || {
|
|
12
13
|
level: 'trace'
|
|
@@ -14,10 +15,19 @@ config.log = config.log || {
|
|
|
14
15
|
|
|
15
16
|
config.log.level = config.log.level || 'trace';
|
|
16
17
|
|
|
18
|
+
// undici raises every connection failure as a generic `TypeError` (e.g. "fetch failed"
|
|
19
|
+
// or "terminated") with the real DNS/socket error attached as err.cause. Those are
|
|
20
|
+
// transient, environmental blips - not code bugs - so they must not be forwarded to
|
|
21
|
+
// error tracking, where they pile up as useless "fetch failed" reports. Genuine
|
|
22
|
+
// TypeError/RangeError bugs have no network errno cause and still get reported.
|
|
23
|
+
function isTransientFetchError(err) {
|
|
24
|
+
return err && err.name === 'TypeError' && err.cause && TRANSIENT_NETWORK_CODES.has(err.cause.code);
|
|
25
|
+
}
|
|
26
|
+
|
|
17
27
|
let logger = pino({
|
|
18
28
|
formatters: {
|
|
19
29
|
log(object) {
|
|
20
|
-
if (object.err && ['TypeError', 'RangeError'].includes(object.err.name)) {
|
|
30
|
+
if (object.err && ['TypeError', 'RangeError'].includes(object.err.name) && !isTransientFetchError(object.err)) {
|
|
21
31
|
if (logger.notifyError) {
|
|
22
32
|
let meta = {};
|
|
23
33
|
for (let key of ['msg', 'path', 'cid']) {
|
package/lib/routes-ui.js
CHANGED
|
@@ -35,7 +35,8 @@ function applyRoutes(server, call) {
|
|
|
35
35
|
// Network, SMTP server, IMAP proxy, and browser config routes
|
|
36
36
|
networkConfigRoutes({ server, call });
|
|
37
37
|
|
|
38
|
-
// Document Store (Elasticsearch) config routes
|
|
38
|
+
// Document Store (Elasticsearch) config routes (deprecated feature; the module self-gates
|
|
39
|
+
// and registers no routes unless the Document Store feature is enabled)
|
|
39
40
|
documentStoreRoutes({ server });
|
|
40
41
|
|
|
41
42
|
// Admin auth and user-profile routes (login, logout, TOTP, passkeys, password)
|
package/lib/tools.js
CHANGED
|
@@ -57,7 +57,26 @@ const AGENT_OPTS = {
|
|
|
57
57
|
const RETRY_OPTS = {
|
|
58
58
|
maxRetries: URL_FETCH_RETRY_MAX,
|
|
59
59
|
methods: ['GET', 'PUT', 'HEAD', 'OPTIONS', 'DELETE', 'POST'],
|
|
60
|
-
statusCodes: [429] // do not retry 5xx errors
|
|
60
|
+
statusCodes: [429], // do not retry 5xx errors
|
|
61
|
+
// undici does not retry transient DNS failures (EAI_AGAIN) or socket connect
|
|
62
|
+
// timeouts (ETIMEDOUT) by default, so a name-resolution blip bubbles up as a hard
|
|
63
|
+
// `TypeError: fetch failed`. Add them to the retry list. Passing errorCodes REPLACES
|
|
64
|
+
// undici's defaults, so its default codes are re-listed here.
|
|
65
|
+
errorCodes: [
|
|
66
|
+
// undici defaults
|
|
67
|
+
'ECONNRESET',
|
|
68
|
+
'ECONNREFUSED',
|
|
69
|
+
'ENOTFOUND',
|
|
70
|
+
'ENETDOWN',
|
|
71
|
+
'ENETUNREACH',
|
|
72
|
+
'EHOSTDOWN',
|
|
73
|
+
'EHOSTUNREACH',
|
|
74
|
+
'EPIPE',
|
|
75
|
+
'UND_ERR_SOCKET',
|
|
76
|
+
// additions: transient DNS / socket connect-timeout failures
|
|
77
|
+
'EAI_AGAIN',
|
|
78
|
+
'ETIMEDOUT'
|
|
79
|
+
]
|
|
61
80
|
};
|
|
62
81
|
|
|
63
82
|
// Shared mutable object -- consumers import the object reference and access
|
|
@@ -2198,6 +2217,7 @@ vWuuhT9ely8AUX2F
|
|
|
2198
2217
|
mF3GOI+Ev7mJODtG
|
|
2199
2218
|
nQNwPlZ+tyx24XVo
|
|
2200
2219
|
olObiSmdzVaMp7lH
|
|
2220
|
+
SruXHv6plKL9YuAW
|
|
2201
2221
|
cyp1GKV4Cl+eC8G/
|
|
2202
2222
|
Q1y/TzbtUQCqnotR
|
|
2203
2223
|
59Q2qOkuyTLzQDhd
|
|
@@ -2221,9 +2241,13 @@ EsbWnuADOq9qe/EZ
|
|
|
2221
2241
|
YfUSxQtq1+kpwW/W
|
|
2222
2242
|
QodzxvoJ6NPGhCgW
|
|
2223
2243
|
5/VK+O950efBio0t
|
|
2224
|
-
RA/lrkwxcTX80nxX
|
|
2225
2244
|
b85BMAFRHn10uX8y
|
|
2226
2245
|
vV2RgfFH2YFTYsly
|
|
2246
|
+
zcmIO3obSzUVHGtF
|
|
2247
|
+
tzsA68uFVAffHmec
|
|
2248
|
+
DkCU3OpaSa+rIVfU
|
|
2249
|
+
dcDjqpxVxivT46Rl
|
|
2250
|
+
QSgNcpNukC9CBNfe
|
|
2227
2251
|
`
|
|
2228
2252
|
.split(/\r?\n/)
|
|
2229
2253
|
.map(l => l.trim())
|
|
@@ -23,6 +23,7 @@ const timezonesList = require('timezones-list').default;
|
|
|
23
23
|
const { redis, submitQueue, notifyQueue, documentsQueue } = require('../db');
|
|
24
24
|
const { getByteSize, formatByteSize, getDuration, failAction, hasEnvValue, readEnvValue, httpAgent } = require('../tools');
|
|
25
25
|
const { llmPreProcess } = require('../llm-pre-process');
|
|
26
|
+
const { documentStoreFeatureEnabled } = require('../document-store');
|
|
26
27
|
const { locales } = require('../translations');
|
|
27
28
|
const { settingsSchema } = require('../schemas');
|
|
28
29
|
const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
|
|
@@ -217,7 +218,7 @@ function init(args) {
|
|
|
217
218
|
values,
|
|
218
219
|
|
|
219
220
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
220
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
221
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
221
222
|
},
|
|
222
223
|
{
|
|
223
224
|
layout: 'app'
|
|
@@ -307,7 +308,7 @@ function init(args) {
|
|
|
307
308
|
),
|
|
308
309
|
|
|
309
310
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
310
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
311
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
311
312
|
},
|
|
312
313
|
{
|
|
313
314
|
layout: 'app'
|
|
@@ -356,7 +357,7 @@ function init(args) {
|
|
|
356
357
|
errors,
|
|
357
358
|
|
|
358
359
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
359
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
360
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
360
361
|
},
|
|
361
362
|
{
|
|
362
363
|
layout: 'app'
|
|
@@ -16,7 +16,7 @@ const { REDIS_PREFIX } = require('../consts');
|
|
|
16
16
|
const { failAction } = require('../tools');
|
|
17
17
|
const { settingsSchema } = require('../schemas');
|
|
18
18
|
const { defaultMappings } = require('../es');
|
|
19
|
-
const { getESClient } = require('../document-store');
|
|
19
|
+
const { getESClient, documentStoreFeatureEnabled } = require('../document-store');
|
|
20
20
|
const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
|
|
21
21
|
|
|
22
22
|
const FIELD_TYPES = [
|
|
@@ -95,6 +95,12 @@ const configDocumentStoreSchema = {
|
|
|
95
95
|
function init(args) {
|
|
96
96
|
const { server } = args;
|
|
97
97
|
|
|
98
|
+
// Deprecated Document Store feature: when the gate is off, register no routes so that
|
|
99
|
+
// every /admin/config/document-store* page behaves like a regular 404.
|
|
100
|
+
if (!documentStoreFeatureEnabled) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
98
104
|
server.route({
|
|
99
105
|
method: 'GET',
|
|
100
106
|
path: '/admin/config/document-store',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "emailengine-app",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.71.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"productTitle": "EmailEngine",
|
|
6
6
|
"description": "Email Sync Engine",
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
"single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/6' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty",
|
|
12
12
|
"gmail": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=2 EENGINE_CORS_ORIGIN='*' node --inspect server --dbs.redis='redis://127.0.0.1:6379/11' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.gmail.txt | pino-pretty",
|
|
13
13
|
"test": "NODE_ENV=test grunt",
|
|
14
|
+
"test:unit": "NODE_ENV=test grunt test-unit",
|
|
15
|
+
"test:integration": "NODE_ENV=test grunt test-integration",
|
|
14
16
|
"lint": "npx eslint 'lib/**/*.js' 'workers/**/*.js' 'test/**/*.js' server.js Gruntfile.js",
|
|
15
17
|
"swagger": "./getswagger.sh",
|
|
16
18
|
"build-source": "rm -rf node_modules && npm install && rm -rf node_modules && npm ci --omit=dev && rm -rf node_modules/ace-builds node_modules/@postalsys/ee-client && ./update-info.sh",
|
|
@@ -56,21 +58,21 @@
|
|
|
56
58
|
"@hapi/vision": "7.0.3",
|
|
57
59
|
"@phc/pbkdf2": "1.1.14",
|
|
58
60
|
"@postalsys/bounce-classifier": "3.0.0",
|
|
59
|
-
"@postalsys/certs": "1.0.
|
|
61
|
+
"@postalsys/certs": "1.0.15",
|
|
60
62
|
"@postalsys/ee-client": "1.3.0",
|
|
61
|
-
"@postalsys/email-ai-tools": "1.13.
|
|
62
|
-
"@postalsys/email-text-tools": "2.4.
|
|
63
|
+
"@postalsys/email-ai-tools": "1.13.6",
|
|
64
|
+
"@postalsys/email-text-tools": "2.4.7",
|
|
63
65
|
"@postalsys/gettext": "4.1.1",
|
|
64
66
|
"@postalsys/joi-messages": "1.0.5",
|
|
65
67
|
"@postalsys/templates": "2.0.1",
|
|
66
|
-
"@sentry/node": "10.
|
|
68
|
+
"@sentry/node": "10.58.0",
|
|
67
69
|
"@simplewebauthn/browser": "13.3.0",
|
|
68
70
|
"@simplewebauthn/server": "13.3.1",
|
|
69
71
|
"@zone-eu/mailsplit": "5.4.12",
|
|
70
72
|
"@zone-eu/wild-config": "1.7.5",
|
|
71
73
|
"ace-builds": "1.44.0",
|
|
72
74
|
"base32.js": "0.1.0",
|
|
73
|
-
"bullmq": "5.78.
|
|
75
|
+
"bullmq": "5.78.1",
|
|
74
76
|
"compare-versions": "6.1.1",
|
|
75
77
|
"dotenv": "17.4.2",
|
|
76
78
|
"encoding-japanese": "2.2.0",
|
|
@@ -84,31 +86,31 @@
|
|
|
84
86
|
"html-to-text": "10.0.0",
|
|
85
87
|
"ical.js": "1.5.0",
|
|
86
88
|
"iconv-lite": "0.7.2",
|
|
87
|
-
"imapflow": "1.4.
|
|
89
|
+
"imapflow": "1.4.1",
|
|
88
90
|
"ioredfour": "1.4.1",
|
|
89
91
|
"ioredis": "5.11.1",
|
|
90
92
|
"ipaddr.js": "2.4.0",
|
|
91
|
-
"joi": "17.13.
|
|
93
|
+
"joi": "17.13.4",
|
|
92
94
|
"jquery": "4.0.0",
|
|
93
95
|
"libbase64": "1.3.0",
|
|
94
96
|
"libmime": "5.3.8",
|
|
95
97
|
"libqp": "2.1.1",
|
|
96
98
|
"license-checker": "25.0.1",
|
|
97
|
-
"mailparser": "3.9.
|
|
99
|
+
"mailparser": "3.9.10",
|
|
98
100
|
"marked": "9.1.6",
|
|
99
101
|
"minimist": "1.2.8",
|
|
100
102
|
"msgpack5": "6.0.2",
|
|
101
103
|
"murmurhash": "2.0.1",
|
|
102
104
|
"nanoid": "3.3.8",
|
|
103
|
-
"nodemailer": "
|
|
105
|
+
"nodemailer": "9.0.0",
|
|
104
106
|
"pino": "10.3.1",
|
|
105
107
|
"popper.js": "1.16.1",
|
|
106
108
|
"prom-client": "15.1.3",
|
|
107
109
|
"psl": "1.15.0",
|
|
108
|
-
"pubface": "1.1.
|
|
110
|
+
"pubface": "1.1.3",
|
|
109
111
|
"punycode.js": "2.3.1",
|
|
110
112
|
"qrcode": "1.5.4",
|
|
111
|
-
"smtp-server": "3.
|
|
113
|
+
"smtp-server": "3.19.0",
|
|
112
114
|
"socks": "2.8.9",
|
|
113
115
|
"speakeasy": "2.0.0",
|
|
114
116
|
"startbootstrap-sb-admin-2": "3.3.7",
|
|
@@ -118,15 +120,14 @@
|
|
|
118
120
|
},
|
|
119
121
|
"devDependencies": {
|
|
120
122
|
"@eslint/js": "10.0.1",
|
|
121
|
-
"acorn": "
|
|
122
|
-
"acorn-walk": "
|
|
123
|
+
"acorn": "8.17.0",
|
|
124
|
+
"acorn-walk": "8.3.5",
|
|
123
125
|
"chai": "4.3.10",
|
|
124
126
|
"eerawlog": "1.5.3",
|
|
125
|
-
"eslint": "10.
|
|
127
|
+
"eslint": "10.5.0",
|
|
126
128
|
"grunt": "1.6.2",
|
|
127
129
|
"grunt-cli": "1.5.0",
|
|
128
130
|
"grunt-shell-spawn": "0.5.0",
|
|
129
|
-
"grunt-wait": "0.3.0",
|
|
130
131
|
"pino-pretty": "13.0.0",
|
|
131
132
|
"prettier": "3.8.4",
|
|
132
133
|
"resedit": "3.0.2",
|
|
@@ -158,6 +159,8 @@
|
|
|
158
159
|
"node_modules/@postalsys/joi-messages/translations/*",
|
|
159
160
|
"node_modules/@postalsys/bounce-classifier/model/**/*",
|
|
160
161
|
"node_modules/jsdom/lib/jsdom/browser/default-stylesheet.css",
|
|
162
|
+
"node_modules/nodemailer/lib/well-known/*.json",
|
|
163
|
+
"node_modules/nodemailer/package.json",
|
|
161
164
|
"LICENSE_EMAILENGINE.txt",
|
|
162
165
|
"version-info.json",
|
|
163
166
|
"sbom.json"
|