emailengine-app 2.70.0 → 2.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/codeql.yml +3 -0
- package/.github/workflows/e2e.yml +56 -0
- package/.github/workflows/test.yml +81 -12
- package/.ncurc.js +20 -20
- package/CHANGELOG.md +25 -0
- package/Gruntfile.js +19 -23
- package/bin/emailengine.js +8 -1
- package/config/default.toml +5 -0
- package/config/e2e.toml +35 -0
- package/config/test.toml +5 -0
- package/data/google-crawlers.json +1 -1
- package/getswagger.sh +4 -0
- package/lib/account.js +31 -25
- package/lib/api-routes/message-routes.js +125 -121
- package/lib/auth-token.js +83 -0
- package/lib/delivery-error.js +62 -0
- package/lib/document-store.js +22 -1
- package/lib/email-client/base-client.js +3 -2
- package/lib/email-client/gmail-client.js +33 -1
- package/lib/email-client/imap/mailbox.js +2 -2
- package/lib/email-client/notification-handler.js +2 -2
- package/lib/export.js +12 -0
- package/lib/feature-flags.js +6 -0
- package/lib/imap-proxy-auth.js +81 -0
- package/lib/imapproxy/imap-server.js +8 -103
- package/lib/license-beacon.js +367 -0
- package/lib/logger.js +11 -1
- package/lib/oauth/gmail.js +3 -0
- package/lib/oauth/outlook.js +3 -0
- package/lib/oauth2-apps.js +100 -11
- package/lib/routes-ui.js +2 -1
- package/lib/smtp-auth.js +70 -0
- package/lib/sub-script.js +8 -2
- package/lib/tools.js +26 -2
- package/lib/ui-routes/admin-config-routes.js +4 -3
- package/lib/ui-routes/document-store-routes.js +7 -1
- package/package.json +30 -24
- package/playwright.config.js +45 -0
- package/sbom.json +1 -1
- package/server.js +30 -8
- package/static/licenses.html +108 -128
- package/test-coverage-plan.md +233 -0
- package/translations/de.mo +0 -0
- package/translations/de.po +154 -142
- package/translations/et.mo +0 -0
- package/translations/et.po +129 -131
- package/translations/fr.mo +0 -0
- package/translations/fr.po +133 -136
- package/translations/ja.mo +0 -0
- package/translations/ja.po +126 -129
- package/translations/messages.pot +37 -37
- package/translations/nl.mo +0 -0
- package/translations/nl.po +128 -130
- package/translations/pl.mo +0 -0
- package/translations/pl.po +125 -128
- package/views/dashboard.hbs +22 -0
- package/workers/api.js +22 -5
- package/workers/export.js +58 -43
- package/workers/smtp.js +5 -85
- package/workers/submit.js +2 -12
|
@@ -46,7 +46,7 @@ const idMapSchema = Joi.array()
|
|
|
46
46
|
.label('IdMapArray');
|
|
47
47
|
|
|
48
48
|
async function init(args) {
|
|
49
|
-
const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT } = args;
|
|
49
|
+
const { server, call, CORS_CONFIG, MAX_ATTACHMENT_SIZE, MAX_BODY_SIZE, MAX_PAYLOAD_TIMEOUT, documentStoreFeatureEnabled } = args;
|
|
50
50
|
|
|
51
51
|
// GET /v1/account/{account}/message/{message}/source - Download raw message
|
|
52
52
|
server.route({
|
|
@@ -1133,138 +1133,142 @@ async function init(args) {
|
|
|
1133
1133
|
});
|
|
1134
1134
|
|
|
1135
1135
|
// POST /v1/unified/search - Unified search for messages
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1136
|
+
// Deprecated Document Store feature: only register this endpoint when the feature is enabled,
|
|
1137
|
+
// otherwise it behaves like a regular 404.
|
|
1138
|
+
if (documentStoreFeatureEnabled) {
|
|
1139
|
+
server.route({
|
|
1140
|
+
method: 'POST',
|
|
1141
|
+
path: '/v1/unified/search',
|
|
1142
|
+
|
|
1143
|
+
async handler(request, h) {
|
|
1144
|
+
let accountObject = new Account({
|
|
1145
|
+
redis,
|
|
1146
|
+
call,
|
|
1147
|
+
secret: await getSecret(),
|
|
1148
|
+
esClient: await h.getESClient(request.logger),
|
|
1149
|
+
timeout: request.headers['x-ee-timeout']
|
|
1150
|
+
});
|
|
1151
|
+
|
|
1152
|
+
let extraValidationErrors = [];
|
|
1148
1153
|
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
|
|
1154
|
+
for (let key of ['seq', 'modseq']) {
|
|
1155
|
+
if (request.payload.search && key in request.payload.search) {
|
|
1156
|
+
extraValidationErrors.push({ message: 'Not available when using Document Store', context: { key } });
|
|
1157
|
+
}
|
|
1154
1158
|
}
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
if (extraValidationErrors.length) {
|
|
1158
|
-
let error = new Error('Input validation failed');
|
|
1159
|
-
error.details = extraValidationErrors;
|
|
1160
|
-
return failAction(request, h, error);
|
|
1161
|
-
}
|
|
1162
1159
|
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
|
-
try {
|
|
1171
|
-
return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
|
|
1172
|
-
} catch (err) {
|
|
1173
|
-
handleError(request, err);
|
|
1174
|
-
}
|
|
1175
|
-
},
|
|
1176
|
-
options: {
|
|
1177
|
-
description: 'Unified search for messages',
|
|
1178
|
-
notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
|
|
1179
|
-
tags: ['api', 'Deprecated endpoints (Document Store)'],
|
|
1160
|
+
if (extraValidationErrors.length) {
|
|
1161
|
+
let error = new Error('Input validation failed');
|
|
1162
|
+
error.details = extraValidationErrors;
|
|
1163
|
+
return failAction(request, h, error);
|
|
1164
|
+
}
|
|
1180
1165
|
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1166
|
+
let documentStoreEnabled = await settings.get('documentStoreEnabled');
|
|
1167
|
+
if (!documentStoreEnabled) {
|
|
1168
|
+
let error = new Error('Document store not enabled');
|
|
1169
|
+
error.details = extraValidationErrors;
|
|
1170
|
+
return failAction(request, h, error);
|
|
1184
1171
|
}
|
|
1185
|
-
},
|
|
1186
1172
|
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1173
|
+
try {
|
|
1174
|
+
return await accountObject.searchMessages(Object.assign({ documentStore: true }, request.query, request.payload), { unified: true });
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
handleError(request, err);
|
|
1177
|
+
}
|
|
1190
1178
|
},
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1179
|
+
options: {
|
|
1180
|
+
description: 'Unified search for messages',
|
|
1181
|
+
notes: 'Filter messages from the Document Store for multiple accounts or paths. Document Store must be enabled for the unified search to work.',
|
|
1182
|
+
tags: ['api', 'Deprecated endpoints (Document Store)'],
|
|
1183
|
+
|
|
1184
|
+
plugins: {
|
|
1185
|
+
'hapi-swagger': {
|
|
1186
|
+
responses: errorResponses(400, 401, 403, 429, 500)
|
|
1187
|
+
}
|
|
1198
1188
|
},
|
|
1199
|
-
failAction,
|
|
1200
1189
|
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
.
|
|
1217
|
-
|
|
1190
|
+
auth: {
|
|
1191
|
+
strategy: 'api-token',
|
|
1192
|
+
mode: 'required'
|
|
1193
|
+
},
|
|
1194
|
+
cors: CORS_CONFIG,
|
|
1195
|
+
|
|
1196
|
+
validate: {
|
|
1197
|
+
options: {
|
|
1198
|
+
stripUnknown: false,
|
|
1199
|
+
abortEarly: false,
|
|
1200
|
+
convert: true
|
|
1201
|
+
},
|
|
1202
|
+
failAction,
|
|
1203
|
+
|
|
1204
|
+
query: Joi.object({
|
|
1205
|
+
page: Joi.number()
|
|
1206
|
+
.integer()
|
|
1207
|
+
.min(0)
|
|
1208
|
+
.max(1024 * 1024)
|
|
1209
|
+
.default(0)
|
|
1210
|
+
.example(0)
|
|
1211
|
+
.description('Page number (zero indexed, so use 0 for first page)'),
|
|
1212
|
+
pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'),
|
|
1213
|
+
exposeQuery: Joi.boolean()
|
|
1214
|
+
.truthy('Y', 'true', '1')
|
|
1215
|
+
.falsy('N', 'false', 0)
|
|
1216
|
+
.description('If enabled then includes the combined query (as the documentStoreQuery field) in the response for debugging')
|
|
1217
|
+
.label('exposeQuery')
|
|
1218
|
+
.optional()
|
|
1219
|
+
.meta({ swaggerHidden: true })
|
|
1220
|
+
}),
|
|
1218
1221
|
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1222
|
+
payload: Joi.object({
|
|
1223
|
+
accounts: Joi.array()
|
|
1224
|
+
.items(Joi.string().empty('').trim().max(256).example('example'))
|
|
1225
|
+
.single()
|
|
1226
|
+
.description('Optional list of account ID values')
|
|
1227
|
+
.label('UnifiedSearchAccounts'),
|
|
1228
|
+
paths: Joi.array()
|
|
1229
|
+
.items(Joi.string().optional().example('INBOX'))
|
|
1230
|
+
.single()
|
|
1231
|
+
.description('Optional list of mailbox folder paths or specialUse flags')
|
|
1232
|
+
.label('UnifiedSearchPaths'),
|
|
1233
|
+
search: searchSchema,
|
|
1234
|
+
documentQuery: Joi.object().min(1).description('Document Store query').label('DocumentQuery').unknown().meta({ swaggerHidden: true })
|
|
1235
|
+
}).label('UnifiedSearchQuery')
|
|
1236
|
+
},
|
|
1234
1237
|
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1238
|
+
response: {
|
|
1239
|
+
schema: Joi.object({
|
|
1240
|
+
total: Joi.number()
|
|
1241
|
+
.integer()
|
|
1242
|
+
.example(120)
|
|
1243
|
+
.description('Total number of matching messages (capped at 10000 by the Document Store)')
|
|
1244
|
+
.label('UnifiedTotalNumber'),
|
|
1245
|
+
page: Joi.number().integer().example(0).description('Current page number (zero-based)').label('UnifiedPageNumber'),
|
|
1246
|
+
pages: Joi.number().integer().example(24).description('Total number of pages available').label('UnifiedPagesNumber'),
|
|
1247
|
+
accounts: Joi.array()
|
|
1248
|
+
.items(Joi.string().example('example'))
|
|
1249
|
+
.description('Account filter used for the search, if provided')
|
|
1250
|
+
.label('UnifiedSearchAccountsEcho'),
|
|
1251
|
+
paths: Joi.array()
|
|
1252
|
+
.items(Joi.string().example('INBOX'))
|
|
1253
|
+
.description('Path filter used for the search, if provided')
|
|
1254
|
+
.label('UnifiedSearchPathsEcho'),
|
|
1255
|
+
messages: Joi.array()
|
|
1256
|
+
.items(
|
|
1257
|
+
messageEntrySchema
|
|
1258
|
+
.keys({
|
|
1259
|
+
account: accountIdSchema.description('Account ID this message belongs to')
|
|
1260
|
+
})
|
|
1261
|
+
.unknown()
|
|
1262
|
+
.label('UnifiedMessageListEntry')
|
|
1263
|
+
)
|
|
1264
|
+
.label('UnifiedPageMessages'),
|
|
1265
|
+
documentStoreQuery: documentStoreQuerySchema
|
|
1266
|
+
}).label('UnifiedSearchResponse'),
|
|
1267
|
+
failAction: 'log'
|
|
1268
|
+
}
|
|
1265
1269
|
}
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1270
|
+
});
|
|
1271
|
+
}
|
|
1268
1272
|
|
|
1269
1273
|
// GET /v1/account/{account}/text/{text} - Retrieve message text
|
|
1270
1274
|
server.route({
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Shared token validation for the SMTP and IMAP-proxy submission servers. Both
|
|
4
|
+
// servers accept a 64-char hex API token as the password and apply the same
|
|
5
|
+
// checks (account binding, scope, IP allowlist). The logic lives here so the two
|
|
6
|
+
// auth handlers (lib/smtp-auth.js, lib/imap-proxy-auth.js) cannot drift apart -
|
|
7
|
+
// a token security-policy change is made once and applies to both surfaces.
|
|
8
|
+
|
|
9
|
+
const logger = require('./logger');
|
|
10
|
+
const tokens = require('./tokens');
|
|
11
|
+
const { matchIp } = require('./utils/network');
|
|
12
|
+
|
|
13
|
+
// Reason-specific denial messages, shared verbatim by both auth handlers. The
|
|
14
|
+
// generic "failed to authenticate" fallback and any protocol-specific error
|
|
15
|
+
// decoration are left to each caller.
|
|
16
|
+
const REASON_MESSAGES = {
|
|
17
|
+
username: 'Access denied, invalid username',
|
|
18
|
+
scope: 'Access denied, invalid scope',
|
|
19
|
+
ip: 'Access denied, traffic not accepted from this IP'
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Validates a 64-char hex API token supplied as a server password.
|
|
24
|
+
*
|
|
25
|
+
* Performs the token lookup plus the account-binding, scope and IP-allowlist
|
|
26
|
+
* checks. Does NOT throw - the caller maps the returned reason to its own
|
|
27
|
+
* protocol-specific error (SMTP and IMAP use different response shapes).
|
|
28
|
+
*
|
|
29
|
+
* @param {Object} opts
|
|
30
|
+
* @param {String} opts.password - supplied password (candidate token)
|
|
31
|
+
* @param {String} opts.account - username the client authenticated as
|
|
32
|
+
* @param {String} opts.requiredScope - scope the token must hold ('smtp' | 'imap-proxy')
|
|
33
|
+
* @param {String} opts.remoteAddress - client IP, checked against token restrictions
|
|
34
|
+
* @returns {Promise<{authenticated: Boolean, reason: (null|'username'|'scope'|'ip')}>}
|
|
35
|
+
*/
|
|
36
|
+
async function validateAuthToken({ password, account, requiredScope, remoteAddress }) {
|
|
37
|
+
if (!/^[0-9a-f]{64}$/i.test(password)) {
|
|
38
|
+
return { authenticated: false, reason: null };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let tokenData;
|
|
42
|
+
try {
|
|
43
|
+
tokenData = await tokens.get(password, false, { log: true, remoteAddress });
|
|
44
|
+
} catch (err) {
|
|
45
|
+
logger.error({ msg: 'Failed to fetch token', err });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (!tokenData) {
|
|
49
|
+
return { authenticated: false, reason: null };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if (tokenData.account && tokenData.account !== account) {
|
|
53
|
+
return { authenticated: false, reason: 'username' };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (tokenData.scopes && !tokenData.scopes.includes(requiredScope) && !tokenData.scopes.includes('*')) {
|
|
57
|
+
logger.error({
|
|
58
|
+
msg: 'Trying to use invalid scope for a token',
|
|
59
|
+
tokenAccount: tokenData.account,
|
|
60
|
+
tokenId: tokenData.id,
|
|
61
|
+
account,
|
|
62
|
+
requestedScope: requiredScope,
|
|
63
|
+
scopes: tokenData.scopes
|
|
64
|
+
});
|
|
65
|
+
return { authenticated: false, reason: 'scope' };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(remoteAddress, tokenData.restrictions.addresses)) {
|
|
69
|
+
logger.error({
|
|
70
|
+
msg: 'Trying to use invalid IP for a token',
|
|
71
|
+
tokenAccount: tokenData.account,
|
|
72
|
+
tokenId: tokenData.id,
|
|
73
|
+
account,
|
|
74
|
+
remoteAddress,
|
|
75
|
+
addressAllowlist: tokenData.restrictions.addresses
|
|
76
|
+
});
|
|
77
|
+
return { authenticated: false, reason: 'ip' };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { authenticated: true, reason: null };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
module.exports = { validateAuthToken, REASON_MESSAGES };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Classification of outbound delivery errors for the submit worker.
|
|
4
|
+
//
|
|
5
|
+
// This logic lives in its own module so it can be unit tested without booting
|
|
6
|
+
// the BullMQ submit worker (workers/submit.js connects to Redis and expects a
|
|
7
|
+
// worker-thread parentPort at require time).
|
|
8
|
+
|
|
9
|
+
// Nodemailer/SMTP error codes that represent permanent failures. A message that
|
|
10
|
+
// fails with one of these will fail again on every retry, so the job is
|
|
11
|
+
// discarded instead of being retried.
|
|
12
|
+
const NON_RETRYABLE_CODES = new Set([
|
|
13
|
+
'EAUTH', // authentication failed
|
|
14
|
+
'ENOAUTH', // no credentials provided
|
|
15
|
+
'EOAUTH2', // OAuth2 token failure
|
|
16
|
+
'ETLS', // TLS handshake failed
|
|
17
|
+
'EENVELOPE', // invalid sender/recipients
|
|
18
|
+
'EMESSAGE', // message content error
|
|
19
|
+
'EPROTOCOL' // SMTP protocol mismatch
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Determines whether a delivery error is permanent (must not be retried).
|
|
24
|
+
*
|
|
25
|
+
* An error is permanent when either:
|
|
26
|
+
* - the SMTP status code is a 5xx other than 503 (503 is treated as a
|
|
27
|
+
* transient "try again later" response), or
|
|
28
|
+
* - the error carries one of the NON_RETRYABLE_CODES.
|
|
29
|
+
*
|
|
30
|
+
* @param {Object} err - Error thrown during submission
|
|
31
|
+
* @param {Number} [err.statusCode] - SMTP/HTTP status code
|
|
32
|
+
* @param {String} [err.code] - Nodemailer error code
|
|
33
|
+
* @returns {Boolean} True if the error should never be retried
|
|
34
|
+
*/
|
|
35
|
+
function isPermanentDeliveryError(err) {
|
|
36
|
+
if (!err) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
const isPermanentSmtp = err.statusCode >= 500 && err.statusCode !== 503;
|
|
40
|
+
const isPermanentCode = NON_RETRYABLE_CODES.has(err.code);
|
|
41
|
+
return isPermanentSmtp || isPermanentCode;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Determines whether the submit worker should discard a job (stop retrying).
|
|
46
|
+
*
|
|
47
|
+
* A job is discarded only when the error is permanent AND there are still
|
|
48
|
+
* attempts remaining. When attempts are already exhausted, BullMQ will fail the
|
|
49
|
+
* job naturally, so there is nothing to discard.
|
|
50
|
+
*
|
|
51
|
+
* @param {Object} err - Error thrown during submission
|
|
52
|
+
* @param {Object} job - BullMQ job
|
|
53
|
+
* @param {Number} job.attemptsMade - Attempts already made
|
|
54
|
+
* @param {Object} job.opts - Job options
|
|
55
|
+
* @param {Number} job.opts.attempts - Configured max attempts
|
|
56
|
+
* @returns {Boolean} True if the job should be discarded
|
|
57
|
+
*/
|
|
58
|
+
function shouldDiscardJob(err, job) {
|
|
59
|
+
return isPermanentDeliveryError(err) && job.attemptsMade < job.opts.attempts;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
module.exports = { NON_RETRYABLE_CODES, isPermanentDeliveryError, shouldDiscardJob };
|
package/lib/document-store.js
CHANGED
|
@@ -1,12 +1,33 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const config = require('@zone-eu/wild-config');
|
|
3
4
|
const settings = require('./settings');
|
|
4
5
|
const { Client: ElasticSearch } = require('@elastic/elasticsearch');
|
|
5
6
|
const { ensureIndex } = require('./es');
|
|
7
|
+
const { hasEnvValue, readEnvValue, getBoolean } = require('./tools');
|
|
8
|
+
|
|
9
|
+
// Deployment-level gate for the deprecated Document Store feature. When this is false the
|
|
10
|
+
// "documents" worker is not spawned, document-store-only endpoints are not registered, and
|
|
11
|
+
// all runtime document-store code takes its existing "disabled" path. Set via the
|
|
12
|
+
// --documentStore.enabled CLI flag / [documentStore] enabled config or EENGINE_DOCUMENT_STORE_ENABLED.
|
|
13
|
+
const documentStoreFeatureEnabled = hasEnvValue('EENGINE_DOCUMENT_STORE_ENABLED')
|
|
14
|
+
? getBoolean(readEnvValue('EENGINE_DOCUMENT_STORE_ENABLED'))
|
|
15
|
+
: getBoolean(config.documentStore && config.documentStore.enabled);
|
|
16
|
+
|
|
17
|
+
// Effective runtime state: the feature must be available (gate) AND enabled in settings.
|
|
18
|
+
// When the gate is off this is always false, so callers reuse the already-tested
|
|
19
|
+
// "documentStoreEnabled is false" code paths.
|
|
20
|
+
const isDocumentStoreEnabled = async () => documentStoreFeatureEnabled && !!(await settings.get('documentStoreEnabled'));
|
|
6
21
|
|
|
7
22
|
const clientCache = { version: -1, config: false, client: false, index: false };
|
|
8
23
|
|
|
9
24
|
const getESClient = async logger => {
|
|
25
|
+
// Feature gate is off: behave exactly as a disabled document store. Checked before any
|
|
26
|
+
// Redis access so disabled deployments do not pay a round-trip on every getESClient call.
|
|
27
|
+
if (!documentStoreFeatureEnabled) {
|
|
28
|
+
return clientCache;
|
|
29
|
+
}
|
|
30
|
+
|
|
10
31
|
const documentStoreVersion = (await settings.get('documentStoreVersion')) || 0;
|
|
11
32
|
if (clientCache.version === documentStoreVersion) {
|
|
12
33
|
return clientCache;
|
|
@@ -51,4 +72,4 @@ const getESClient = async logger => {
|
|
|
51
72
|
return clientCache;
|
|
52
73
|
};
|
|
53
74
|
|
|
54
|
-
module.exports = { getESClient };
|
|
75
|
+
module.exports = { getESClient, documentStoreFeatureEnabled, isDocumentStoreEnabled };
|
|
@@ -4,6 +4,7 @@ const { threadId: workerThreadId } = require('worker_threads');
|
|
|
4
4
|
const crypto = require('crypto');
|
|
5
5
|
const logger = require('../logger');
|
|
6
6
|
const settings = require('../settings');
|
|
7
|
+
const { isDocumentStoreEnabled } = require('../document-store');
|
|
7
8
|
const msgpack = require('msgpack5')();
|
|
8
9
|
const { templates } = require('../templates');
|
|
9
10
|
const { Gateway } = require('../gateway');
|
|
@@ -1076,7 +1077,7 @@ class BaseClient {
|
|
|
1076
1077
|
// Resolve reference and update reference/in-reply-to headers
|
|
1077
1078
|
if (data.reference && data.reference.message) {
|
|
1078
1079
|
// Try document store first if enabled
|
|
1079
|
-
if (data.reference.documentStore && (await
|
|
1080
|
+
if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
|
|
1080
1081
|
try {
|
|
1081
1082
|
referencedMessage = await this.accountObject.getMessage(data.reference.message, {
|
|
1082
1083
|
documentStore: true,
|
|
@@ -1578,7 +1579,7 @@ class BaseClient {
|
|
|
1578
1579
|
// Resolve reference and update reference/in-reply-to headers
|
|
1579
1580
|
if (data.reference && data.reference.message) {
|
|
1580
1581
|
// Try document store first if enabled
|
|
1581
|
-
if (data.reference.documentStore && (await
|
|
1582
|
+
if (data.reference.documentStore && (await isDocumentStoreEnabled())) {
|
|
1582
1583
|
try {
|
|
1583
1584
|
referencedMessage = await this.accountObject.getMessage(data.reference.message, {
|
|
1584
1585
|
documentStore: true,
|
|
@@ -257,6 +257,9 @@ class GmailClient extends BaseClient {
|
|
|
257
257
|
|
|
258
258
|
this.processingHistory = null;
|
|
259
259
|
this.renewWatchTimer = null;
|
|
260
|
+
// Set once renewWatch() has logged that the linked Pub/Sub app is missing its topic/IAM
|
|
261
|
+
// markers, so the warning is emitted once per connection instead of every renewal cycle.
|
|
262
|
+
this._loggedMissingWatchMarkers = false;
|
|
260
263
|
|
|
261
264
|
this.cachedLabels = null;
|
|
262
265
|
this.cachedDetailedLabels = null;
|
|
@@ -2047,6 +2050,9 @@ class GmailClient extends BaseClient {
|
|
|
2047
2050
|
watchExpiration,
|
|
2048
2051
|
watchFailure: null
|
|
2049
2052
|
});
|
|
2053
|
+
// Markers are present and the watch armed - allow the missing-markers warning
|
|
2054
|
+
// to fire again if the app's markers ever disappear.
|
|
2055
|
+
this._loggedMissingWatchMarkers = false;
|
|
2050
2056
|
this.logger.info({
|
|
2051
2057
|
msg: 'Renewed Gmail pubsub watch',
|
|
2052
2058
|
account: this.account,
|
|
@@ -2067,6 +2073,23 @@ class GmailClient extends BaseClient {
|
|
|
2067
2073
|
err
|
|
2068
2074
|
});
|
|
2069
2075
|
}
|
|
2076
|
+
} else {
|
|
2077
|
+
// pubSubApp is linked and a renewal is due, but the topic/IAM markers required to
|
|
2078
|
+
// arm the watch are not recorded on the linked Pub/Sub app. This is the silent
|
|
2079
|
+
// failure that leaves Gmail push disabled (e.g. Pub/Sub resources pre-provisioned in
|
|
2080
|
+
// GCP, so ensurePubsub never persisted the markers). Surface it once per connection -
|
|
2081
|
+
// the renewal timer re-fires ~hourly and this branch never updates lastWatch, so an
|
|
2082
|
+
// unguarded warning would otherwise repeat every cycle.
|
|
2083
|
+
if (!this._loggedMissingWatchMarkers) {
|
|
2084
|
+
this._loggedMissingWatchMarkers = true;
|
|
2085
|
+
this.logger.warn({
|
|
2086
|
+
msg: 'Skipping Gmail watch renewal: Pub/Sub app linked but topic/IAM markers are not recorded',
|
|
2087
|
+
account: this.account,
|
|
2088
|
+
pubSubApp: accountData._app?.pubSubApp,
|
|
2089
|
+
hasTopic: !!appData?.pubSubTopic,
|
|
2090
|
+
hasIamPolicy: !!appData?.pubSubIamPolicy
|
|
2091
|
+
});
|
|
2092
|
+
}
|
|
2070
2093
|
}
|
|
2071
2094
|
}
|
|
2072
2095
|
}
|
|
@@ -3076,4 +3099,13 @@ class GmailClient extends BaseClient {
|
|
|
3076
3099
|
}
|
|
3077
3100
|
}
|
|
3078
3101
|
|
|
3079
|
-
|
|
3102
|
+
// PageCursor and the label maps are exported for unit testing. The maps are
|
|
3103
|
+
// exported as frozen copies so importers (or tests) cannot mutate the live
|
|
3104
|
+
// objects used by the running Gmail client.
|
|
3105
|
+
module.exports = {
|
|
3106
|
+
GmailClient,
|
|
3107
|
+
PageCursor,
|
|
3108
|
+
SKIP_LABELS: Object.freeze([...SKIP_LABELS]),
|
|
3109
|
+
SYSTEM_LABELS: Object.freeze({ ...SYSTEM_LABELS }),
|
|
3110
|
+
SYSTEM_NAMES: Object.freeze({ ...SYSTEM_NAMES })
|
|
3111
|
+
};
|
|
@@ -15,7 +15,7 @@ const ical = require('ical.js');
|
|
|
15
15
|
const addressparser = require('nodemailer/lib/addressparser');
|
|
16
16
|
const { llmPreProcess } = require('../../llm-pre-process');
|
|
17
17
|
|
|
18
|
-
const { getESClient } = require('../../document-store');
|
|
18
|
+
const { getESClient, isDocumentStoreEnabled } = require('../../document-store');
|
|
19
19
|
|
|
20
20
|
const {
|
|
21
21
|
MESSAGE_NEW_NOTIFY,
|
|
@@ -1958,7 +1958,7 @@ class Mailbox {
|
|
|
1958
1958
|
async publishSyncedEvents(storedStatus) {
|
|
1959
1959
|
let messageFetchOptions = {};
|
|
1960
1960
|
|
|
1961
|
-
let documentStoreEnabled = await
|
|
1961
|
+
let documentStoreEnabled = await isDocumentStoreEnabled();
|
|
1962
1962
|
|
|
1963
1963
|
// Configure text fetching
|
|
1964
1964
|
let notifyText = await settings.get('notifyText');
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const { parentPort } = require('worker_threads');
|
|
4
4
|
const logger = require('../logger');
|
|
5
5
|
const { webhooks: Webhooks } = require('../webhooks');
|
|
6
|
-
const { getESClient } = require('../document-store');
|
|
6
|
+
const { getESClient, isDocumentStoreEnabled } = require('../document-store');
|
|
7
7
|
const { getThread } = require('../threads');
|
|
8
8
|
const settings = require('../settings');
|
|
9
9
|
|
|
@@ -126,7 +126,7 @@ class NotificationHandler {
|
|
|
126
126
|
return false;
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
-
return await
|
|
129
|
+
return await isDocumentStoreEnabled();
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
/**
|
package/lib/export.js
CHANGED
|
@@ -99,6 +99,17 @@ function isFolderMissingError(err) {
|
|
|
99
99
|
return ['FolderNotFound', 'NotFound'].includes(errCode);
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
// Per-message errors worth retrying during a batch export: rate limits, a dropped batch response
|
|
103
|
+
// (Outlook reports these as EMISSING_RESPONSE), and the same transient network/5xx/timeout errors
|
|
104
|
+
// isTransientError already covers. Used by the export worker's API-account batch fetch path so a
|
|
105
|
+
// single transient blip does not fail an entire multi-message export.
|
|
106
|
+
function isRetryableError(err) {
|
|
107
|
+
if (err.statusCode === 429 || ['rateLimitExceeded', 'userRateLimitExceeded', 'EMISSING_RESPONSE'].includes(err.code)) {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
return isTransientError(err);
|
|
111
|
+
}
|
|
112
|
+
|
|
102
113
|
async function tryAddToActiveSet(account, exportId) {
|
|
103
114
|
const maxConcurrent = (await settings.get('exportMaxConcurrent')) || DEFAULT_EXPORT_MAX_CONCURRENT;
|
|
104
115
|
const maxGlobalConcurrent = (await settings.get('exportMaxGlobalConcurrent')) || DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT;
|
|
@@ -618,5 +629,6 @@ module.exports = {
|
|
|
618
629
|
isTransientError,
|
|
619
630
|
isSkippableError,
|
|
620
631
|
isFolderMissingError,
|
|
632
|
+
isRetryableError,
|
|
621
633
|
DEFAULT_EXPORT_MAX_MESSAGE_SIZE
|
|
622
634
|
};
|
package/lib/feature-flags.js
CHANGED
|
@@ -34,6 +34,12 @@ for (let key of Object.keys(process.env)) {
|
|
|
34
34
|
module.exports = {
|
|
35
35
|
enabled(key) {
|
|
36
36
|
return FEATURES.has(formatFeatureKey(key));
|
|
37
|
+
},
|
|
38
|
+
|
|
39
|
+
// Sorted list of the currently enabled feature flag keys. Used by the license beacon
|
|
40
|
+
// to report which EENGINE_FEATURE_* flags are active on this instance.
|
|
41
|
+
listEnabled() {
|
|
42
|
+
return Array.from(FEATURES).sort();
|
|
37
43
|
}
|
|
38
44
|
};
|
|
39
45
|
|