emailengine-app 2.65.0 → 2.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy.yml +8 -8
- package/.github/workflows/release.yaml +9 -9
- package/.github/workflows/test.yml +2 -2
- package/CHANGELOG.md +53 -0
- package/bin/emailengine.js +3 -0
- package/data/google-crawlers.json +7 -1
- package/lib/account.js +35 -29
- package/lib/consts.js +5 -0
- package/lib/email-client/gmail-client.js +23 -27
- package/lib/email-client/imap/mailbox.js +46 -19
- package/lib/email-client/imap/sync-operations.js +51 -19
- package/lib/email-client/imap-client.js +28 -5
- package/lib/email-client/outlook-client.js +155 -1
- package/lib/oauth/gmail.js +52 -1
- package/lib/passkeys.js +206 -0
- package/lib/routes-ui.js +522 -21
- package/lib/ui-routes/oauth-routes.js +6 -1
- package/package.json +13 -11
- package/sbom.json +1 -1
- package/static/js/login-passkey.js +75 -0
- package/static/js/passkey-register.js +107 -0
- package/static/licenses.html +238 -38
- package/static/vendor/handlebars/handlebars.min-v4.7.9.js +29 -0
- package/static/vendor/simplewebauthn/browser.min.js +2 -0
- package/translations/de.mo +0 -0
- package/translations/de.po +91 -53
- package/translations/en.mo +0 -0
- package/translations/en.po +84 -52
- package/translations/et.mo +0 -0
- package/translations/et.po +95 -60
- package/translations/fr.mo +0 -0
- package/translations/fr.po +102 -65
- package/translations/ja.mo +0 -0
- package/translations/ja.po +93 -57
- package/translations/messages.pot +101 -76
- package/translations/nl.mo +0 -0
- package/translations/nl.po +92 -56
- package/translations/pl.mo +0 -0
- package/translations/pl.po +106 -70
- package/views/account/login.hbs +35 -25
- package/views/account/password.hbs +4 -4
- package/views/account/security.hbs +101 -12
- package/views/account/totp.hbs +3 -3
- package/views/config/oauth/app.hbs +25 -0
- package/views/layout/app.hbs +2 -2
- package/views/layout/login.hbs +6 -1
- package/views/oauth-scope-error.hbs +29 -0
- package/workers/api.js +81 -3
- package/workers/imap.js +4 -0
- package/static/vendor/handlebars/handlebars.min-v4.7.7.js +0 -29
|
@@ -16,7 +16,7 @@ jobs:
|
|
|
16
16
|
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout
|
|
19
|
-
uses: actions/checkout@
|
|
19
|
+
uses: actions/checkout@v6
|
|
20
20
|
|
|
21
21
|
- name: Use Node.js 24
|
|
22
22
|
uses: actions/setup-node@v4
|
|
@@ -63,27 +63,27 @@ jobs:
|
|
|
63
63
|
|
|
64
64
|
steps:
|
|
65
65
|
- name: Checkout
|
|
66
|
-
uses: actions/checkout@
|
|
66
|
+
uses: actions/checkout@v6
|
|
67
67
|
|
|
68
68
|
- name: Set up QEMU
|
|
69
|
-
uses: docker/setup-qemu-action@
|
|
69
|
+
uses: docker/setup-qemu-action@v4
|
|
70
70
|
with:
|
|
71
71
|
platforms: 'arm64,arm'
|
|
72
72
|
|
|
73
73
|
- name: Set up Docker Buildx
|
|
74
74
|
id: buildx
|
|
75
|
-
uses: docker/setup-buildx-action@
|
|
75
|
+
uses: docker/setup-buildx-action@v4
|
|
76
76
|
with:
|
|
77
77
|
platforms: linux/amd64,linux/arm64/v8
|
|
78
78
|
|
|
79
79
|
- name: Login to Docker Hub
|
|
80
|
-
uses: docker/login-action@
|
|
80
|
+
uses: docker/login-action@v4
|
|
81
81
|
with:
|
|
82
82
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
83
83
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
84
84
|
|
|
85
85
|
- name: Login to GHCR
|
|
86
|
-
uses: docker/login-action@
|
|
86
|
+
uses: docker/login-action@v4
|
|
87
87
|
with:
|
|
88
88
|
registry: ghcr.io
|
|
89
89
|
username: ${{ github.repository_owner }}
|
|
@@ -91,7 +91,7 @@ jobs:
|
|
|
91
91
|
|
|
92
92
|
- name: Docker meta
|
|
93
93
|
id: meta
|
|
94
|
-
uses: docker/metadata-action@
|
|
94
|
+
uses: docker/metadata-action@v6
|
|
95
95
|
with:
|
|
96
96
|
images: |
|
|
97
97
|
${{ github.repository }}
|
|
@@ -100,7 +100,7 @@ jobs:
|
|
|
100
100
|
type=raw,value=latest,enable=true
|
|
101
101
|
|
|
102
102
|
- name: Build and push
|
|
103
|
-
uses: docker/build-push-action@
|
|
103
|
+
uses: docker/build-push-action@v7
|
|
104
104
|
with:
|
|
105
105
|
context: .
|
|
106
106
|
platforms: ${{ steps.buildx.outputs.platforms }}
|
|
@@ -27,14 +27,14 @@ jobs:
|
|
|
27
27
|
release-type: node
|
|
28
28
|
|
|
29
29
|
# The logic below handles the npm publication:
|
|
30
|
-
- uses: actions/checkout@
|
|
30
|
+
- uses: actions/checkout@v6
|
|
31
31
|
# these if statements ensure that a publication only occurs when
|
|
32
32
|
# a new release is created:
|
|
33
33
|
if: ${{ steps.release.outputs.release_created }}
|
|
34
34
|
|
|
35
35
|
- uses: actions/setup-node@v4
|
|
36
36
|
with:
|
|
37
|
-
node-version:
|
|
37
|
+
node-version: 24
|
|
38
38
|
registry-url: "https://registry.npmjs.org"
|
|
39
39
|
if: ${{ steps.release.outputs.release_created }}
|
|
40
40
|
|
|
@@ -50,7 +50,7 @@ jobs:
|
|
|
50
50
|
contents: read
|
|
51
51
|
id-token: write
|
|
52
52
|
steps:
|
|
53
|
-
- uses: actions/checkout@
|
|
53
|
+
- uses: actions/checkout@v6
|
|
54
54
|
- uses: actions/setup-node@v4
|
|
55
55
|
with:
|
|
56
56
|
node-version: 24
|
|
@@ -66,34 +66,34 @@ jobs:
|
|
|
66
66
|
steps:
|
|
67
67
|
- run: echo version v${{needs.release_please.outputs.major}}.${{needs.release_please.outputs.minor}}.${{needs.release_please.outputs.patch}}
|
|
68
68
|
|
|
69
|
-
- uses: actions/checkout@
|
|
69
|
+
- uses: actions/checkout@v6
|
|
70
70
|
|
|
71
71
|
- name: Set up QEMU
|
|
72
|
-
uses: docker/setup-qemu-action@
|
|
72
|
+
uses: docker/setup-qemu-action@v4
|
|
73
73
|
with:
|
|
74
74
|
platforms: "arm64,arm"
|
|
75
75
|
|
|
76
76
|
- name: Set up Docker Buildx
|
|
77
77
|
id: buildx
|
|
78
|
-
uses: docker/setup-buildx-action@
|
|
78
|
+
uses: docker/setup-buildx-action@v4
|
|
79
79
|
with:
|
|
80
80
|
platforms: linux/amd64,linux/arm64/v8
|
|
81
81
|
|
|
82
82
|
- name: Login to DockerHub
|
|
83
|
-
uses: docker/login-action@
|
|
83
|
+
uses: docker/login-action@v4
|
|
84
84
|
with:
|
|
85
85
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
86
86
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
87
87
|
|
|
88
88
|
- name: Login to GHCR
|
|
89
|
-
uses: docker/login-action@
|
|
89
|
+
uses: docker/login-action@v4
|
|
90
90
|
with:
|
|
91
91
|
registry: ghcr.io
|
|
92
92
|
username: ${{ github.repository_owner }}
|
|
93
93
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
94
94
|
|
|
95
95
|
- name: Build and push
|
|
96
|
-
uses: docker/build-push-action@
|
|
96
|
+
uses: docker/build-push-action@v7
|
|
97
97
|
with:
|
|
98
98
|
context: .
|
|
99
99
|
platforms: ${{ steps.buildx.outputs.platforms }}
|
|
@@ -14,7 +14,7 @@ jobs:
|
|
|
14
14
|
runs-on: ubuntu-24.04
|
|
15
15
|
# Service containers to run with `container-job`
|
|
16
16
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
18
|
|
|
19
19
|
- name: Use Node.js 24
|
|
20
20
|
uses: actions/setup-node@v4
|
|
@@ -49,7 +49,7 @@ jobs:
|
|
|
49
49
|
ports:
|
|
50
50
|
- 6379:6379
|
|
51
51
|
steps:
|
|
52
|
-
- uses: actions/checkout@
|
|
52
|
+
- uses: actions/checkout@v6
|
|
53
53
|
- name: Use Node.js ${{ matrix.node }}
|
|
54
54
|
uses: actions/setup-node@v4
|
|
55
55
|
with:
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,58 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [2.67.0](https://github.com/postalsys/emailengine/compare/v2.66.0...v2.67.0) (2026-03-31)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* handle MS Graph 'missed' lifecycle event for Outlook notification recovery ([d55f48c](https://github.com/postalsys/emailengine/commit/d55f48c9a183f9e0108801563b1085c176dbeb06))
|
|
9
|
+
* show human-readable missing scopes on OAuth scope error page ([4ee2365](https://github.com/postalsys/emailengine/commit/4ee2365449e3cbece6f4a83d540ae63c18464535))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
### Bug Fixes
|
|
13
|
+
|
|
14
|
+
* add IMAP fetch retry timeout and harden passkey registration ([3a1b61b](https://github.com/postalsys/emailengine/commit/3a1b61b00e916c308cae5e7493a2207164d9d45f))
|
|
15
|
+
* close command client proactively after subconnection setup ([02d3687](https://github.com/postalsys/emailengine/commit/02d36874d8533beabe5ad0fba978851b3791cbb5))
|
|
16
|
+
* correct translation errors and remove EmailEngine branding from OAuth scope error ([9284ddd](https://github.com/postalsys/emailengine/commit/9284ddd1e28e40fd24abeea7296636b21c4b3878))
|
|
17
|
+
* use STATUS instead of NOOP as keepalive for 163.com IMAP ([3b3516f](https://github.com/postalsys/emailengine/commit/3b3516feb14e64bab5ac586378237f2160cdf010))
|
|
18
|
+
|
|
19
|
+
## [2.66.0](https://github.com/postalsys/emailengine/compare/v2.65.0...v2.66.0) (2026-03-29)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Features
|
|
23
|
+
|
|
24
|
+
* add audit logging for admin authentication events ([0ea15d4](https://github.com/postalsys/emailengine/commit/0ea15d444ed589cd872b11190d07f7751f5cf54b))
|
|
25
|
+
* add passkey (WebAuthn) authentication for admin login ([a39b362](https://github.com/postalsys/emailengine/commit/a39b362152597bcc864307cd7c46cdc78a3808e6))
|
|
26
|
+
* always use persistent sessions and support remember-me for passkey login ([6a6fc74](https://github.com/postalsys/emailengine/commit/6a6fc74748f92372e135e88478a9fa0f83cf690f))
|
|
27
|
+
* show curl example for service account OAuth2 apps ([4ab5eda](https://github.com/postalsys/emailengine/commit/4ab5edabf64373e8b2dd4bb0ea658847d40ba904))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
### Bug Fixes
|
|
31
|
+
|
|
32
|
+
* broken Handlebars script tag, Okta session fall-through, login rate limiting, and passkey schema validation ([01e5721](https://github.com/postalsys/emailengine/commit/01e5721ee02ad66ce9c30b58c406988b1cfd1ae3))
|
|
33
|
+
* clear passkey credentials on CLI password reset and document remember-me behavior ([35f7f00](https://github.com/postalsys/emailengine/commit/35f7f00970b7e7f3a514d2ea0d97fce5320f6e1b))
|
|
34
|
+
* do not prefill login username field ([59835a8](https://github.com/postalsys/emailengine/commit/59835a84f3e2a01c6461f29ecb80763dc4a42fe2))
|
|
35
|
+
* harden passkey auth, IMAP sync error handling, and login form UX ([c16b983](https://github.com/postalsys/emailengine/commit/c16b98312f35d87fd8b21974a7eb7eca6459cc3b))
|
|
36
|
+
* harden passkey authentication with validation, rate limits, and audit logging ([75dd289](https://github.com/postalsys/emailengine/commit/75dd289940e7345fa56246c90ff6bc8d01694e6c))
|
|
37
|
+
* login page divider logic, select() log level, and missing trailing newlines ([97ff93e](https://github.com/postalsys/emailengine/commit/97ff93e2cc6c865a0f597a42a124e0abb140e34a))
|
|
38
|
+
* normalize copy across login and security pages ([60e132a](https://github.com/postalsys/emailengine/commit/60e132a9e4e8f96fc621b4b454f95896fb4d48c1))
|
|
39
|
+
* normalize sign-in/sign-out copy to sentence case ([1ccfb16](https://github.com/postalsys/emailengine/commit/1ccfb167a674d887af0ac2db8df3ff434f23e5bb))
|
|
40
|
+
* per-IP passkey rate limiting and credential ownership check ([2455cbe](https://github.com/postalsys/emailengine/commit/2455cbe9d6af5d6eff8ae81ecc335b6820bfc904))
|
|
41
|
+
* prevent message event loss during IMAP sync under heavy load ([ceb139b](https://github.com/postalsys/emailengine/commit/ceb139b825eaf7fc06bf1c88b9bcc89824abbcdb))
|
|
42
|
+
* prevent open redirects via next parameter and require password for passkey registration ([0e7f52a](https://github.com/postalsys/emailengine/commit/0e7f52ada9687234682027ecb93e81515fcb9dbb))
|
|
43
|
+
* prevent unhandled promise rejections during mailbox sync ([e6174de](https://github.com/postalsys/emailengine/commit/e6174defbf1c94f54ac7cbcb45eaa9e5c547926e))
|
|
44
|
+
* reject OAuth2 grants with missing Google granular consent scopes ([3f277d1](https://github.com/postalsys/emailengine/commit/3f277d1428e4a861752eb18b5d74279afefd1efa))
|
|
45
|
+
* remove password hash from error logs and update passkey description copy ([d28dd16](https://github.com/postalsys/emailengine/commit/d28dd16e61fc84d0800546c4f33d4535f52e2a22))
|
|
46
|
+
* remove unnecessary min-height from login form ([f16940d](https://github.com/postalsys/emailengine/commit/f16940d9957df114ce0ff4240107a82619a58705))
|
|
47
|
+
* resolve OAuth2 provider for delegated Outlook accounts ([f35c816](https://github.com/postalsys/emailengine/commit/f35c816e3149eeaa568d86c06fa6783c649c54c5))
|
|
48
|
+
* update client-side Handlebars to 4.7.9 and harden passkey input validation ([882891c](https://github.com/postalsys/emailengine/commit/882891c36ed3704d96ccf227e682e152f5395036))
|
|
49
|
+
* upgrade handlebars to 4.7.9 to resolve prototype pollution vulnerability ([452f5f5](https://github.com/postalsys/emailengine/commit/452f5f54f4007e4b765e18861ce01cde312cc83c))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
### Performance Improvements
|
|
53
|
+
|
|
54
|
+
* optimize mailbox listing for accounts with many folders ([a39e5f7](https://github.com/postalsys/emailengine/commit/a39e5f7b02156fc73ab44d1ca4f6e977e4290312))
|
|
55
|
+
|
|
3
56
|
## [2.65.0](https://github.com/postalsys/emailengine/compare/v2.64.0...v2.65.0) (2026-03-23)
|
|
4
57
|
|
|
5
58
|
|
package/bin/emailengine.js
CHANGED
|
@@ -373,6 +373,9 @@ function run() {
|
|
|
373
373
|
await settings.set('totpEnabled', false);
|
|
374
374
|
await settings.set('totpSeed', false);
|
|
375
375
|
|
|
376
|
+
let passkeys = require('../lib/passkeys');
|
|
377
|
+
await passkeys.deleteAllCredentials(authData.user);
|
|
378
|
+
|
|
376
379
|
return { password, passwordHash };
|
|
377
380
|
};
|
|
378
381
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"creationTime": "2026-03-
|
|
2
|
+
"creationTime": "2026-03-30T14:45:52.000000",
|
|
3
3
|
"prefixes": [
|
|
4
4
|
{
|
|
5
5
|
"ipv6Prefix": "2001:4860:4801:2008::/64"
|
|
@@ -397,6 +397,9 @@
|
|
|
397
397
|
{
|
|
398
398
|
"ipv6Prefix": "2001:4860:4801:20b5::/64"
|
|
399
399
|
},
|
|
400
|
+
{
|
|
401
|
+
"ipv6Prefix": "2001:4860:4801:20b6::/64"
|
|
402
|
+
},
|
|
400
403
|
{
|
|
401
404
|
"ipv4Prefix": "108.177.2.0/27"
|
|
402
405
|
},
|
|
@@ -433,6 +436,9 @@
|
|
|
433
436
|
{
|
|
434
437
|
"ipv4Prefix": "192.178.16.192/27"
|
|
435
438
|
},
|
|
439
|
+
{
|
|
440
|
+
"ipv4Prefix": "192.178.16.224/27"
|
|
441
|
+
},
|
|
436
442
|
{
|
|
437
443
|
"ipv4Prefix": "192.178.16.32/27"
|
|
438
444
|
},
|
package/lib/account.js
CHANGED
|
@@ -262,26 +262,6 @@ class Account {
|
|
|
262
262
|
return list;
|
|
263
263
|
}
|
|
264
264
|
|
|
265
|
-
async getMailboxInfo(path) {
|
|
266
|
-
let redisKey = BigInt('0x' + crypto.createHash(MAILBOX_HASH).update(normalizePath(path)).digest('hex')).toString(36);
|
|
267
|
-
|
|
268
|
-
let data = await this.redis.hgetall(`${REDIS_PREFIX}iam:${this.account}:h:${redisKey}`);
|
|
269
|
-
if (!data || !Object.keys(data).length) {
|
|
270
|
-
return {};
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
let encodedMailboxData = await this.redis.hgetBuffer(this.getMailboxListKey(), path);
|
|
274
|
-
if (!encodedMailboxData) {
|
|
275
|
-
return {};
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
return {
|
|
279
|
-
path: data.path || path,
|
|
280
|
-
messages: data.messages && !isNaN(data.messages) ? Number(data.messages) : false,
|
|
281
|
-
uidNext: data.uidNext && !isNaN(data.uidNext) ? Number(data.uidNext) : false
|
|
282
|
-
};
|
|
283
|
-
}
|
|
284
|
-
|
|
285
265
|
getAccountKey() {
|
|
286
266
|
return `${REDIS_PREFIX}iad:${this.account}`;
|
|
287
267
|
}
|
|
@@ -1177,26 +1157,52 @@ class Account {
|
|
|
1177
1157
|
let mailboxes = [];
|
|
1178
1158
|
let storedListing = await this.redis.hgetallBuffer(this.getMailboxListKey());
|
|
1179
1159
|
|
|
1180
|
-
|
|
1160
|
+
let mailboxListingMap;
|
|
1161
|
+
if (mailboxListing) {
|
|
1162
|
+
mailboxListingMap = new Map();
|
|
1163
|
+
for (let entry of mailboxListing) {
|
|
1164
|
+
if (entry.status) {
|
|
1165
|
+
delete entry.status.path;
|
|
1166
|
+
}
|
|
1167
|
+
mailboxListingMap.set(entry.path, entry);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
let paths = Object.keys(storedListing || {});
|
|
1172
|
+
|
|
1173
|
+
// Batch into a single round-trip to avoid per-mailbox latency with many folders
|
|
1174
|
+
let pipeline = this.redis.pipeline();
|
|
1175
|
+
for (let path of paths) {
|
|
1176
|
+
let redisKey = BigInt('0x' + crypto.createHash(MAILBOX_HASH).update(normalizePath(path)).digest('hex')).toString(36);
|
|
1177
|
+
pipeline.hgetall(`${REDIS_PREFIX}iam:${this.account}:h:${redisKey}`);
|
|
1178
|
+
}
|
|
1179
|
+
let pipelineResults = await pipeline.exec();
|
|
1180
|
+
|
|
1181
|
+
for (let i = 0; i < paths.length; i++) {
|
|
1182
|
+
let path = paths[i];
|
|
1181
1183
|
try {
|
|
1182
1184
|
let decoded = msgpack.decode(storedListing[path]);
|
|
1183
1185
|
|
|
1184
1186
|
if (decoded.path && decoded.delimiter && decoded.path.indexOf(decoded.delimiter) >= 0) {
|
|
1185
|
-
decoded.parentPath = decoded.path.
|
|
1187
|
+
decoded.parentPath = decoded.path.substring(0, decoded.path.lastIndexOf(decoded.delimiter));
|
|
1186
1188
|
}
|
|
1187
1189
|
|
|
1188
|
-
let listedMailboxInfo;
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1190
|
+
let listedMailboxInfo = mailboxListingMap ? mailboxListingMap.get(path) : undefined;
|
|
1191
|
+
|
|
1192
|
+
let mailboxInfo = {};
|
|
1193
|
+
let [pipelineErr, data] = pipelineResults[i] || [];
|
|
1194
|
+
if (!pipelineErr && data && Object.keys(data).length) {
|
|
1195
|
+
mailboxInfo = {
|
|
1196
|
+
path: data.path || path,
|
|
1197
|
+
messages: data.messages && !isNaN(data.messages) ? Number(data.messages) : false,
|
|
1198
|
+
uidNext: data.uidNext && !isNaN(data.uidNext) ? Number(data.uidNext) : false
|
|
1199
|
+
};
|
|
1194
1200
|
}
|
|
1195
1201
|
|
|
1196
1202
|
mailboxes.push(
|
|
1197
1203
|
Object.assign(
|
|
1198
1204
|
decoded,
|
|
1199
|
-
|
|
1205
|
+
mailboxInfo,
|
|
1200
1206
|
listedMailboxInfo && listedMailboxInfo.status
|
|
1201
1207
|
? {
|
|
1202
1208
|
status: listedMailboxInfo.status
|
package/lib/consts.js
CHANGED
|
@@ -154,6 +154,9 @@ module.exports = {
|
|
|
154
154
|
|
|
155
155
|
TOTP_WINDOW_SIZE: 6,
|
|
156
156
|
|
|
157
|
+
WEBAUTHN_CHALLENGE_TTL: 300, // 5 minutes
|
|
158
|
+
MAX_PASSKEYS_PER_USER: 20,
|
|
159
|
+
|
|
157
160
|
// How many times to retry an email sending before it is considered as failing
|
|
158
161
|
DEFAULT_DELIVERY_ATTEMPTS: 10,
|
|
159
162
|
|
|
@@ -190,6 +193,8 @@ module.exports = {
|
|
|
190
193
|
FETCH_RETRY_INTERVAL: 1000,
|
|
191
194
|
FETCH_RETRY_EXPONENTIAL: 2,
|
|
192
195
|
FETCH_RETRY_MAX: 60 * 1000,
|
|
196
|
+
FETCH_RETRY_MAX_TIME: 24 * 60 * 60 * 1000, // 1 day maximum retry duration
|
|
197
|
+
FETCH_RETRY_MIN_ATTEMPTS: 20, // minimum attempts before time limit applies
|
|
193
198
|
DEFAULT_FETCH_BATCH_SIZE: 1000,
|
|
194
199
|
|
|
195
200
|
// MS Graph webhook subscription settings
|
|
@@ -27,7 +27,7 @@ const {
|
|
|
27
27
|
|
|
28
28
|
const settings = require('../settings');
|
|
29
29
|
|
|
30
|
-
const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest } = require('./gmail/gmail-api');
|
|
30
|
+
const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest, executeBatchRequests } = require('./gmail/gmail-api');
|
|
31
31
|
|
|
32
32
|
const MAX_GMAIL_BATCH_SIZE = 50;
|
|
33
33
|
|
|
@@ -253,6 +253,8 @@ class GmailClient extends BaseClient {
|
|
|
253
253
|
this.renewWatchTimer = null;
|
|
254
254
|
|
|
255
255
|
this.cachedLabels = null;
|
|
256
|
+
this.cachedDetailedLabels = null;
|
|
257
|
+
this.cachedDetailedLabelsTime = null;
|
|
256
258
|
}
|
|
257
259
|
|
|
258
260
|
/**
|
|
@@ -485,6 +487,8 @@ class GmailClient extends BaseClient {
|
|
|
485
487
|
// Clean up cached data
|
|
486
488
|
this.cachedLabels = null;
|
|
487
489
|
this.cachedLabelsTime = null;
|
|
490
|
+
this.cachedDetailedLabels = null;
|
|
491
|
+
this.cachedDetailedLabelsTime = null;
|
|
488
492
|
this.pendingHistoryId = null;
|
|
489
493
|
|
|
490
494
|
if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
|
|
@@ -508,6 +512,8 @@ class GmailClient extends BaseClient {
|
|
|
508
512
|
// Clean up cached data
|
|
509
513
|
this.cachedLabels = null;
|
|
510
514
|
this.cachedLabelsTime = null;
|
|
515
|
+
this.cachedDetailedLabels = null;
|
|
516
|
+
this.cachedDetailedLabelsTime = null;
|
|
511
517
|
this.pendingHistoryId = null;
|
|
512
518
|
|
|
513
519
|
if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
|
|
@@ -538,34 +544,21 @@ class GmailClient extends BaseClient {
|
|
|
538
544
|
|
|
539
545
|
let resultLabels;
|
|
540
546
|
if (options && options.statusQuery?.unseen) {
|
|
541
|
-
//
|
|
542
|
-
let
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
throw entry.reason;
|
|
553
|
-
}
|
|
554
|
-
if (entry.value) {
|
|
555
|
-
resultLabels.push(entry.value);
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
promises = [];
|
|
559
|
-
};
|
|
547
|
+
// Short-lived cache avoids N per-label API calls on repeated requests within 60s
|
|
548
|
+
let now = Date.now();
|
|
549
|
+
if (this.cachedDetailedLabels && now <= this.cachedDetailedLabelsTime + 60 * 1000) {
|
|
550
|
+
resultLabels = this.cachedDetailedLabels;
|
|
551
|
+
} else {
|
|
552
|
+
resultLabels = await executeBatchRequests(
|
|
553
|
+
this,
|
|
554
|
+
labels,
|
|
555
|
+
label => this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/labels/${label.id}`),
|
|
556
|
+
MAX_GMAIL_BATCH_SIZE
|
|
557
|
+
);
|
|
560
558
|
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
promises.push(this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/labels/${label.id}`));
|
|
564
|
-
if (promises.length > LIST_BATCH_SIZE) {
|
|
565
|
-
await resolvePromises();
|
|
566
|
-
}
|
|
559
|
+
this.cachedDetailedLabels = resultLabels;
|
|
560
|
+
this.cachedDetailedLabelsTime = now;
|
|
567
561
|
}
|
|
568
|
-
await resolvePromises();
|
|
569
562
|
} else {
|
|
570
563
|
resultLabels = labels;
|
|
571
564
|
}
|
|
@@ -1732,6 +1725,7 @@ class GmailClient extends BaseClient {
|
|
|
1732
1725
|
|
|
1733
1726
|
// clear cache
|
|
1734
1727
|
this.cachedLabels = null;
|
|
1728
|
+
this.cachedDetailedLabels = null;
|
|
1735
1729
|
} catch (err) {
|
|
1736
1730
|
switch (err?.oauthRequest?.response?.error?.code) {
|
|
1737
1731
|
case 409:
|
|
@@ -1814,6 +1808,7 @@ class GmailClient extends BaseClient {
|
|
|
1814
1808
|
|
|
1815
1809
|
// clear cache
|
|
1816
1810
|
this.cachedLabels = null;
|
|
1811
|
+
this.cachedDetailedLabels = null;
|
|
1817
1812
|
} catch (err) {
|
|
1818
1813
|
switch (err?.oauthRequest?.response?.error?.code) {
|
|
1819
1814
|
case 409:
|
|
@@ -1863,6 +1858,7 @@ class GmailClient extends BaseClient {
|
|
|
1863
1858
|
|
|
1864
1859
|
// clear cache
|
|
1865
1860
|
this.cachedLabels = null;
|
|
1861
|
+
this.cachedDetailedLabels = null;
|
|
1866
1862
|
} catch (err) {
|
|
1867
1863
|
switch (err?.oauthRequest?.response?.error?.code) {
|
|
1868
1864
|
case 409:
|
|
@@ -30,7 +30,8 @@ const {
|
|
|
30
30
|
REDIS_PREFIX,
|
|
31
31
|
MAX_INLINE_ATTACHMENT_SIZE,
|
|
32
32
|
MAX_ALLOWED_DOWNLOAD_SIZE,
|
|
33
|
-
MAILBOX_HASH
|
|
33
|
+
MAILBOX_HASH,
|
|
34
|
+
TRANSIENT_NETWORK_CODES
|
|
34
35
|
} = require('../../consts');
|
|
35
36
|
|
|
36
37
|
const {
|
|
@@ -403,11 +404,13 @@ class Mailbox {
|
|
|
403
404
|
this.logger.debug({ msg: 'Deleted mailbox', path: this.listingEntry.path });
|
|
404
405
|
|
|
405
406
|
if (!opts.skipNotify) {
|
|
406
|
-
this.connection
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
407
|
+
this.connection
|
|
408
|
+
.notify(this, MAILBOX_DELETED_NOTIFY, {
|
|
409
|
+
path: this.listingEntry.path,
|
|
410
|
+
name: this.listingEntry.name,
|
|
411
|
+
specialUse: this.listingEntry.specialUse || false
|
|
412
|
+
})
|
|
413
|
+
.catch(err => this.logger.error({ msg: 'Failed to send mailbox deleted notification', err }));
|
|
411
414
|
}
|
|
412
415
|
}
|
|
413
416
|
|
|
@@ -1982,20 +1985,24 @@ class Mailbox {
|
|
|
1982
1985
|
messageFetchOptions.fetchHeaders = Array.from(fetchHeaders);
|
|
1983
1986
|
}
|
|
1984
1987
|
|
|
1985
|
-
//
|
|
1988
|
+
// Peek-then-remove: read entries without removing so that connection errors
|
|
1989
|
+
// leave unprocessed entries in Redis for retry on reconnect.
|
|
1986
1990
|
let queuedEntry;
|
|
1987
1991
|
let hadUpdates = false;
|
|
1988
|
-
|
|
1992
|
+
let notificationsKey = this.getNotificationsKey();
|
|
1993
|
+
while ((queuedEntry = await this.connection.redis.zrange(notificationsKey, 0, 0, 'WITHSCORES')) && queuedEntry.length) {
|
|
1989
1994
|
hadUpdates = true;
|
|
1990
1995
|
|
|
1991
|
-
let [
|
|
1996
|
+
let [rawMember, uid] = queuedEntry;
|
|
1992
1997
|
uid = Number(uid);
|
|
1998
|
+
let messageData;
|
|
1993
1999
|
try {
|
|
1994
|
-
messageData = JSON.parse(
|
|
2000
|
+
messageData = JSON.parse(rawMember);
|
|
1995
2001
|
if (typeof messageData.internalDate === 'string') {
|
|
1996
2002
|
messageData.internalDate = new Date(messageData.internalDate);
|
|
1997
2003
|
}
|
|
1998
2004
|
} catch (err) {
|
|
2005
|
+
await this.connection.redis.zrem(notificationsKey, rawMember);
|
|
1999
2006
|
continue;
|
|
2000
2007
|
}
|
|
2001
2008
|
|
|
@@ -2003,11 +2010,25 @@ class Mailbox {
|
|
|
2003
2010
|
let canSync = documentStoreEnabled && (!this.connection.syncFrom || messageData.internalDate >= this.connection.syncFrom);
|
|
2004
2011
|
|
|
2005
2012
|
if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom && !canSync) {
|
|
2006
|
-
|
|
2013
|
+
await this.connection.redis.zrem(notificationsKey, rawMember);
|
|
2007
2014
|
continue;
|
|
2008
2015
|
}
|
|
2009
2016
|
|
|
2010
|
-
|
|
2017
|
+
try {
|
|
2018
|
+
await this.processNew(messageData, messageFetchOptions, canSync, storedStatus);
|
|
2019
|
+
await this.connection.redis.zrem(notificationsKey, rawMember);
|
|
2020
|
+
} catch (err) {
|
|
2021
|
+
if (
|
|
2022
|
+
err.code === 'NoConnection' ||
|
|
2023
|
+
err.code === 'IMAPConnectionClosing' ||
|
|
2024
|
+
err.code === 'EConnectionClosed' ||
|
|
2025
|
+
TRANSIENT_NETWORK_CODES.has(err.code)
|
|
2026
|
+
) {
|
|
2027
|
+
throw err;
|
|
2028
|
+
}
|
|
2029
|
+
await this.connection.redis.zrem(notificationsKey, rawMember);
|
|
2030
|
+
this.logger.error({ msg: 'Failed to process new message', uid, id: messageData.id, err });
|
|
2031
|
+
}
|
|
2011
2032
|
}
|
|
2012
2033
|
|
|
2013
2034
|
if (hadUpdates) {
|
|
@@ -2119,19 +2140,25 @@ class Mailbox {
|
|
|
2119
2140
|
// fully synced, so not new anymore
|
|
2120
2141
|
this.listingEntry.isNew = false;
|
|
2121
2142
|
this.logger.debug({ msg: 'New mailbox', path: this.listingEntry.path });
|
|
2122
|
-
this.connection
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2143
|
+
this.connection
|
|
2144
|
+
.notify(this, MAILBOX_NEW_NOTIFY, {
|
|
2145
|
+
path: this.listingEntry.path,
|
|
2146
|
+
name: this.listingEntry.name,
|
|
2147
|
+
specialUse: this.listingEntry.specialUse || false,
|
|
2148
|
+
uidValidity: mailboxStatus.uidValidity.toString()
|
|
2149
|
+
})
|
|
2150
|
+
.catch(err => this.logger.error({ msg: 'Failed to send new mailbox notification', err }));
|
|
2128
2151
|
}
|
|
2129
2152
|
|
|
2130
2153
|
// Resolve sync promise or start IDLE
|
|
2131
2154
|
if (this.synced) {
|
|
2132
2155
|
this.synced();
|
|
2133
2156
|
} else {
|
|
2134
|
-
|
|
2157
|
+
try {
|
|
2158
|
+
await this.select();
|
|
2159
|
+
} catch (err) {
|
|
2160
|
+
this.logger.warn({ msg: 'Failed to select after sync', err });
|
|
2161
|
+
}
|
|
2135
2162
|
}
|
|
2136
2163
|
}
|
|
2137
2164
|
}
|