emailengine-app 2.71.0 → 2.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/codeql.yml +3 -0
- package/.github/workflows/e2e.yml +56 -0
- package/.github/workflows/test.yml +8 -0
- package/.ncurc.js +20 -20
- package/CHANGELOG.md +14 -0
- package/config/e2e.toml +35 -0
- package/data/google-crawlers.json +1 -1
- package/lib/api-routes/route-helpers.js +11 -5
- package/lib/auth-token.js +83 -0
- package/lib/delivery-error.js +62 -0
- package/lib/email-client/gmail-client.js +33 -1
- package/lib/imap-proxy-auth.js +81 -0
- package/lib/imapproxy/imap-server.js +8 -103
- package/lib/oauth/gmail.js +3 -0
- package/lib/oauth/outlook.js +3 -0
- package/lib/oauth2-apps.js +100 -11
- package/lib/smtp-auth.js +70 -0
- package/lib/sub-script.js +8 -2
- package/package.json +21 -18
- package/playwright.config.js +45 -0
- package/sbom.json +1 -1
- package/static/licenses.html +171 -31
- package/test-coverage-plan.md +233 -0
- package/translations/messages.pot +1 -1
- package/workers/smtp.js +5 -85
- package/workers/submit.js +2 -12
|
@@ -22,6 +22,9 @@ on:
|
|
|
22
22
|
jobs:
|
|
23
23
|
analyze:
|
|
24
24
|
name: Analyze (${{ matrix.language }})
|
|
25
|
+
# Skip release-please's auto-generated release PR: it only changes CHANGELOG/version,
|
|
26
|
+
# so there is no new code to scan. (Matches the guard in test.yml.)
|
|
27
|
+
if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
|
|
25
28
|
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
|
26
29
|
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
|
27
30
|
# - https://gh.io/supported-runners-and-hardware-resources
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: E2E Tests
|
|
2
|
+
|
|
3
|
+
# Browser-driven happy-path end-to-end suite (test/e2e), kept separate from the unit and
|
|
4
|
+
# integration tiers so it is easy to re-run on its own and has its own requirements: it drives a
|
|
5
|
+
# real browser (Playwright/Chromium) and reaches external services (Ethereal for the test mailbox,
|
|
6
|
+
# postalsys.com for the 14-day trial). The trial rate limit is bypassed for the e2e serviceUrl
|
|
7
|
+
# (https://e2e.emailengine.app/) - see the postalsys-web trial allowlist.
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
push:
|
|
12
|
+
branches: [master]
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
concurrency:
|
|
18
|
+
group: e2e-${{ github.ref }}
|
|
19
|
+
cancel-in-progress: true
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
e2e:
|
|
23
|
+
name: E2E (Playwright)
|
|
24
|
+
timeout-minutes: 20
|
|
25
|
+
runs-on: ubuntu-24.04
|
|
26
|
+
services:
|
|
27
|
+
redis:
|
|
28
|
+
image: redis
|
|
29
|
+
options: >-
|
|
30
|
+
--health-cmd "redis-cli ping"
|
|
31
|
+
--health-interval 10s
|
|
32
|
+
--health-timeout 5s
|
|
33
|
+
--health-retries 5
|
|
34
|
+
ports:
|
|
35
|
+
- 6379:6379
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/checkout@v6
|
|
38
|
+
- name: Use Node.js 24
|
|
39
|
+
uses: actions/setup-node@v6
|
|
40
|
+
with:
|
|
41
|
+
node-version: 24
|
|
42
|
+
cache: npm
|
|
43
|
+
- run: npm install
|
|
44
|
+
- name: Install Playwright browser
|
|
45
|
+
run: npx playwright install --with-deps chromium
|
|
46
|
+
- name: Run e2e tests
|
|
47
|
+
run: npm run test:e2e
|
|
48
|
+
env:
|
|
49
|
+
NODE_ENV: e2e
|
|
50
|
+
- name: Upload Playwright report
|
|
51
|
+
uses: actions/upload-artifact@v4
|
|
52
|
+
if: ${{ !cancelled() }}
|
|
53
|
+
with:
|
|
54
|
+
name: playwright-report
|
|
55
|
+
path: playwright-report/
|
|
56
|
+
retention-days: 7
|
|
@@ -12,8 +12,13 @@ concurrency:
|
|
|
12
12
|
cancel-in-progress: true
|
|
13
13
|
|
|
14
14
|
jobs:
|
|
15
|
+
# Skip CI for release-please's auto-generated release PR / branch: it only bumps the
|
|
16
|
+
# version + CHANGELOG (no code change), so re-running the suite is wasted work. This is
|
|
17
|
+
# gated per job rather than with [skip ci] in the release commit, because that text would
|
|
18
|
+
# ride the squash-merge into master and skip the Manage Release run, breaking releases.
|
|
15
19
|
license_check:
|
|
16
20
|
name: License Compliance Check
|
|
21
|
+
if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
|
|
17
22
|
runs-on: ubuntu-24.04
|
|
18
23
|
# Service containers to run with `container-job`
|
|
19
24
|
steps:
|
|
@@ -32,6 +37,7 @@ jobs:
|
|
|
32
37
|
|
|
33
38
|
lint:
|
|
34
39
|
name: Lint
|
|
40
|
+
if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
|
|
35
41
|
runs-on: ubuntu-24.04
|
|
36
42
|
timeout-minutes: 10
|
|
37
43
|
steps:
|
|
@@ -48,6 +54,7 @@ jobs:
|
|
|
48
54
|
|
|
49
55
|
unit:
|
|
50
56
|
name: Unit Tests
|
|
57
|
+
if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
|
|
51
58
|
timeout-minutes: 10
|
|
52
59
|
runs-on: ubuntu-24.04
|
|
53
60
|
# Service containers to run with `container-job`
|
|
@@ -95,6 +102,7 @@ jobs:
|
|
|
95
102
|
|
|
96
103
|
integration:
|
|
97
104
|
name: Integration Tests
|
|
105
|
+
if: ${{ !startsWith(github.ref_name, 'release-please--') && !startsWith(github.head_ref, 'release-please--') }}
|
|
98
106
|
timeout-minutes: 15 # Increased timeout for Gmail API tests
|
|
99
107
|
runs-on: ubuntu-24.04
|
|
100
108
|
# Service containers to run with `container-job`
|
package/.ncurc.js
CHANGED
|
@@ -1,30 +1,30 @@
|
|
|
1
1
|
module.exports = {
|
|
2
2
|
upgrade: true,
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
3
|
+
// Packages capped inside their current major: the next major is ESM-only (this codebase is
|
|
4
|
+
// CommonJS and is bundled into a binary with pkg) or it breaks a peer dependency. Using 'minor'
|
|
5
|
+
// instead of a blanket reject so these still receive security/patch updates within the safe
|
|
6
|
+
// major instead of being frozen at one exact version. Verified against Node 20 (Docker) 2026-06-17.
|
|
7
|
+
target: name => (['joi', 'nanoid', 'ical.js', 'gettext-parser', 'xgettext-template', 'chai', 'undici', 'marked'].includes(name) ? 'minor' : 'latest'),
|
|
8
|
+
// joi - hapi-swagger (repo archived 2026-02-04) peer-requires joi 17.x; permanent ceiling
|
|
9
|
+
// nanoid - 4.x dropped the CommonJS require export (ESM-only)
|
|
10
|
+
// ical.js - 2.x is ESM-only
|
|
11
|
+
// gettext-parser - 8.x is ESM-only
|
|
12
|
+
// xgettext-template - 6.x is ESM-only (translation build tool)
|
|
13
|
+
// chai - 5.x is ESM-only (only used by the vendored imap-core tests)
|
|
14
|
+
// undici - 8.x requires Node >=22.19 and crashes at require() on Node 20; EmailEngine supports Node 20+
|
|
15
|
+
// marked - 16.x dropped the CommonJS build (ESM-only, needs require(esm)/Node >=20.19); 15.x is the
|
|
16
|
+
// last require()-compatible line. 15.0.12 verified on Node 20-24 and in a yao/pkg node24 build.
|
|
6
17
|
reject: [
|
|
7
|
-
//
|
|
8
|
-
|
|
9
|
-
'gettext-parser',
|
|
10
|
-
'xgettext-template',
|
|
11
|
-
'chai',
|
|
12
|
-
'js-beautify',
|
|
13
|
-
'ical.js',
|
|
18
|
+
// 8.16+ pulls apache-arrow (ESM) into the pkg bundle; 9.x drops it but is a major bump for
|
|
19
|
+
// the deprecated, default-off Document Store. Even the 8.19 minor is unsafe to bundle.
|
|
14
20
|
'@elastic/elasticsearch',
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
// no support for Node 16
|
|
19
|
-
'marked',
|
|
20
|
-
|
|
21
|
-
// some kind of CVE in later versions. Only needed for license reference, so the actual version does not matter anyway
|
|
22
|
+
// v4.x adds vulnerable jquery + bootstrap runtime dependencies; v3.3.7 has none. Used only
|
|
23
|
+
// for the generated software-license listing, never executed at runtime.
|
|
22
24
|
'startbootstrap-sb-admin-2',
|
|
23
25
|
|
|
24
26
|
// @asamuzakjp/css-color >=4.1.2 pulls in @csstools/* v4 which are pure ESM and break pkg bundling
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// undici >=8.0.0 requires Node >=22.19.0; pin to last Node 20-compatible release
|
|
28
|
-
'undici'
|
|
27
|
+
// (transitive via @postalsys/email-text-tools -> jsdom -> cssstyle; also pinned in package.json "overrides").
|
|
28
|
+
'@asamuzakjp/css-color'
|
|
29
29
|
]
|
|
30
30
|
};
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [2.72.1](https://github.com/postalsys/emailengine/compare/v2.72.0...v2.72.1) (2026-06-19)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* log expected API client errors (4xx) at warn instead of error ([c51e347](https://github.com/postalsys/emailengine/commit/c51e3476e346f321af845905a89141bf07a83421))
|
|
9
|
+
|
|
10
|
+
## [2.72.0](https://github.com/postalsys/emailengine/compare/v2.71.0...v2.72.0) (2026-06-18)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Features
|
|
14
|
+
|
|
15
|
+
* adopt pre-existing Gmail Pub/Sub resources with ownership tracking ([fe43fad](https://github.com/postalsys/emailengine/commit/fe43fadad624025a56139382428256eb810f9e08))
|
|
16
|
+
|
|
3
17
|
## [2.71.0](https://github.com/postalsys/emailengine/compare/v2.70.0...v2.71.0) (2026-06-15)
|
|
4
18
|
|
|
5
19
|
|
package/config/e2e.toml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# End-to-end overrides, merged over default.toml by @zone-eu/wild-config when NODE_ENV=e2e.
|
|
2
|
+
# Boots a fresh EmailEngine for the Playwright happy-path suite (test/e2e): enable auth,
|
|
3
|
+
# activate a 14-day trial (hits postalsys.com), register an Ethereal account, send + read back.
|
|
4
|
+
#
|
|
5
|
+
# Uses an isolated Redis DB so the suite can flush it for a clean fresh-instance run without
|
|
6
|
+
# touching dev (db 9), test (db 13), or default (db 8) data. No preparedToken / preparedPassword /
|
|
7
|
+
# preparedLicense here on purpose - the suite enables auth and activates the trial through the UI.
|
|
8
|
+
|
|
9
|
+
# JSON formatted settings, seeded on startup.
|
|
10
|
+
# serviceUrl is the recognizable e2e URL that postalsys-web allowlists to skip the trial rate
|
|
11
|
+
# limit. It is only used as the trial request's `url`; the browser and API talk to 127.0.0.1:7099.
|
|
12
|
+
settings = '''
|
|
13
|
+
{
|
|
14
|
+
"serviceUrl": "https://e2e.emailengine.app/",
|
|
15
|
+
"ignoreMailCertErrors": true
|
|
16
|
+
}
|
|
17
|
+
'''
|
|
18
|
+
|
|
19
|
+
[service]
|
|
20
|
+
# Fixed encryption secret for the throwaway e2e instance (Redis is flushed on every run).
|
|
21
|
+
secret = "e2e encryption secret"
|
|
22
|
+
|
|
23
|
+
[workers]
|
|
24
|
+
imap = 1
|
|
25
|
+
|
|
26
|
+
[log]
|
|
27
|
+
level = "warn"
|
|
28
|
+
|
|
29
|
+
[dbs]
|
|
30
|
+
# Isolated Redis DB for e2e; flushed before each run by test/e2e/flush-redis.js
|
|
31
|
+
redis = "redis://127.0.0.1:6379/14"
|
|
32
|
+
|
|
33
|
+
[api]
|
|
34
|
+
host = "127.0.0.1"
|
|
35
|
+
port = 7099
|
|
@@ -9,14 +9,20 @@ const Boom = require('@hapi/boom');
|
|
|
9
9
|
// machine-readable err.code. This function ALWAYS throws and never returns a value, so callers use it
|
|
10
10
|
// as the final statement inside a catch block: `catch (err) { handleError(request, err); }`.
|
|
11
11
|
function handleError(request, err) {
|
|
12
|
-
request.logger.error({ msg: 'API request failed', err });
|
|
13
|
-
if (Boom.isBoom(err)) {
|
|
14
|
-
throw err;
|
|
15
|
-
}
|
|
16
12
|
// Lower-level libraries (e.g. ImapFlow) flag "this server lacks the required capability" with a
|
|
17
13
|
// machine code but no HTTP status. Surface it as a 422 client error instead of a generic 500 - the
|
|
18
14
|
// request is well-formed, the account just cannot satisfy it (e.g. label search on non-Gmail IMAP).
|
|
19
|
-
let
|
|
15
|
+
let isBoom = Boom.isBoom(err);
|
|
16
|
+
let statusCode = isBoom ? err.output.statusCode : err.statusCode || (err.code === 'MissingServerExtension' ? 422 : 500);
|
|
17
|
+
|
|
18
|
+
// Log expected client errors (4xx, e.g. a 404 from an existence-check probe) at warn so they do not
|
|
19
|
+
// flood the error stream, while genuine server faults (5xx) stay at error.
|
|
20
|
+
let logLevel = statusCode >= 400 && statusCode < 500 ? 'warn' : 'error';
|
|
21
|
+
request.logger[logLevel]({ msg: 'API request failed', statusCode, err });
|
|
22
|
+
|
|
23
|
+
if (isBoom) {
|
|
24
|
+
throw err;
|
|
25
|
+
}
|
|
20
26
|
const error = Boom.boomify(err, { statusCode });
|
|
21
27
|
if (err.code) {
|
|
22
28
|
error.output.payload.code = err.code;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Shared token validation for the SMTP and IMAP-proxy submission servers. Both
|
|
4
|
+
// servers accept a 64-char hex API token as the password and apply the same
|
|
5
|
+
// checks (account binding, scope, IP allowlist). The logic lives here so the two
|
|
6
|
+
// auth handlers (lib/smtp-auth.js, lib/imap-proxy-auth.js) cannot drift apart -
|
|
7
|
+
// a token security-policy change is made once and applies to both surfaces.
|
|
8
|
+
|
|
9
|
+
const logger = require('./logger');
|
|
10
|
+
const tokens = require('./tokens');
|
|
11
|
+
const { matchIp } = require('./utils/network');
|
|
12
|
+
|
|
13
|
+
// Reason-specific denial messages, shared verbatim by both auth handlers. The
|
|
14
|
+
// generic "failed to authenticate" fallback and any protocol-specific error
|
|
15
|
+
// decoration are left to each caller.
|
|
16
|
+
const REASON_MESSAGES = {
|
|
17
|
+
username: 'Access denied, invalid username',
|
|
18
|
+
scope: 'Access denied, invalid scope',
|
|
19
|
+
ip: 'Access denied, traffic not accepted from this IP'
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Validates a 64-char hex API token supplied as a server password.
|
|
24
|
+
*
|
|
25
|
+
* Performs the token lookup plus the account-binding, scope and IP-allowlist
|
|
26
|
+
* checks. Does NOT throw - the caller maps the returned reason to its own
|
|
27
|
+
* protocol-specific error (SMTP and IMAP use different response shapes).
|
|
28
|
+
*
|
|
29
|
+
* @param {Object} opts
|
|
30
|
+
* @param {String} opts.password - supplied password (candidate token)
|
|
31
|
+
* @param {String} opts.account - username the client authenticated as
|
|
32
|
+
* @param {String} opts.requiredScope - scope the token must hold ('smtp' | 'imap-proxy')
|
|
33
|
+
* @param {String} opts.remoteAddress - client IP, checked against token restrictions
|
|
34
|
+
* @returns {Promise<{authenticated: Boolean, reason: (null|'username'|'scope'|'ip')}>}
|
|
35
|
+
*/
|
|
36
|
+
async function validateAuthToken({ password, account, requiredScope, remoteAddress }) {
|
|
37
|
+
if (!/^[0-9a-f]{64}$/i.test(password)) {
|
|
38
|
+
return { authenticated: false, reason: null };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let tokenData;
|
|
42
|
+
try {
|
|
43
|
+
tokenData = await tokens.get(password, false, { log: true, remoteAddress });
|
|
44
|
+
} catch (err) {
|
|
45
|
+
logger.error({ msg: 'Failed to fetch token', err });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (!tokenData) {
|
|
49
|
+
return { authenticated: false, reason: null };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if (tokenData.account && tokenData.account !== account) {
|
|
53
|
+
return { authenticated: false, reason: 'username' };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (tokenData.scopes && !tokenData.scopes.includes(requiredScope) && !tokenData.scopes.includes('*')) {
|
|
57
|
+
logger.error({
|
|
58
|
+
msg: 'Trying to use invalid scope for a token',
|
|
59
|
+
tokenAccount: tokenData.account,
|
|
60
|
+
tokenId: tokenData.id,
|
|
61
|
+
account,
|
|
62
|
+
requestedScope: requiredScope,
|
|
63
|
+
scopes: tokenData.scopes
|
|
64
|
+
});
|
|
65
|
+
return { authenticated: false, reason: 'scope' };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(remoteAddress, tokenData.restrictions.addresses)) {
|
|
69
|
+
logger.error({
|
|
70
|
+
msg: 'Trying to use invalid IP for a token',
|
|
71
|
+
tokenAccount: tokenData.account,
|
|
72
|
+
tokenId: tokenData.id,
|
|
73
|
+
account,
|
|
74
|
+
remoteAddress,
|
|
75
|
+
addressAllowlist: tokenData.restrictions.addresses
|
|
76
|
+
});
|
|
77
|
+
return { authenticated: false, reason: 'ip' };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { authenticated: true, reason: null };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
module.exports = { validateAuthToken, REASON_MESSAGES };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Classification of outbound delivery errors for the submit worker.
|
|
4
|
+
//
|
|
5
|
+
// This logic lives in its own module so it can be unit tested without booting
|
|
6
|
+
// the BullMQ submit worker (workers/submit.js connects to Redis and expects a
|
|
7
|
+
// worker-thread parentPort at require time).
|
|
8
|
+
|
|
9
|
+
// Nodemailer/SMTP error codes that represent permanent failures. A message that
|
|
10
|
+
// fails with one of these will fail again on every retry, so the job is
|
|
11
|
+
// discarded instead of being retried.
|
|
12
|
+
const NON_RETRYABLE_CODES = new Set([
|
|
13
|
+
'EAUTH', // authentication failed
|
|
14
|
+
'ENOAUTH', // no credentials provided
|
|
15
|
+
'EOAUTH2', // OAuth2 token failure
|
|
16
|
+
'ETLS', // TLS handshake failed
|
|
17
|
+
'EENVELOPE', // invalid sender/recipients
|
|
18
|
+
'EMESSAGE', // message content error
|
|
19
|
+
'EPROTOCOL' // SMTP protocol mismatch
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Determines whether a delivery error is permanent (must not be retried).
|
|
24
|
+
*
|
|
25
|
+
* An error is permanent when either:
|
|
26
|
+
* - the SMTP status code is a 5xx other than 503 (503 is treated as a
|
|
27
|
+
* transient "try again later" response), or
|
|
28
|
+
* - the error carries one of the NON_RETRYABLE_CODES.
|
|
29
|
+
*
|
|
30
|
+
* @param {Object} err - Error thrown during submission
|
|
31
|
+
* @param {Number} [err.statusCode] - SMTP/HTTP status code
|
|
32
|
+
* @param {String} [err.code] - Nodemailer error code
|
|
33
|
+
* @returns {Boolean} True if the error should never be retried
|
|
34
|
+
*/
|
|
35
|
+
function isPermanentDeliveryError(err) {
|
|
36
|
+
if (!err) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
const isPermanentSmtp = err.statusCode >= 500 && err.statusCode !== 503;
|
|
40
|
+
const isPermanentCode = NON_RETRYABLE_CODES.has(err.code);
|
|
41
|
+
return isPermanentSmtp || isPermanentCode;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Determines whether the submit worker should discard a job (stop retrying).
|
|
46
|
+
*
|
|
47
|
+
* A job is discarded only when the error is permanent AND there are still
|
|
48
|
+
* attempts remaining. When attempts are already exhausted, BullMQ will fail the
|
|
49
|
+
* job naturally, so there is nothing to discard.
|
|
50
|
+
*
|
|
51
|
+
* @param {Object} err - Error thrown during submission
|
|
52
|
+
* @param {Object} job - BullMQ job
|
|
53
|
+
* @param {Number} job.attemptsMade - Attempts already made
|
|
54
|
+
* @param {Object} job.opts - Job options
|
|
55
|
+
* @param {Number} job.opts.attempts - Configured max attempts
|
|
56
|
+
* @returns {Boolean} True if the job should be discarded
|
|
57
|
+
*/
|
|
58
|
+
function shouldDiscardJob(err, job) {
|
|
59
|
+
return isPermanentDeliveryError(err) && job.attemptsMade < job.opts.attempts;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
module.exports = { NON_RETRYABLE_CODES, isPermanentDeliveryError, shouldDiscardJob };
|
|
@@ -257,6 +257,9 @@ class GmailClient extends BaseClient {
|
|
|
257
257
|
|
|
258
258
|
this.processingHistory = null;
|
|
259
259
|
this.renewWatchTimer = null;
|
|
260
|
+
// Set once renewWatch() has logged that the linked Pub/Sub app is missing its topic/IAM
|
|
261
|
+
// markers, so the warning is emitted once per connection instead of every renewal cycle.
|
|
262
|
+
this._loggedMissingWatchMarkers = false;
|
|
260
263
|
|
|
261
264
|
this.cachedLabels = null;
|
|
262
265
|
this.cachedDetailedLabels = null;
|
|
@@ -2047,6 +2050,9 @@ class GmailClient extends BaseClient {
|
|
|
2047
2050
|
watchExpiration,
|
|
2048
2051
|
watchFailure: null
|
|
2049
2052
|
});
|
|
2053
|
+
// Markers are present and the watch armed - allow the missing-markers warning
|
|
2054
|
+
// to fire again if the app's markers ever disappear.
|
|
2055
|
+
this._loggedMissingWatchMarkers = false;
|
|
2050
2056
|
this.logger.info({
|
|
2051
2057
|
msg: 'Renewed Gmail pubsub watch',
|
|
2052
2058
|
account: this.account,
|
|
@@ -2067,6 +2073,23 @@ class GmailClient extends BaseClient {
|
|
|
2067
2073
|
err
|
|
2068
2074
|
});
|
|
2069
2075
|
}
|
|
2076
|
+
} else {
|
|
2077
|
+
// pubSubApp is linked and a renewal is due, but the topic/IAM markers required to
|
|
2078
|
+
// arm the watch are not recorded on the linked Pub/Sub app. This is the silent
|
|
2079
|
+
// failure that leaves Gmail push disabled (e.g. Pub/Sub resources pre-provisioned in
|
|
2080
|
+
// GCP, so ensurePubsub never persisted the markers). Surface it once per connection -
|
|
2081
|
+
// the renewal timer re-fires ~hourly and this branch never updates lastWatch, so an
|
|
2082
|
+
// unguarded warning would otherwise repeat every cycle.
|
|
2083
|
+
if (!this._loggedMissingWatchMarkers) {
|
|
2084
|
+
this._loggedMissingWatchMarkers = true;
|
|
2085
|
+
this.logger.warn({
|
|
2086
|
+
msg: 'Skipping Gmail watch renewal: Pub/Sub app linked but topic/IAM markers are not recorded',
|
|
2087
|
+
account: this.account,
|
|
2088
|
+
pubSubApp: accountData._app?.pubSubApp,
|
|
2089
|
+
hasTopic: !!appData?.pubSubTopic,
|
|
2090
|
+
hasIamPolicy: !!appData?.pubSubIamPolicy
|
|
2091
|
+
});
|
|
2092
|
+
}
|
|
2070
2093
|
}
|
|
2071
2094
|
}
|
|
2072
2095
|
}
|
|
@@ -3076,4 +3099,13 @@ class GmailClient extends BaseClient {
|
|
|
3076
3099
|
}
|
|
3077
3100
|
}
|
|
3078
3101
|
|
|
3079
|
-
|
|
3102
|
+
// PageCursor and the label maps are exported for unit testing. The maps are
|
|
3103
|
+
// exported as frozen copies so importers (or tests) cannot mutate the live
|
|
3104
|
+
// objects used by the running Gmail client.
|
|
3105
|
+
module.exports = {
|
|
3106
|
+
GmailClient,
|
|
3107
|
+
PageCursor,
|
|
3108
|
+
SKIP_LABELS: Object.freeze([...SKIP_LABELS]),
|
|
3109
|
+
SYSTEM_LABELS: Object.freeze({ ...SYSTEM_LABELS }),
|
|
3110
|
+
SYSTEM_NAMES: Object.freeze({ ...SYSTEM_NAMES })
|
|
3111
|
+
};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// IMAP proxy authentication. Extracted from lib/imapproxy/imap-server.js so the
|
|
4
|
+
// auth decision can be unit tested without booting the proxy worker (which uses
|
|
5
|
+
// a parentPort at require time). Only the authentication portion is extracted;
|
|
6
|
+
// the backend IMAP connection config is still built by the server.
|
|
7
|
+
|
|
8
|
+
const logger = require('./logger');
|
|
9
|
+
const settings = require('./settings');
|
|
10
|
+
const { redis } = require('./db');
|
|
11
|
+
const { Account } = require('./account');
|
|
12
|
+
const getSecret = require('./get-secret');
|
|
13
|
+
const { isApiBasedApp } = require('./oauth2-apps');
|
|
14
|
+
const { validateAuthToken, REASON_MESSAGES } = require('./auth-token');
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Builds the IMAP proxy authentication handler.
|
|
18
|
+
*
|
|
19
|
+
* @param {Object} deps
|
|
20
|
+
* @param {Function} deps.call - RPC function passed to the Account instance
|
|
21
|
+
* @returns {Function} async authenticate(auth, session) -> { accountObject, accountData }
|
|
22
|
+
*/
|
|
23
|
+
function createImapProxyAuthHandler({ call }) {
|
|
24
|
+
return async function authenticate(auth, session) {
|
|
25
|
+
let account = auth.username;
|
|
26
|
+
|
|
27
|
+
let imapPassword = await settings.get('imapProxyServerPassword');
|
|
28
|
+
if (!imapPassword || auth.password !== imapPassword) {
|
|
29
|
+
// fall back to API token authentication
|
|
30
|
+
let result = await validateAuthToken({
|
|
31
|
+
password: auth.password,
|
|
32
|
+
account: auth.username,
|
|
33
|
+
requiredScope: 'imap-proxy',
|
|
34
|
+
remoteAddress: session.remoteAddress
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
if (!result.authenticated) {
|
|
38
|
+
let err = new Error(REASON_MESSAGES[result.reason] || 'Access denied, failed to authenticate user');
|
|
39
|
+
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
40
|
+
err.responseStatus = 'NO';
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
let accountObject = new Account({ account, redis, call, secret: await getSecret() });
|
|
46
|
+
let accountData;
|
|
47
|
+
try {
|
|
48
|
+
accountData = await accountObject.loadAccountData();
|
|
49
|
+
} catch (err) {
|
|
50
|
+
let respErr = new Error('Failed to authenticate user');
|
|
51
|
+
respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
52
|
+
respErr.responseStatus = 'NO';
|
|
53
|
+
|
|
54
|
+
if (!err.output || err.output.statusCode !== 404) {
|
|
55
|
+
// only log non-obvious errors
|
|
56
|
+
logger.error({ msg: 'Failed to load account data', account: auth.username, err });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
throw respErr;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (isApiBasedApp(accountData?._app)) {
|
|
63
|
+
let respErr = new Error('IMAP is not supported for API-based accounts');
|
|
64
|
+
respErr.authenticationFailed = true;
|
|
65
|
+
respErr.serverResponseCode = 'ACCOUNTDISABLED';
|
|
66
|
+
respErr.responseStatus = 'NO';
|
|
67
|
+
throw respErr;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!accountData) {
|
|
71
|
+
let err = new Error('Access denied, failed to authenticate user');
|
|
72
|
+
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
73
|
+
err.responseStatus = 'NO';
|
|
74
|
+
throw err;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return { accountObject, accountData };
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { createImapProxyAuthHandler };
|
|
@@ -4,16 +4,15 @@ const { parentPort } = require('worker_threads');
|
|
|
4
4
|
|
|
5
5
|
const config = require('@zone-eu/wild-config');
|
|
6
6
|
const logger = require('../logger');
|
|
7
|
-
const { oauth2Apps, oauth2ProviderData
|
|
7
|
+
const { oauth2Apps, oauth2ProviderData } = require('../oauth2-apps');
|
|
8
8
|
|
|
9
9
|
const { getDuration, getBoolean, resolveCredentials, hasEnvValue, readEnvValue, emitChangeEvent, loadTlsConfig } = require('../tools');
|
|
10
|
-
const {
|
|
10
|
+
const { getLocalAddress } = require('../utils/network');
|
|
11
11
|
|
|
12
12
|
const { redis } = require('../db');
|
|
13
|
-
const {
|
|
13
|
+
const { createImapProxyAuthHandler } = require('../imap-proxy-auth');
|
|
14
14
|
const getSecret = require('../get-secret');
|
|
15
15
|
const settings = require('../settings');
|
|
16
|
-
const tokens = require('../tokens');
|
|
17
16
|
|
|
18
17
|
const { encrypt, decrypt } = require('../encrypt');
|
|
19
18
|
const { Certs } = require('@postalsys/certs');
|
|
@@ -150,108 +149,14 @@ class PassThroughLogger extends PassThrough {
|
|
|
150
149
|
}
|
|
151
150
|
}
|
|
152
151
|
|
|
152
|
+
// Authentication logic lives in lib/imap-proxy-auth.js so it can be unit tested
|
|
153
|
+
// without booting this worker. call() is injected for the Account instance.
|
|
154
|
+
const authenticateImapProxy = createImapProxyAuthHandler({ call });
|
|
155
|
+
|
|
153
156
|
async function onAuth(auth, session) {
|
|
154
157
|
let account = auth.username;
|
|
155
158
|
|
|
156
|
-
let
|
|
157
|
-
let authPass = false;
|
|
158
|
-
|
|
159
|
-
if (!imapPassword || auth.password !== imapPassword) {
|
|
160
|
-
if (/^[0-9a-f]{64}$/i.test(auth.password)) {
|
|
161
|
-
// fallback to tokens
|
|
162
|
-
let tokenData;
|
|
163
|
-
try {
|
|
164
|
-
tokenData = await tokens.get(auth.password, false, { log: true, remoteAddress: session.remoteAddress });
|
|
165
|
-
} catch (err) {
|
|
166
|
-
logger.error({
|
|
167
|
-
msg: 'Failed to fetch token',
|
|
168
|
-
err
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
if (tokenData) {
|
|
173
|
-
if (tokenData.account && tokenData.account !== auth.username) {
|
|
174
|
-
let err = new Error('Access denied, invalid username');
|
|
175
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
176
|
-
err.responseStatus = 'NO';
|
|
177
|
-
throw err;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
if (tokenData.scopes && !tokenData.scopes.includes('imap-proxy') && !tokenData.scopes.includes('*')) {
|
|
181
|
-
logger.error({
|
|
182
|
-
msg: 'Trying to use invalid scope for a token',
|
|
183
|
-
tokenAccount: tokenData.account,
|
|
184
|
-
tokenId: tokenData.id,
|
|
185
|
-
account,
|
|
186
|
-
requestedScope: 'imap-proxy',
|
|
187
|
-
scopes: tokenData.scopes
|
|
188
|
-
});
|
|
189
|
-
|
|
190
|
-
let err = new Error('Access denied, invalid scope');
|
|
191
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
192
|
-
err.responseStatus = 'NO';
|
|
193
|
-
throw err;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(session.remoteAddress, tokenData.restrictions.addresses)) {
|
|
197
|
-
logger.error({
|
|
198
|
-
msg: 'Trying to use invalid IP for a token',
|
|
199
|
-
tokenAccount: tokenData.account,
|
|
200
|
-
tokenId: tokenData.id,
|
|
201
|
-
account,
|
|
202
|
-
remoteAddress: session.remoteAddress,
|
|
203
|
-
addressAllowlist: tokenData.restrictions.addresses
|
|
204
|
-
});
|
|
205
|
-
|
|
206
|
-
let err = new Error('Access denied, traffic not accepted from this IP');
|
|
207
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
208
|
-
err.responseStatus = 'NO';
|
|
209
|
-
throw err;
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
authPass = true;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
if (!authPass) {
|
|
217
|
-
let err = new Error('Access denied, failed to authenticate user');
|
|
218
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
219
|
-
err.responseStatus = 'NO';
|
|
220
|
-
throw err;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
let accountObject = new Account({ account, redis, call, secret: await getSecret() });
|
|
225
|
-
let accountData;
|
|
226
|
-
try {
|
|
227
|
-
accountData = await accountObject.loadAccountData();
|
|
228
|
-
} catch (err) {
|
|
229
|
-
let respErr = new Error('Failed to authenticate user');
|
|
230
|
-
respErr.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
231
|
-
respErr.responseStatus = 'NO';
|
|
232
|
-
|
|
233
|
-
if (!err.output || err.output.statusCode !== 404) {
|
|
234
|
-
// only log non-obvious errors
|
|
235
|
-
logger.error({ msg: 'Failed to load account data', account: auth.username, err });
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
throw respErr;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
if (isApiBasedApp(accountData?._app)) {
|
|
242
|
-
let respErr = new Error('IMAP is not supported for API-based accounts');
|
|
243
|
-
respErr.authenticationFailed = true;
|
|
244
|
-
respErr.serverResponseCode = 'ACCOUNTDISABLED';
|
|
245
|
-
respErr.responseStatus = 'NO';
|
|
246
|
-
throw respErr;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
if (!accountData) {
|
|
250
|
-
let err = new Error('Access denied, failed to authenticate user');
|
|
251
|
-
err.serverResponseCode = 'AUTHENTICATIONFAILED';
|
|
252
|
-
err.responseStatus = 'NO';
|
|
253
|
-
throw err;
|
|
254
|
-
}
|
|
159
|
+
let { accountObject, accountData } = await authenticateImapProxy(auth, session);
|
|
255
160
|
|
|
256
161
|
if (!accountData.imap && !accountData.oauth2) {
|
|
257
162
|
// can not make connection
|