emailengine-app 2.61.4 → 2.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/data/google-crawlers.json +1 -1
- package/lib/account.js +20 -7
- package/lib/api-routes/account-routes.js +28 -5
- package/lib/api-routes/chat-routes.js +1 -1
- package/lib/api-routes/export-routes.js +316 -0
- package/lib/api-routes/message-routes.js +28 -23
- package/lib/api-routes/template-routes.js +28 -7
- package/lib/arf-detect.js +1 -1
- package/lib/consts.js +16 -0
- package/lib/db.js +3 -0
- package/lib/email-client/base-client.js +6 -4
- package/lib/email-client/gmail-client.js +204 -33
- package/lib/email-client/imap/mailbox.js +99 -8
- package/lib/email-client/imap/subconnection.js +5 -5
- package/lib/email-client/imap-client.js +76 -16
- package/lib/email-client/message-builder.js +3 -1
- package/lib/email-client/notification-handler.js +12 -9
- package/lib/email-client/outlook-client.js +362 -69
- package/lib/email-client/smtp-pool-manager.js +1 -1
- package/lib/export.js +528 -0
- package/lib/oauth/gmail.js +21 -13
- package/lib/oauth/mail-ru.js +23 -10
- package/lib/oauth/outlook.js +26 -16
- package/lib/oauth/pubsub/google.js +5 -0
- package/lib/routes-ui.js +236 -2
- package/lib/schemas.js +260 -80
- package/lib/stream-encrypt.js +263 -0
- package/lib/tools.js +30 -4
- package/lib/ui-routes/account-routes.js +24 -1
- package/lib/ui-routes/admin-config-routes.js +11 -4
- package/lib/ui-routes/admin-entities-routes.js +18 -0
- package/lib/webhooks.js +16 -20
- package/package.json +17 -17
- package/sbom.json +1 -1
- package/server.js +41 -5
- package/static/js/ace/ace.js +1 -1
- package/static/js/ace/ext-language_tools.js +1 -1
- package/static/licenses.html +47 -127
- package/translations/de.mo +0 -0
- package/translations/de.po +63 -36
- package/translations/en.mo +0 -0
- package/translations/en.po +64 -37
- package/translations/et.mo +0 -0
- package/translations/et.po +63 -36
- package/translations/fr.mo +0 -0
- package/translations/fr.po +63 -36
- package/translations/ja.mo +0 -0
- package/translations/ja.po +63 -36
- package/translations/messages.pot +88 -55
- package/translations/nl.mo +0 -0
- package/translations/nl.po +63 -36
- package/translations/pl.mo +0 -0
- package/translations/pl.po +63 -36
- package/views/accounts/account.hbs +375 -2
- package/views/config/service.hbs +35 -0
- package/workers/api.js +124 -45
- package/workers/documents.js +1 -0
- package/workers/export.js +926 -0
- package/workers/imap.js +29 -0
- package/workers/submit.js +25 -5
- package/workers/webhooks.js +11 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,92 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [2.62.0](https://github.com/postalsys/emailengine/compare/v2.61.5...v2.62.0) (2026-02-06)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* add AES-256-GCM encryption at rest for export files ([4f67269](https://github.com/postalsys/emailengine/commit/4f6726915bd40087c0f447a7f2cb36943e8a849a))
|
|
9
|
+
* add configurable batch sizes for Gmail/Outlook exports ([a4d178c](https://github.com/postalsys/emailengine/commit/a4d178ceee195825d7f05a263897a873c361b296))
|
|
10
|
+
* add export beta notice and status indicator in UI ([271bd18](https://github.com/postalsys/emailengine/commit/271bd1899241903c50f582827e0f08306c79cb1d))
|
|
11
|
+
* add export reliability improvements and resume capability ([9f78644](https://github.com/postalsys/emailengine/commit/9f78644cc1801bf64725bbde47c0910aab69769b))
|
|
12
|
+
* add export UI to admin account page ([cf3ce49](https://github.com/postalsys/emailengine/commit/cf3ce4979b6a4547081c2bea534a7327ebe73009))
|
|
13
|
+
* add global concurrent export limit and performance optimizations ([20a1272](https://github.com/postalsys/emailengine/commit/20a1272ca20ee9961c5163a0c5d8c5452cb6b556))
|
|
14
|
+
* add include attachments option to export UI ([2b8d2c6](https://github.com/postalsys/emailengine/commit/2b8d2c6cd60eb72556a1f1a4db03abf0cdc14e6f))
|
|
15
|
+
* add parallel message fetching for Gmail/Outlook export ([12de963](https://github.com/postalsys/emailengine/commit/12de963abe8f6922a8840e8291e57a5dcf03e584))
|
|
16
|
+
* display export expiration date in UI ([5e1e633](https://github.com/postalsys/emailengine/commit/5e1e6334e13b3b1ff012bfd76b933548c6cb6d83))
|
|
17
|
+
* expose outlookSubscription in account info API ([0d97b51](https://github.com/postalsys/emailengine/commit/0d97b51bd60458ad7a8cdd9170aa50605fd1c127))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Bug Fixes
|
|
21
|
+
|
|
22
|
+
* accept base64-encoded nonces for backward compatibility ([2c46822](https://github.com/postalsys/emailengine/commit/2c46822a46ce1b2dc632f9a20f5831eabad3b623))
|
|
23
|
+
* add BullMQ stalled job configuration to prevent queue hangs ([23a74c3](https://github.com/postalsys/emailengine/commit/23a74c359e131baafd12dcf08d213eac0d96840f))
|
|
24
|
+
* add labels to remaining Joi schemas for stable OpenAPI names ([4e7e064](https://github.com/postalsys/emailengine/commit/4e7e064063f9e882ec94ada9d0dd1aad6567c675))
|
|
25
|
+
* add null guards to prevent unhandled exceptions ([9f48f64](https://github.com/postalsys/emailengine/commit/9f48f64b4ded00afa819d06d85be469ef4ea3ab6))
|
|
26
|
+
* address 14 bugs found since v2.61.5 ([839ce55](https://github.com/postalsys/emailengine/commit/839ce55502d28a23f86868012a51bd0077b31f65))
|
|
27
|
+
* address 5 release blockers for export and account APIs ([762c201](https://github.com/postalsys/emailengine/commit/762c20112e9b4de705f037659702eca18b750092))
|
|
28
|
+
* address 6 critical/high bugs and make export limits opt-in ([eb14c69](https://github.com/postalsys/emailengine/commit/eb14c69db2fad7c6fe8fd140de24c41f26922f48))
|
|
29
|
+
* address critical, high, and medium export feature issues ([576345a](https://github.com/postalsys/emailengine/commit/576345ad82d30193a13eea1ea130bf711cfae483))
|
|
30
|
+
* address must-fix and should-fix issues for release ([6def6ef](https://github.com/postalsys/emailengine/commit/6def6efc809e204fc39bd1bf6181a477fae0481f))
|
|
31
|
+
* address verified warnings from release review ([bb08f54](https://github.com/postalsys/emailengine/commit/bb08f54c2aba248e4204b704a6730c8bdf1eeb0e))
|
|
32
|
+
* downgrade transient connection/timeout error logs to warn level ([9064529](https://github.com/postalsys/emailengine/commit/9064529a0d21cd6198386c73d9abca3447f6dc31))
|
|
33
|
+
* downgrade transient PubSub poll errors from error to warn ([d9f2e2f](https://github.com/postalsys/emailengine/commit/d9f2e2f6300b7e99a9ae57c6466fe147614bb891))
|
|
34
|
+
* enrich OAuth token error messages for BullMQ job visibility ([29da53f](https://github.com/postalsys/emailengine/commit/29da53ff38c12e59b54468f5be3579c758a651c0))
|
|
35
|
+
* force Swagger UI to light mode only ([4c26d12](https://github.com/postalsys/emailengine/commit/4c26d12437feaacd26da88a6fb3e85a7ca2ee238))
|
|
36
|
+
* guard against null job in export worker BullMQ failed handler ([4cb1532](https://github.com/postalsys/emailengine/commit/4cb15324b913b7a8fbcbf4f5714c26e3c06226b3))
|
|
37
|
+
* handle missing attachments in ARF detection ([fe08f6a](https://github.com/postalsys/emailengine/commit/fe08f6a8c3120a01e04860c5770f78e925678797))
|
|
38
|
+
* handle missing attachments in Outlook message conversion ([46cf25a](https://github.com/postalsys/emailengine/commit/46cf25adefa9ca615f05381e257ee13a6c3e49b9))
|
|
39
|
+
* handle non-iterable messageInfo.attachments in mailbox sync ([3187571](https://github.com/postalsys/emailengine/commit/31875714e0ceb8e8711c910f6c561e1b1fe81a50))
|
|
40
|
+
* handle notificationBaseUrl without trailing slash in prepareUrl ([1048818](https://github.com/postalsys/emailengine/commit/1048818fe1b1a193a8d5f1b99a37285b54bd9317))
|
|
41
|
+
* handle uncaught EPIPE in ResponseStream for SSE endpoints ([ca7af4a](https://github.com/postalsys/emailengine/commit/ca7af4ad7a4fa6ccc02dcaee79f06d33bd38e8b8))
|
|
42
|
+
* harden OAuth token request body serialization and error handling ([296c4e9](https://github.com/postalsys/emailengine/commit/296c4e99fd00928fa0682f2c735c2d81c1a74bcf))
|
|
43
|
+
* improve BullMQ efficiency with jitter, retention, and cleanup ([fb48fd8](https://github.com/postalsys/emailengine/commit/fb48fd84065fe44a4bf5054ced45f4bffc9c8153))
|
|
44
|
+
* improve packUid robustness with fallback and validation ([4b4253e](https://github.com/postalsys/emailengine/commit/4b4253e3fd025070b42c3d012071dd6376191692))
|
|
45
|
+
* improve submit resilience during worker restarts and add batch endpoint ([f559c38](https://github.com/postalsys/emailengine/commit/f559c3818bb727b137f5a2b8b32b0efee48a093d))
|
|
46
|
+
* leverage Nodemailer error codes for better retry logic and UI messages ([0f3068a](https://github.com/postalsys/emailengine/commit/0f3068abc4b63d87a9f93ba927f8be07a5737f0c))
|
|
47
|
+
* preserve threadId for large Gmail threaded replies via multipart upload ([b04c2ca](https://github.com/postalsys/emailengine/commit/b04c2cac6bc7a6dfe347a800b57ed0bd1a21291a))
|
|
48
|
+
* prevent ArrayBuffer detachment and IMAP null reference errors ([3b97372](https://github.com/postalsys/emailengine/commit/3b9737234b6f71ae9c1adbf4018134cc6bb4a070))
|
|
49
|
+
* prevent concurrent export race condition with atomic Redis operation ([dbc14f6](https://github.com/postalsys/emailengine/commit/dbc14f683ef8509003699a12ff60ec26001522fa))
|
|
50
|
+
* prevent sync state corruption from invalid uidNext values ([533f026](https://github.com/postalsys/emailengine/commit/533f026933fbbefb8b6998e412dc8cf3693c2b9b))
|
|
51
|
+
* prevent sync state corruption from invalid uidValidity values ([976fdb7](https://github.com/postalsys/emailengine/commit/976fdb7a1a74127c6c738b331f2c5aa8b7daddb4))
|
|
52
|
+
* prevent UTF-8 data corruption in OAuth request Buffer handling ([14361fd](https://github.com/postalsys/emailengine/commit/14361fdc4c7c04ad2ca5934eb4d418abf1c66289))
|
|
53
|
+
* reject invalid nonce format instead of silently regenerating ([422ea5c](https://github.com/postalsys/emailengine/commit/422ea5c3dc56f4cf1679a1783d4c108d1486f455))
|
|
54
|
+
* remove BullMQ job when marking interrupted exports as failed ([ad587a7](https://github.com/postalsys/emailengine/commit/ad587a7e0e536f2cb1647f420daa338e24d24233))
|
|
55
|
+
* replace blocking scryptSync with async scrypt in DecryptStream ([c9c6ede](https://github.com/postalsys/emailengine/commit/c9c6edefb11bf7908e5fd64118ff4211f5f6bd4e))
|
|
56
|
+
* resolve 11 bugs in export functionality ([f5d2621](https://github.com/postalsys/emailengine/commit/f5d2621bbaa2cf3b1dab06c67a7e2d5bf29075ed))
|
|
57
|
+
* resolve Gmail label IDs to human-readable names ([02c306f](https://github.com/postalsys/emailengine/commit/02c306f10f29d8941f242f9283684621fefcd5c3))
|
|
58
|
+
* resolve OpenAPI spec validation errors for token restrictions ([a9cffe1](https://github.com/postalsys/emailengine/commit/a9cffe150843fb7f8b4f7f5d01afc69ef451bac2))
|
|
59
|
+
* restore retry for empty Buffer payloads and fix large threaded Gmail replies ([107c164](https://github.com/postalsys/emailengine/commit/107c16475a2278a9428903d7280f481cf62a64ec))
|
|
60
|
+
* return WorkerNotAvailable immediately and remove batch submit endpoint ([aed5c45](https://github.com/postalsys/emailengine/commit/aed5c45aeec37a42b6c7113a3e45cb7153ae67bb))
|
|
61
|
+
* revert jQuery to 3.7.1 and harden export resilience ([43f2a74](https://github.com/postalsys/emailengine/commit/43f2a74587fc8915dbb6dce6497f60a8159f6140))
|
|
62
|
+
* send Buffer for Outlook sendMail base64 payload to avoid JSON quoting ([e89a924](https://github.com/postalsys/emailengine/commit/e89a924343bf8ba3ea1fd7e75bc09aa53c2177b2))
|
|
63
|
+
* share Lock instance across Account objects to prevent Redis connection leak ([56f421b](https://github.com/postalsys/emailengine/commit/56f421b866a12187937cfa935764878f14f0ab1b))
|
|
64
|
+
* stabilize Swagger model names for SDK generation ([8078830](https://github.com/postalsys/emailengine/commit/80788305f23eee9d021b4c72dce21961493ca093))
|
|
65
|
+
* tighten export route validation and apply default export limits ([f45d83f](https://github.com/postalsys/emailengine/commit/f45d83f1d27ab05984f1a0213174025f5f79d71e))
|
|
66
|
+
* update test expectations for email-text-tools 2.3.5+ behavior ([1e28abf](https://github.com/postalsys/emailengine/commit/1e28abf77282ac4c550f9834f61064ae8fde7de9))
|
|
67
|
+
* update test expectations for email-text-tools 2.4.x ([6333fa3](https://github.com/postalsys/emailengine/commit/6333fa3308b1b822a5c364ad5cb25c1c211edeca))
|
|
68
|
+
* use consistent index source in batch submit success and failure paths ([3a73704](https://github.com/postalsys/emailengine/commit/3a737047c1f1e4fda6bd7c4b6822ca31ebe27fd2))
|
|
69
|
+
* validate nonce format before using data.n from cached URLs ([d825303](https://github.com/postalsys/emailengine/commit/d82530395e93e88f5841156a53fe41d476eec9da))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
### Performance Improvements
|
|
73
|
+
|
|
74
|
+
* use MS Graph batch API for Outlook message export ([f031f77](https://github.com/postalsys/emailengine/commit/f031f770278a14aeb2fd5a589e31bb5621b0c410))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
### Reverts
|
|
78
|
+
|
|
79
|
+
* remove Swagger UI light mode forcing ([9c9bd25](https://github.com/postalsys/emailengine/commit/9c9bd25e4488828e3112e671df12d1c551957dac))
|
|
80
|
+
|
|
81
|
+
## [2.61.5](https://github.com/postalsys/emailengine/compare/v2.61.4...v2.61.5) (2026-01-15)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
### Bug Fixes
|
|
85
|
+
|
|
86
|
+
* use base64url encoding for OAuth state nonce in /v1/authentication/form ([1f2cecf](https://github.com/postalsys/emailengine/commit/1f2cecf9efbee8a12c3a0d27c9879bfbbf7dfa39))
|
|
87
|
+
* use base64url encoding for OAuth state nonce in /v1/authentication/form ([dead38c](https://github.com/postalsys/emailengine/commit/dead38c348f1c0204f2bfba09997090cb86c348b))
|
|
88
|
+
* use base64url encoding for OAuth state nonce in remaining locations ([961b710](https://github.com/postalsys/emailengine/commit/961b710357d836782c9bbce329178aa96e08ee20))
|
|
89
|
+
|
|
3
90
|
## [2.61.4](https://github.com/postalsys/emailengine/compare/v2.61.3...v2.61.4) (2026-01-14)
|
|
4
91
|
|
|
5
92
|
|
package/lib/account.js
CHANGED
|
@@ -23,12 +23,20 @@ const settings = require('./settings');
|
|
|
23
23
|
const redisScanDelete = require('./redis-scan-delete');
|
|
24
24
|
const { customAlphabet } = require('nanoid');
|
|
25
25
|
const Lock = require('ioredfour');
|
|
26
|
+
const { redis } = require('./db');
|
|
26
27
|
const nanoid = customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 16);
|
|
27
28
|
const { REDIS_PREFIX, ACCOUNT_DELETED_NOTIFY, MAILBOX_HASH } = require('./consts');
|
|
28
29
|
const { mimeHtml } = require('@postalsys/email-text-tools');
|
|
29
30
|
const { checkAccountScopes: checkScopes } = require('./oauth/scope-checker');
|
|
30
31
|
const { ACCOUNT_STATES, calculateEffectiveState, validateAccountState, getDisplayState, formatLastError } = require('./account/account-state');
|
|
31
32
|
|
|
33
|
+
// Shared Lock instance to prevent Redis connection leak (one subscriber per Lock instance)
|
|
34
|
+
// Previously, each Account created its own Lock, causing ~30k connections with 30k accounts
|
|
35
|
+
const defaultLock = new Lock({
|
|
36
|
+
redis,
|
|
37
|
+
namespace: 'ee'
|
|
38
|
+
});
|
|
39
|
+
|
|
32
40
|
class Account {
|
|
33
41
|
constructor(options) {
|
|
34
42
|
this.redis = options.redis;
|
|
@@ -48,13 +56,7 @@ class Account {
|
|
|
48
56
|
}
|
|
49
57
|
|
|
50
58
|
getLock() {
|
|
51
|
-
|
|
52
|
-
this.lock = new Lock({
|
|
53
|
-
redis: this.redis,
|
|
54
|
-
namespace: 'ee'
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
return this.lock;
|
|
59
|
+
return this.lock || defaultLock;
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
/**
|
|
@@ -1602,6 +1604,17 @@ class Account {
|
|
|
1602
1604
|
return messageData;
|
|
1603
1605
|
}
|
|
1604
1606
|
|
|
1607
|
+
async getMessages(messageIds, options) {
|
|
1608
|
+
await this.loadAccountData(this.account, true);
|
|
1609
|
+
return await this.call({
|
|
1610
|
+
cmd: 'getMessages',
|
|
1611
|
+
account: this.account,
|
|
1612
|
+
messageIds,
|
|
1613
|
+
options,
|
|
1614
|
+
timeout: this.timeout
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1605
1618
|
async listMessages(query) {
|
|
1606
1619
|
if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
|
|
1607
1620
|
await this.loadAccountData(this.account, false);
|
|
@@ -368,7 +368,7 @@ async function init(args) {
|
|
|
368
368
|
response: {
|
|
369
369
|
schema: Joi.object({
|
|
370
370
|
account: accountIdSchema.required()
|
|
371
|
-
}),
|
|
371
|
+
}).label('UpdateAccountResponse'),
|
|
372
372
|
failAction: 'log'
|
|
373
373
|
}
|
|
374
374
|
}
|
|
@@ -568,7 +568,7 @@ async function init(args) {
|
|
|
568
568
|
schema: Joi.object({
|
|
569
569
|
account: accountIdSchema.required(),
|
|
570
570
|
deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the account deleted')
|
|
571
|
-
}).label('
|
|
571
|
+
}).label('DeleteAccountResponse'),
|
|
572
572
|
failAction: 'log'
|
|
573
573
|
}
|
|
574
574
|
}
|
|
@@ -732,7 +732,8 @@ async function init(args) {
|
|
|
732
732
|
.required()
|
|
733
733
|
.valid('init', 'syncing', 'connecting', 'connected', 'authenticationError', 'connectError', 'unset', 'disconnected')
|
|
734
734
|
.example('connected')
|
|
735
|
-
.description('Account state')
|
|
735
|
+
.description('Account state')
|
|
736
|
+
.label('AccountListState'),
|
|
736
737
|
webhooks: Joi.string()
|
|
737
738
|
.uri({
|
|
738
739
|
scheme: ['http', 'https'],
|
|
@@ -816,13 +817,24 @@ async function init(args) {
|
|
|
816
817
|
'connections',
|
|
817
818
|
'webhooksCustomHeaders',
|
|
818
819
|
'locale',
|
|
819
|
-
'tz'
|
|
820
|
+
'tz',
|
|
821
|
+
'outlookSubscription'
|
|
820
822
|
]) {
|
|
821
823
|
if (key in accountData) {
|
|
822
824
|
result[key] = accountData[key];
|
|
823
825
|
}
|
|
824
826
|
}
|
|
825
827
|
|
|
828
|
+
if (result.outlookSubscription) {
|
|
829
|
+
try {
|
|
830
|
+
let parsed = typeof result.outlookSubscription === 'string' ? JSON.parse(result.outlookSubscription) : result.outlookSubscription;
|
|
831
|
+
delete parsed.clientState;
|
|
832
|
+
result.outlookSubscription = parsed;
|
|
833
|
+
} catch (err) {
|
|
834
|
+
result.outlookSubscription = {};
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
826
838
|
// default false
|
|
827
839
|
for (let key of ['logs']) {
|
|
828
840
|
result[key] = !!result[key];
|
|
@@ -995,7 +1007,8 @@ async function init(args) {
|
|
|
995
1007
|
.trim()
|
|
996
1008
|
.valid(...['imap', 'api', 'pubsub'])
|
|
997
1009
|
.example('imap')
|
|
998
|
-
.description('OAuth2 Base Scopes')
|
|
1010
|
+
.description('OAuth2 Base Scopes')
|
|
1011
|
+
.label('AccountBaseScopes'),
|
|
999
1012
|
|
|
1000
1013
|
counters: accountCountersSchema,
|
|
1001
1014
|
|
|
@@ -1012,6 +1025,16 @@ async function init(args) {
|
|
|
1012
1025
|
|
|
1013
1026
|
syncTime: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('Last sync time'),
|
|
1014
1027
|
|
|
1028
|
+
outlookSubscription: Joi.object({
|
|
1029
|
+
id: Joi.string().description('Microsoft Graph subscription ID'),
|
|
1030
|
+
expirationDateTime: Joi.date().iso().description('When the subscription expires'),
|
|
1031
|
+
state: Joi.object({
|
|
1032
|
+
state: Joi.string().valid('creating', 'created', 'error').description('Subscription state'),
|
|
1033
|
+
time: Joi.number().description('Timestamp of last state change'),
|
|
1034
|
+
error: Joi.string().description('Error message if state is error')
|
|
1035
|
+
}).description('Current subscription state')
|
|
1036
|
+
}).description('Microsoft Graph subscription details (Outlook accounts only)'),
|
|
1037
|
+
|
|
1015
1038
|
lastError: lastErrorSchema.allow(null)
|
|
1016
1039
|
}).label('AccountResponse'),
|
|
1017
1040
|
failAction: 'log'
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const { Export } = require('../export');
|
|
5
|
+
const Boom = require('@hapi/boom');
|
|
6
|
+
const Joi = require('joi');
|
|
7
|
+
const { failAction } = require('../tools');
|
|
8
|
+
const { accountIdSchema, exportRequestSchema, exportStatusSchema, exportListSchema, exportIdSchema } = require('../schemas');
|
|
9
|
+
const getSecret = require('../get-secret');
|
|
10
|
+
const { createDecryptStream } = require('../stream-encrypt');
|
|
11
|
+
|
|
12
|
+
function handleError(request, err) {
|
|
13
|
+
request.logger.error({ msg: 'API request failed', err });
|
|
14
|
+
if (Boom.isBoom(err)) {
|
|
15
|
+
throw err;
|
|
16
|
+
}
|
|
17
|
+
const error = Boom.boomify(err, { statusCode: err.statusCode || 500 });
|
|
18
|
+
if (err.code) {
|
|
19
|
+
error.output.payload.code = err.code;
|
|
20
|
+
}
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async function init(args) {
|
|
25
|
+
const { server, CORS_CONFIG } = args;
|
|
26
|
+
|
|
27
|
+
server.route({
|
|
28
|
+
method: 'POST',
|
|
29
|
+
path: '/v1/account/{account}/export',
|
|
30
|
+
|
|
31
|
+
async handler(request) {
|
|
32
|
+
try {
|
|
33
|
+
return await Export.create(request.params.account, {
|
|
34
|
+
folders: request.payload.folders,
|
|
35
|
+
startDate: request.payload.startDate,
|
|
36
|
+
endDate: request.payload.endDate,
|
|
37
|
+
textType: request.payload.textType,
|
|
38
|
+
maxBytes: request.payload.maxBytes,
|
|
39
|
+
includeAttachments: request.payload.includeAttachments
|
|
40
|
+
});
|
|
41
|
+
} catch (err) {
|
|
42
|
+
handleError(request, err);
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
options: {
|
|
47
|
+
description: 'Create export',
|
|
48
|
+
notes: 'Creates a new bulk message export job. The export runs asynchronously and notifies via webhook when complete.',
|
|
49
|
+
tags: ['api', 'Export (Beta)'],
|
|
50
|
+
|
|
51
|
+
auth: {
|
|
52
|
+
strategy: 'api-token',
|
|
53
|
+
mode: 'required'
|
|
54
|
+
},
|
|
55
|
+
cors: CORS_CONFIG,
|
|
56
|
+
|
|
57
|
+
validate: {
|
|
58
|
+
options: {
|
|
59
|
+
stripUnknown: false,
|
|
60
|
+
abortEarly: false,
|
|
61
|
+
convert: true
|
|
62
|
+
},
|
|
63
|
+
failAction,
|
|
64
|
+
|
|
65
|
+
params: Joi.object({
|
|
66
|
+
account: accountIdSchema.required()
|
|
67
|
+
}).label('CreateExportParams'),
|
|
68
|
+
|
|
69
|
+
payload: exportRequestSchema
|
|
70
|
+
},
|
|
71
|
+
|
|
72
|
+
response: {
|
|
73
|
+
schema: Joi.object({
|
|
74
|
+
exportId: Joi.string().example('exp_abc123def456abc123def456').description('Export job identifier'),
|
|
75
|
+
status: Joi.string().example('queued').description('Export status'),
|
|
76
|
+
created: Joi.date().iso().example('2024-01-15T10:30:00Z').description('When export was created')
|
|
77
|
+
}).label('CreateExportResponse'),
|
|
78
|
+
failAction: 'log'
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
server.route({
|
|
84
|
+
method: 'GET',
|
|
85
|
+
path: '/v1/account/{account}/export/{exportId}',
|
|
86
|
+
|
|
87
|
+
async handler(request) {
|
|
88
|
+
try {
|
|
89
|
+
const result = await Export.get(request.params.account, request.params.exportId);
|
|
90
|
+
if (!result) {
|
|
91
|
+
throw Boom.notFound('Export not found');
|
|
92
|
+
}
|
|
93
|
+
return result;
|
|
94
|
+
} catch (err) {
|
|
95
|
+
handleError(request, err);
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
|
|
99
|
+
options: {
|
|
100
|
+
description: 'Get export status',
|
|
101
|
+
notes: 'Returns the status and progress of an export job.',
|
|
102
|
+
tags: ['api', 'Export (Beta)'],
|
|
103
|
+
|
|
104
|
+
auth: {
|
|
105
|
+
strategy: 'api-token',
|
|
106
|
+
mode: 'required'
|
|
107
|
+
},
|
|
108
|
+
cors: CORS_CONFIG,
|
|
109
|
+
|
|
110
|
+
validate: {
|
|
111
|
+
options: {
|
|
112
|
+
stripUnknown: false,
|
|
113
|
+
abortEarly: false,
|
|
114
|
+
convert: true
|
|
115
|
+
},
|
|
116
|
+
failAction,
|
|
117
|
+
|
|
118
|
+
params: Joi.object({
|
|
119
|
+
account: accountIdSchema.required(),
|
|
120
|
+
exportId: exportIdSchema
|
|
121
|
+
}).label('GetExportParams')
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
response: {
|
|
125
|
+
schema: exportStatusSchema,
|
|
126
|
+
failAction: 'log'
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
server.route({
|
|
132
|
+
method: 'GET',
|
|
133
|
+
path: '/v1/account/{account}/export/{exportId}/download',
|
|
134
|
+
|
|
135
|
+
async handler(request, h) {
|
|
136
|
+
try {
|
|
137
|
+
const { account, exportId } = request.params;
|
|
138
|
+
const fileInfo = await Export.getFile(account, exportId);
|
|
139
|
+
if (!fileInfo) {
|
|
140
|
+
throw Boom.notFound('Export not found');
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const fileReadStream = fs.createReadStream(fileInfo.filePath);
|
|
144
|
+
let stream = fileReadStream;
|
|
145
|
+
|
|
146
|
+
stream.on('error', err => {
|
|
147
|
+
request.logger.error({ msg: 'Export download stream error', exportId, err });
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
if (fileInfo.isEncrypted) {
|
|
151
|
+
const secret = await getSecret();
|
|
152
|
+
if (!secret) {
|
|
153
|
+
fileReadStream.destroy();
|
|
154
|
+
throw Boom.serverUnavailable('Encryption secret not available for decryption');
|
|
155
|
+
}
|
|
156
|
+
const decryptStream = await createDecryptStream(secret);
|
|
157
|
+
decryptStream.on('error', err => {
|
|
158
|
+
request.logger.error({ msg: 'Export decryption error', exportId, err });
|
|
159
|
+
fileReadStream.destroy();
|
|
160
|
+
});
|
|
161
|
+
stream = fileReadStream.pipe(decryptStream);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return h
|
|
165
|
+
.response(stream)
|
|
166
|
+
.type('application/gzip')
|
|
167
|
+
.header('Content-Disposition', `attachment; filename="${fileInfo.filename}"`)
|
|
168
|
+
.header('Content-Encoding', 'identity');
|
|
169
|
+
} catch (err) {
|
|
170
|
+
handleError(request, err);
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
|
|
174
|
+
options: {
|
|
175
|
+
description: 'Download export file',
|
|
176
|
+
notes: 'Downloads the completed export file as gzip-compressed NDJSON.',
|
|
177
|
+
tags: ['api', 'Export (Beta)'],
|
|
178
|
+
|
|
179
|
+
plugins: {
|
|
180
|
+
'hapi-swagger': {
|
|
181
|
+
produces: ['application/gzip']
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
auth: {
|
|
186
|
+
strategy: 'api-token',
|
|
187
|
+
mode: 'required'
|
|
188
|
+
},
|
|
189
|
+
cors: CORS_CONFIG,
|
|
190
|
+
|
|
191
|
+
validate: {
|
|
192
|
+
options: {
|
|
193
|
+
stripUnknown: false,
|
|
194
|
+
abortEarly: false,
|
|
195
|
+
convert: true
|
|
196
|
+
},
|
|
197
|
+
failAction,
|
|
198
|
+
|
|
199
|
+
params: Joi.object({
|
|
200
|
+
account: accountIdSchema.required(),
|
|
201
|
+
exportId: exportIdSchema
|
|
202
|
+
}).label('DownloadExportParams')
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
server.route({
|
|
208
|
+
method: 'DELETE',
|
|
209
|
+
path: '/v1/account/{account}/export/{exportId}',
|
|
210
|
+
|
|
211
|
+
async handler(request) {
|
|
212
|
+
try {
|
|
213
|
+
const deleted = await Export.delete(request.params.account, request.params.exportId);
|
|
214
|
+
if (!deleted) {
|
|
215
|
+
throw Boom.notFound('Export not found');
|
|
216
|
+
}
|
|
217
|
+
return { deleted: true };
|
|
218
|
+
} catch (err) {
|
|
219
|
+
handleError(request, err);
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
|
|
223
|
+
options: {
|
|
224
|
+
description: 'Delete export',
|
|
225
|
+
notes: 'Cancels a pending export or deletes a completed export. Removes both Redis data and the export file.',
|
|
226
|
+
tags: ['api', 'Export (Beta)'],
|
|
227
|
+
|
|
228
|
+
auth: {
|
|
229
|
+
strategy: 'api-token',
|
|
230
|
+
mode: 'required'
|
|
231
|
+
},
|
|
232
|
+
cors: CORS_CONFIG,
|
|
233
|
+
|
|
234
|
+
validate: {
|
|
235
|
+
options: {
|
|
236
|
+
stripUnknown: false,
|
|
237
|
+
abortEarly: false,
|
|
238
|
+
convert: true
|
|
239
|
+
},
|
|
240
|
+
failAction,
|
|
241
|
+
|
|
242
|
+
params: Joi.object({
|
|
243
|
+
account: accountIdSchema.required(),
|
|
244
|
+
exportId: exportIdSchema
|
|
245
|
+
}).label('DeleteExportParams')
|
|
246
|
+
},
|
|
247
|
+
|
|
248
|
+
response: {
|
|
249
|
+
schema: Joi.object({
|
|
250
|
+
deleted: Joi.boolean().example(true).description('True if export was deleted')
|
|
251
|
+
}).label('DeleteExportResponse'),
|
|
252
|
+
failAction: 'log'
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
server.route({
|
|
258
|
+
method: 'GET',
|
|
259
|
+
path: '/v1/account/{account}/exports',
|
|
260
|
+
|
|
261
|
+
async handler(request) {
|
|
262
|
+
try {
|
|
263
|
+
return await Export.list(request.params.account, {
|
|
264
|
+
page: request.query.page,
|
|
265
|
+
pageSize: request.query.pageSize
|
|
266
|
+
});
|
|
267
|
+
} catch (err) {
|
|
268
|
+
handleError(request, err);
|
|
269
|
+
}
|
|
270
|
+
},
|
|
271
|
+
|
|
272
|
+
options: {
|
|
273
|
+
description: 'List exports',
|
|
274
|
+
notes: 'Lists all exports for an account with pagination.',
|
|
275
|
+
tags: ['api', 'Export (Beta)'],
|
|
276
|
+
|
|
277
|
+
auth: {
|
|
278
|
+
strategy: 'api-token',
|
|
279
|
+
mode: 'required'
|
|
280
|
+
},
|
|
281
|
+
cors: CORS_CONFIG,
|
|
282
|
+
|
|
283
|
+
validate: {
|
|
284
|
+
options: {
|
|
285
|
+
stripUnknown: false,
|
|
286
|
+
abortEarly: false,
|
|
287
|
+
convert: true
|
|
288
|
+
},
|
|
289
|
+
failAction,
|
|
290
|
+
|
|
291
|
+
params: Joi.object({
|
|
292
|
+
account: accountIdSchema.required()
|
|
293
|
+
}).label('ListExportsParams'),
|
|
294
|
+
|
|
295
|
+
query: Joi.object({
|
|
296
|
+
page: Joi.number()
|
|
297
|
+
.integer()
|
|
298
|
+
.min(0)
|
|
299
|
+
.max(1024 * 1024)
|
|
300
|
+
.default(0)
|
|
301
|
+
.example(0)
|
|
302
|
+
.description('Page number (zero indexed)')
|
|
303
|
+
.label('PageNumber'),
|
|
304
|
+
pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page').label('PageSize')
|
|
305
|
+
}).label('ListExportsQuery')
|
|
306
|
+
},
|
|
307
|
+
|
|
308
|
+
response: {
|
|
309
|
+
schema: exportListSchema,
|
|
310
|
+
failAction: 'log'
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
module.exports = init;
|