mail-time 5.2.1 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -12
- package/adapters/mongo.d.ts +9 -8
- package/adapters/mongo.js +8 -8
- package/adapters/postgres.d.ts +5 -0
- package/adapters/postgres.js +35 -10
- package/adapters/redis.d.ts +1 -1
- package/adapters/redis.js +37 -18
- package/index.cjs +580 -141
- package/index.d.cts +20 -76
- package/index.d.ts +20 -76
- package/index.js +307 -79
- package/package.json +24 -5
- package/recipient-policy-lease.js +40 -8
- package/recipient-policy.js +155 -20
package/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { PostgresQueue } from './adapters/postgres.js';
|
|
|
7
7
|
import { mailTimePreset, presets, presetNames } from './presets.js';
|
|
8
8
|
import { debug, escapeHtml, logError, hasOwnProp, deepMerge, equals, isPlainObject, extractEmail, toAddressList, filterAddressField } from './helpers.js';
|
|
9
9
|
|
|
10
|
-
import { validateRecipientPolicies, policyError, normalizePolicyAddress, preparePolicyEnvelope, evaluatePolicyPhase, mergePolicyResults, summarizePolicyTask } from './recipient-policy.js';
|
|
10
|
+
import { validateRecipientPolicies, validatePolicyMailOptions, policyError, isAddressError, normalizePolicyAddress, rewritePolicyHeaders, preparePolicyEnvelope, evaluatePolicyPhase, mergePolicyResults, summarizePolicyTask } from './recipient-policy.js';
|
|
11
11
|
import { normalizeRejections } from './recipient-rejections.js';
|
|
12
12
|
import { RecipientPolicyLease } from './recipient-policy-lease.js';
|
|
13
13
|
|
|
@@ -280,9 +280,21 @@ let DEFAULT_TEMPLATE = '<!DOCTYPE html><html xmlns=http://www.w3.org/1999/xhtml>
|
|
|
280
280
|
*/
|
|
281
281
|
|
|
282
282
|
/**
|
|
283
|
-
* @typedef {{
|
|
283
|
+
* @typedef {{ failedWrites: number }} MailTimeDrainResult
|
|
284
284
|
*/
|
|
285
285
|
|
|
286
|
+
/**
|
|
287
|
+
* `details` passed to `onError`. `phase` is `'verify'` (transport verification, `email === null`), `'complete'` (a storage write recording a send outcome), or `'checkpoint'` (recipient-policy results written after SMTP). Without `phase` it is the SMTP `info` of the failed attempt.
|
|
288
|
+
* @typedef {{ phase?: 'verify' | 'complete' | 'checkpoint', transportIndex?: number, attempt?: number, [key: string]: unknown }} MailTimeErrorDetails
|
|
289
|
+
*/
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* @typedef {{ queue: RedisQueue | MongoQueue | PostgresQueue | CustomQueue, type?: 'server' | 'client', from?: string | ((transport: MailTimeTransport, details: MailTimeFromDetails) => string), transports?: MailTimeTransport[], strategy?: 'backup' | 'balancer', failsToNext?: number, shouldFailOver?: (error: unknown, info: object | undefined, email: MailTimeTask) => boolean, retries?: number, maxTries?: number, retryDelay?: number, interval?: number, keepHistory?: boolean, concatEmails?: boolean | MailTimeConcatEmailsOptions, concatSubject?: string, concatDelimiter?: string, concatDelay?: number, concatThrottling?: number, revolvingInterval?: number, mode?: 'one' | 'batch', concurrency?: number, sendingTimeout?: number, renewClaim?: boolean | number, maxRenewals?: number, strictPayload?: boolean, allowedMailFields?: string[], verifyTransports?: boolean, verifyTimeout?: number, template?: string, prefix?: string, debug?: boolean, josk?: MailTimeJoSkOptions, recipientPolicies?: MailTimeRecipientPolicy[], onError?: (error: unknown, email: MailTimeTask | null, details?: MailTimeErrorDetails, recipients?: MailTimeRecipientResult[], summary?: MailTimeRecipientSummary) => void | Promise<void>, onSent?: (email: MailTimeTask, details?: object, recipients?: MailTimeRecipientResult[], summary?: MailTimeRecipientSummary) => void | Promise<void>, onSuppressed?: (email: MailTimeTask, recipients: MailTimeRecipientResult[], summary: MailTimeRecipientSummary) => void | Promise<void>, onRejected?: (email: MailTimeTask, recipients: MailTimeRecipientResult[], summary: MailTimeRecipientSummary) => void | Promise<void> }} MailTimeOptions
|
|
293
|
+
*/
|
|
294
|
+
|
|
295
|
+
const REPROBE_BASE_MS = 60000;
|
|
296
|
+
const REPROBE_MAX_MS = 900000;
|
|
297
|
+
|
|
286
298
|
/**
|
|
287
299
|
* Class of MailTime.
|
|
288
300
|
* With recipientPolicies, terminal callbacks receive grouped recipients and a complete summary after persistence.
|
|
@@ -385,6 +397,7 @@ class MailTime {
|
|
|
385
397
|
this.__readyPromise = null;
|
|
386
398
|
this.__schedulerTimer = null;
|
|
387
399
|
this.__inFlight = new Set();
|
|
400
|
+
this.__failedWrites = 0;
|
|
388
401
|
this.__pool = createPool(this.concurrency);
|
|
389
402
|
|
|
390
403
|
this.failsToNext = (typeof opts.failsToNext === 'number' && opts.failsToNext > 0) ? opts.failsToNext : 4;
|
|
@@ -392,7 +405,12 @@ class MailTime {
|
|
|
392
405
|
this.transports = Array.isArray(opts.transports) ? opts.transports : [];
|
|
393
406
|
this.transport = 0;
|
|
394
407
|
this.verifyTransports = opts.verifyTransports !== false;
|
|
408
|
+
this.verifyTimeout = (typeof opts.verifyTimeout === 'number' && opts.verifyTimeout > 0) ? Math.min(opts.verifyTimeout, 2147483647) : 30000;
|
|
395
409
|
this.__unhealthyTransports = new Set();
|
|
410
|
+
this.__verifyGen = [];
|
|
411
|
+
this.__reprobe = new Map();
|
|
412
|
+
this.__probeCancels = new Set();
|
|
413
|
+
this.__uncertainRenewals = new WeakMap();
|
|
396
414
|
|
|
397
415
|
if (typeof opts.from === 'string') {
|
|
398
416
|
const fromStr = opts.from;
|
|
@@ -403,6 +421,21 @@ class MailTime {
|
|
|
403
421
|
this.from = false;
|
|
404
422
|
}
|
|
405
423
|
|
|
424
|
+
if (this.__recipientPolicies && this.type === 'server') {
|
|
425
|
+
// A transport `from` that the policy parser rejects would settle every rotated letter as
|
|
426
|
+
// failed; surface it here instead. A `from` callback is a runtime value and is not checked.
|
|
427
|
+
if (typeof opts.from === 'string') normalizePolicyAddress(opts.from, 'from');
|
|
428
|
+
this.transports.forEach((transport, i) => {
|
|
429
|
+
const from = MailTime.transportFrom(transport);
|
|
430
|
+
if (from !== void 0) normalizePolicyAddress(from, `transports[${i}].from`);
|
|
431
|
+
// `___compileMailOpts` merges these ahead of `this.from`, so they are sent as-is.
|
|
432
|
+
for (const key of ['_options', 'options']) {
|
|
433
|
+
const mailFrom = transport?.[key]?.mailOptions?.from;
|
|
434
|
+
if (mailFrom !== void 0) normalizePolicyAddress(mailFrom, `transports[${i}].${key}.mailOptions.from`);
|
|
435
|
+
}
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
406
439
|
this.queue.mailTimeInstance = this;
|
|
407
440
|
|
|
408
441
|
/** @type {string} */
|
|
@@ -575,6 +608,8 @@ class MailTime {
|
|
|
575
608
|
|
|
576
609
|
this.__isDestroyed = true;
|
|
577
610
|
this.__isPaused = false;
|
|
611
|
+
for (const cancel of [...this.__probeCancels]) cancel();
|
|
612
|
+
this.__reprobe.clear();
|
|
578
613
|
// Queued jobs never started SMTP; dropping them lets a scan blocked on a
|
|
579
614
|
// pool slot return, so JoSk shutdown does not wait for in-flight sends.
|
|
580
615
|
this.__pool.cancelQueued();
|
|
@@ -610,14 +645,34 @@ class MailTime {
|
|
|
610
645
|
* @async
|
|
611
646
|
* @memberOf MailTime
|
|
612
647
|
* @name drain
|
|
613
|
-
* @description Wait for all in-flight email send attempts to settle
|
|
614
|
-
* @returns {Promise<
|
|
648
|
+
* @description Wait for all in-flight email send attempts to settle. Resolves with `{ failedWrites }`, the count since this instance was created of failures while recording a send outcome: a storage write that threw, an exception in the completion path, or an outcome write lost after a claim-renewal error (message `outcome write lost (renewal outcome uncertain or lease taken over)`, after one retry with the renewal's stamp (no retry when the claim was already stale at renewal time)). Each is also reported once through `onError` with `details.phase` of `'complete'` or `'checkpoint'`, except after a plain `destroy()`, which still counts but suppresses the hook and the log. A row whose write failed can stay `sending` and be re-sent after `sendingTimeout`. Compare `failedWrites` before and after to scope a shutdown.
|
|
649
|
+
* @returns {Promise<MailTimeDrainResult>}
|
|
615
650
|
*/
|
|
616
651
|
async drain() {
|
|
617
652
|
this.__debug('[drain]');
|
|
618
653
|
if (this.__pool) {
|
|
619
654
|
await this.__pool.drain();
|
|
620
655
|
}
|
|
656
|
+
return { failedWrites: this.__failedWrites };
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* @internal
|
|
661
|
+
* @memberOf MailTime
|
|
662
|
+
* @name ___reportWriteFailure
|
|
663
|
+
* @description Count a failed completion step (a storage write that threw, an exception raised inside the completion path, or an outcome write lost after a claim-renewal error) and surface it once through `onError(error, task, { phase })`. Never throws. After a non-draining `destroy()` it still counts but skips the log and the hook.
|
|
664
|
+
* @param {MailTimeTask} task
|
|
665
|
+
* @param {unknown} error
|
|
666
|
+
* @param {'complete' | 'checkpoint'} [phase]
|
|
667
|
+
* @returns {void}
|
|
668
|
+
*/
|
|
669
|
+
___reportWriteFailure(task, error, phase = 'complete') {
|
|
670
|
+
this.__failedWrites++;
|
|
671
|
+
if (this.__abortInFlight) {
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
logError(`[private send] completion error, ${phase} write failed; row ${task?.uuid} may stay claimed and be re-sent after sendingTimeout`, error);
|
|
675
|
+
callHook('onError', this.onError, error, task || null, { phase, attempt: task?.tries, transportIndex: task?.transport });
|
|
621
676
|
}
|
|
622
677
|
|
|
623
678
|
/**
|
|
@@ -688,7 +743,7 @@ class MailTime {
|
|
|
688
743
|
* @description add email to the queue or append to existing letter if {concatEmails: true}
|
|
689
744
|
* @param {MailTimeMailOptions} opts - email options
|
|
690
745
|
* @returns {Promise<string>} uuid of the email
|
|
691
|
-
* @throws {Error}
|
|
746
|
+
* @throws {Error} missing `html`/`text`, a `raw` field, or an invalid `to`; with `recipientPolicies`, an unparseable address rejects with `error.code === 'MAIL_TIME_INVALID_ADDRESS'` and `error.field` naming the field (header recipients are skipped when `envelope.to` is explicit)
|
|
692
747
|
*/
|
|
693
748
|
async sendMail(opts) {
|
|
694
749
|
opts = (opts && typeof opts === 'object') ? opts : {};
|
|
@@ -722,6 +777,10 @@ class MailTime {
|
|
|
722
777
|
throw new Error('[mail-time] [sendMail] `mailOptions.to` is required and must be a string or non-empty Array');
|
|
723
778
|
}
|
|
724
779
|
|
|
780
|
+
if (this.__recipientPolicies) {
|
|
781
|
+
validatePolicyMailOptions(mailOptions);
|
|
782
|
+
}
|
|
783
|
+
|
|
725
784
|
if (this.concatEmails) {
|
|
726
785
|
sendAt = sendAt + this.concatDelay;
|
|
727
786
|
const task = await this.queue.getPendingTo(mailOptions.to, sendAt);
|
|
@@ -803,21 +862,15 @@ class MailTime {
|
|
|
803
862
|
if (task.tries >= this.maxTries) {
|
|
804
863
|
this.___finalizeRejected(task, info);
|
|
805
864
|
|
|
806
|
-
const
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
let finalized = false;
|
|
816
|
-
if (!this.keepHistory) {
|
|
817
|
-
finalized = await this.queue.remove(task, leaseRemove);
|
|
818
|
-
} else {
|
|
819
|
-
finalized = await this.queue.update(task, leaseUpdate);
|
|
820
|
-
}
|
|
865
|
+
const finalized = await this.___outcomeWrite(task, (t) => (!this.keepHistory
|
|
866
|
+
? this.queue.remove(t, __leaseRemoveOpts(t))
|
|
867
|
+
: this.queue.update(t, __withLease(t, {
|
|
868
|
+
isSent: false,
|
|
869
|
+
isFailed: true,
|
|
870
|
+
isSending: false,
|
|
871
|
+
sendingAt: 0,
|
|
872
|
+
mailOptions: task.mailOptions,
|
|
873
|
+
}))));
|
|
821
874
|
|
|
822
875
|
if (!finalized) {
|
|
823
876
|
this.__debug('[private handleError] lease lost before final failure update, skipping onError', task.uuid);
|
|
@@ -843,12 +896,12 @@ class MailTime {
|
|
|
843
896
|
transportIndex = this.___nextHealthyTransport(transportIndex);
|
|
844
897
|
}
|
|
845
898
|
|
|
846
|
-
const released = await this.queue.update(
|
|
899
|
+
const released = await this.___outcomeWrite(task, (t) => this.queue.update(t, __withLease(t, {
|
|
847
900
|
isSending: false,
|
|
848
901
|
sendingAt: 0,
|
|
849
902
|
sendAt: Date.now() + this.retryDelay,
|
|
850
903
|
transport: transportIndex,
|
|
851
|
-
}));
|
|
904
|
+
})));
|
|
852
905
|
|
|
853
906
|
if (!released) {
|
|
854
907
|
this.__debug('[private handleError] lease lost before retry release, skipping', task.uuid);
|
|
@@ -1250,6 +1303,10 @@ class MailTime {
|
|
|
1250
1303
|
}));
|
|
1251
1304
|
} catch (renewError) {
|
|
1252
1305
|
logError('[private renewClaim] storage error during claim renewal', renewError);
|
|
1306
|
+
// Storage may have applied the write before the driver failed; remember the stamp
|
|
1307
|
+
// so the outcome write can retry with it (see ___outcomeWrite).
|
|
1308
|
+
// A stale claim is remembered as null: the lost outcome write is reported, never retried.
|
|
1309
|
+
this.__uncertainRenewals.set(task, task.sendingAt > renewedAt - this.sendingTimeout ? renewedAt : null);
|
|
1253
1310
|
halt();
|
|
1254
1311
|
return;
|
|
1255
1312
|
}
|
|
@@ -1280,6 +1337,38 @@ class MailTime {
|
|
|
1280
1337
|
return { stop };
|
|
1281
1338
|
}
|
|
1282
1339
|
|
|
1340
|
+
/**
|
|
1341
|
+
* @async
|
|
1342
|
+
* @internal
|
|
1343
|
+
* @memberOf MailTime
|
|
1344
|
+
* @name ___outcomeWrite
|
|
1345
|
+
* @description Run a lease-guarded outcome write. `write(t)` must derive its lease guard from `t`. When the write loses the lease and an earlier claim renewal threw (storage may have applied it), retry once with that renewal's stamp; a peer takeover bumps `tries` or stamps a later `sendingAt` while the lease is live, so the retry cannot match a taken-over row. The stamp is only remembered when the lease was still live at renewal time. If the retry also fails, count and report the lost write. A claim that was already stale when the renewal threw is reported without a retry. After `destroy()` the loss is counted without hook or log. Without an earlier renewal error a lost lease stays a debug-only skip.
|
|
1346
|
+
* @param {MailTimeTask} task
|
|
1347
|
+
* @param {(t: MailTimeTask) => Promise<boolean>} write
|
|
1348
|
+
* @returns {Promise<boolean>}
|
|
1349
|
+
*/
|
|
1350
|
+
async ___outcomeWrite(task, write) {
|
|
1351
|
+
if (await write(task)) {
|
|
1352
|
+
this.__uncertainRenewals.delete(task);
|
|
1353
|
+
return true;
|
|
1354
|
+
}
|
|
1355
|
+
const attempted = this.__uncertainRenewals.get(task);
|
|
1356
|
+
if (attempted === void 0) {
|
|
1357
|
+
return false;
|
|
1358
|
+
}
|
|
1359
|
+
this.__uncertainRenewals.delete(task);
|
|
1360
|
+
const lost = new Error('outcome write lost (renewal outcome uncertain or lease taken over)');
|
|
1361
|
+
if (attempted === null || this.__abortInFlight) {
|
|
1362
|
+
this.___reportWriteFailure(task, lost, 'complete');
|
|
1363
|
+
return false;
|
|
1364
|
+
}
|
|
1365
|
+
if (await write({ ...task, sendingAt: attempted })) {
|
|
1366
|
+
return true;
|
|
1367
|
+
}
|
|
1368
|
+
this.___reportWriteFailure(task, lost, 'complete');
|
|
1369
|
+
return false;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1283
1372
|
/** @internal Serialize every policy checkpoint with this attempt's claim renewal. */
|
|
1284
1373
|
async ___sendWithRecipientPolicies(task) {
|
|
1285
1374
|
if (!task || task.isSent || task.isFailed || task.isCancelled || task.isSettled || this.__abortInFlight) return;
|
|
@@ -1294,9 +1383,12 @@ class MailTime {
|
|
|
1294
1383
|
}
|
|
1295
1384
|
Object.assign(task, fields);
|
|
1296
1385
|
const lease = new RecipientPolicyLease({
|
|
1297
|
-
task, queue: this.queue, interval: this.renewClaim, maxRenewals: this.maxRenewals,
|
|
1386
|
+
task, queue: this.queue, interval: this.renewClaim, maxRenewals: this.maxRenewals, sendingTimeout: this.sendingTimeout,
|
|
1298
1387
|
shouldAbort: () => this.__abortInFlight,
|
|
1299
|
-
report: (error) =>
|
|
1388
|
+
report: (error, phase) => {
|
|
1389
|
+
if (phase === 'complete' || phase === 'checkpoint') this.___reportWriteFailure(task, error, phase);
|
|
1390
|
+
else if (!this.__abortInFlight) logError('[recipientPolicies] claim renewal write failed; renewals stop, the outcome write still runs', error);
|
|
1391
|
+
},
|
|
1300
1392
|
});
|
|
1301
1393
|
try {
|
|
1302
1394
|
if (!lease.active) return;
|
|
@@ -1352,7 +1444,15 @@ class MailTime {
|
|
|
1352
1444
|
context.recipients = context.recipients.filter((r) => eligible.has(r.address));
|
|
1353
1445
|
await this.___attemptPolicyTransport(task, lease, compiled, context);
|
|
1354
1446
|
} catch (error) {
|
|
1355
|
-
if (lease.active)
|
|
1447
|
+
if (!lease.active) return;
|
|
1448
|
+
if (isAddressError(error)) {
|
|
1449
|
+
// Unparseable addresses fail identically on every attempt, so settle now instead of
|
|
1450
|
+
// spending the retry budget. The message names the field, never the address.
|
|
1451
|
+
if (!this.__abortInFlight) logError(`[recipientPolicies] task ${task.uuid} failed without retry:`, error.message);
|
|
1452
|
+
await this.___completePolicyTask(task, lease, error, void 0, true);
|
|
1453
|
+
} else {
|
|
1454
|
+
await this.___retryPolicyTask(task, lease, error, void 0, true, true);
|
|
1455
|
+
}
|
|
1356
1456
|
} finally {
|
|
1357
1457
|
await lease.stop();
|
|
1358
1458
|
}
|
|
@@ -1361,7 +1461,7 @@ class MailTime {
|
|
|
1361
1461
|
/** @internal Persist SMTP acceptance before awaiting rejection classification or observation. */
|
|
1362
1462
|
async ___attemptPolicyTransport(task, lease, compiled, context) {
|
|
1363
1463
|
const transport = this.transports[context.transport.index];
|
|
1364
|
-
const outgoing = { ...compiled, envelope: context.envelope };
|
|
1464
|
+
const outgoing = { ...rewritePolicyHeaders(compiled), envelope: context.envelope };
|
|
1365
1465
|
const { error, info } = await new Promise((resolve) => {
|
|
1366
1466
|
let called = false;
|
|
1367
1467
|
const done = (error, info) => {
|
|
@@ -1569,6 +1669,7 @@ class MailTime {
|
|
|
1569
1669
|
task.tries = tries;
|
|
1570
1670
|
task.isSending = true;
|
|
1571
1671
|
task.sendingAt = sendingAt;
|
|
1672
|
+
this.__uncertainRenewals.delete(task);
|
|
1572
1673
|
|
|
1573
1674
|
let transportIndex = task.transport;
|
|
1574
1675
|
if (!this.___isHealthyTransport(transportIndex)) {
|
|
@@ -1622,20 +1723,14 @@ class MailTime {
|
|
|
1622
1723
|
if (isFullyDelivered) {
|
|
1623
1724
|
this.__debug(`email successfully sent, attempts: #${task.tries}, transport #${transportIndex} to: `, compiledOpts.to);
|
|
1624
1725
|
|
|
1625
|
-
const
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
let completed = false;
|
|
1634
|
-
if (!this.keepHistory) {
|
|
1635
|
-
completed = await this.queue.remove(task, leaseRemove);
|
|
1636
|
-
} else {
|
|
1637
|
-
completed = await this.queue.update(task, leaseUpdate);
|
|
1638
|
-
}
|
|
1726
|
+
const completed = await this.___outcomeWrite(task, (t) => (!this.keepHistory
|
|
1727
|
+
? this.queue.remove(t, __leaseRemoveOpts(t))
|
|
1728
|
+
: this.queue.update(t, __withLease(t, {
|
|
1729
|
+
isSent: true,
|
|
1730
|
+
isSending: false,
|
|
1731
|
+
sendingAt: 0,
|
|
1732
|
+
mailOptions: task.mailOptions,
|
|
1733
|
+
}))));
|
|
1639
1734
|
|
|
1640
1735
|
if (!completed) {
|
|
1641
1736
|
this.__debug('[private send] lease lost before success completion, skipping onSent', task.uuid);
|
|
@@ -1653,21 +1748,15 @@ class MailTime {
|
|
|
1653
1748
|
if (task.tries >= this.maxTries) {
|
|
1654
1749
|
this.___finalizeRejected(task, info);
|
|
1655
1750
|
|
|
1656
|
-
const
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
let finalized = false;
|
|
1666
|
-
if (!this.keepHistory) {
|
|
1667
|
-
finalized = await this.queue.remove(task, leaseRemove);
|
|
1668
|
-
} else {
|
|
1669
|
-
finalized = await this.queue.update(task, leaseUpdate);
|
|
1670
|
-
}
|
|
1751
|
+
const finalized = await this.___outcomeWrite(task, (t) => (!this.keepHistory
|
|
1752
|
+
? this.queue.remove(t, __leaseRemoveOpts(t))
|
|
1753
|
+
: this.queue.update(t, __withLease(t, {
|
|
1754
|
+
isSent: false,
|
|
1755
|
+
isFailed: true,
|
|
1756
|
+
isSending: false,
|
|
1757
|
+
sendingAt: 0,
|
|
1758
|
+
mailOptions: task.mailOptions,
|
|
1759
|
+
}))));
|
|
1671
1760
|
|
|
1672
1761
|
if (!finalized) {
|
|
1673
1762
|
this.__debug('[private send] lease lost before partial-failure completion, skipping onError', task.uuid);
|
|
@@ -1692,19 +1781,19 @@ class MailTime {
|
|
|
1692
1781
|
}
|
|
1693
1782
|
|
|
1694
1783
|
const nextSendAt = Date.now() + this.retryDelay;
|
|
1695
|
-
const released = await this.queue.update(
|
|
1784
|
+
const released = await this.___outcomeWrite(task, (t) => this.queue.update(t, __withLease(t, {
|
|
1696
1785
|
isSending: false,
|
|
1697
1786
|
sendingAt: 0,
|
|
1698
1787
|
sendAt: nextSendAt,
|
|
1699
1788
|
mailOptions: task.mailOptions,
|
|
1700
|
-
}));
|
|
1789
|
+
})));
|
|
1701
1790
|
if (!released) {
|
|
1702
1791
|
this.__debug('[private send] lease lost before partial retry release', task.uuid);
|
|
1703
1792
|
return;
|
|
1704
1793
|
}
|
|
1705
1794
|
this.__debug(`[private send] Partial delivery, next attempt at ${new Date(nextSendAt)}: #${task.tries}/${this.maxTries} for remaining recipients`);
|
|
1706
1795
|
} catch (completionError) {
|
|
1707
|
-
|
|
1796
|
+
this.___reportWriteFailure(task, completionError);
|
|
1708
1797
|
} finally {
|
|
1709
1798
|
resolve();
|
|
1710
1799
|
}
|
|
@@ -1720,7 +1809,11 @@ class MailTime {
|
|
|
1720
1809
|
return;
|
|
1721
1810
|
}
|
|
1722
1811
|
logError('Exception during runtime:', e);
|
|
1723
|
-
|
|
1812
|
+
try {
|
|
1813
|
+
await this.___handleError(task, e, {});
|
|
1814
|
+
} catch (completionError) {
|
|
1815
|
+
this.___reportWriteFailure(task, completionError);
|
|
1816
|
+
}
|
|
1724
1817
|
}
|
|
1725
1818
|
}
|
|
1726
1819
|
|
|
@@ -1736,6 +1829,9 @@ class MailTime {
|
|
|
1736
1829
|
if (this.__isDestroyed || this.__isPaused) {
|
|
1737
1830
|
return;
|
|
1738
1831
|
}
|
|
1832
|
+
// Under `strategy: 'backup'` a quarantined primary is never selected for a send, so the
|
|
1833
|
+
// send path alone would never re-probe it. Sweep here so recovery does not depend on row flow.
|
|
1834
|
+
for (const index of this.__unhealthyTransports) this.___maybeReprobe(index);
|
|
1739
1835
|
const limit = this.mode === 'one' ? 1 : Infinity;
|
|
1740
1836
|
this.__schedulerScans++;
|
|
1741
1837
|
try {
|
|
@@ -1782,51 +1878,183 @@ class MailTime {
|
|
|
1782
1878
|
* @internal
|
|
1783
1879
|
* @memberOf MailTime
|
|
1784
1880
|
* @name ___verifyTransports
|
|
1785
|
-
* @description Probe each transport's `verify()` once at startup.
|
|
1881
|
+
* @description Probe each transport's `verify()` once at startup, in parallel. A transport whose probe fails is quarantined (skipped during rotation) and reported through `onError(error, null, { transportIndex, phase: 'verify' })`. A probe that neither settles nor fails within `verifyTimeout` stops delaying `ready()` and leaves the transport usable; a verdict that arrives later still applies. Throws if every transport is quarantined.
|
|
1786
1882
|
* @returns {Promise<void>}
|
|
1787
1883
|
*/
|
|
1788
1884
|
async ___verifyTransports() {
|
|
1789
1885
|
this.__debug('[private verifyTransports]');
|
|
1790
|
-
|
|
1886
|
+
await Promise.all(this.transports.map(async (transport, index) => {
|
|
1791
1887
|
if (!transport || typeof transport.verify !== 'function') {
|
|
1792
|
-
return
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
const outcome = await this.___verifyOne(transport, index);
|
|
1891
|
+
if (outcome === 'timeout' && !this.__isDestroyed) {
|
|
1892
|
+
console.warn(`[WARN] [mail-time] [verifyTransports] transport #${index} verify() gave no verdict within ${this.verifyTimeout}ms; keeping it usable, a late verdict still applies`);
|
|
1893
|
+
}
|
|
1894
|
+
}));
|
|
1895
|
+
|
|
1896
|
+
if (this.__unhealthyTransports.size === this.transports.length) {
|
|
1897
|
+
throw new Error(`[mail-time] [MailTime#ready] all ${this.transports.length} transport(s) failed verification — nothing can be delivered`);
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
/**
|
|
1902
|
+
* @internal
|
|
1903
|
+
* @memberOf MailTime
|
|
1904
|
+
* @name ___verifyOne
|
|
1905
|
+
* @description Start one `transport.verify(callback)` probe bounded by `verifyTimeout`. A callback is always passed. The first of the callback, a returned thenable, a synchronous throw, or a synchronous `true`/`false` decides the verdict. `false` is what Nodemailer's `Mailer#verify` returns, without calling back, when the underlying transport has no `verify` method. Any other synchronous non-thenable return value (for example the timer handle an expression-bodied arrow returns) is ignored, so a callback that reports an error later still counts, and a verify that never calls back waits for the timeout. The verdict is applied through `___applyVerdict` even when it arrives after the timeout, guarded by a per-transport generation. Resolves `'ok'`, `'fail'`, or `'timeout'` (the wait ended without a verdict or the instance was destroyed).
|
|
1906
|
+
* @param {MailTimeTransport} transport
|
|
1907
|
+
* @param {number} index
|
|
1908
|
+
* @returns {Promise<'ok' | 'fail' | 'timeout'>}
|
|
1909
|
+
*/
|
|
1910
|
+
___verifyOne(transport, index) {
|
|
1911
|
+
const gen = (this.__verifyGen[index] || 0) + 1;
|
|
1912
|
+
this.__verifyGen[index] = gen;
|
|
1913
|
+
return new Promise((resolve) => {
|
|
1914
|
+
let decided = false;
|
|
1915
|
+
let waiting = true;
|
|
1916
|
+
const stopWaiting = (outcome) => {
|
|
1917
|
+
if (!waiting) {
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
waiting = false;
|
|
1921
|
+
clearTimeout(timer);
|
|
1922
|
+
this.__probeCancels.delete(cancel);
|
|
1923
|
+
resolve(outcome);
|
|
1924
|
+
};
|
|
1925
|
+
const cancel = () => stopWaiting('timeout');
|
|
1926
|
+
const timer = setTimeout(cancel, this.verifyTimeout);
|
|
1927
|
+
if (typeof timer.unref === 'function') {
|
|
1928
|
+
timer.unref();
|
|
1793
1929
|
}
|
|
1930
|
+
this.__probeCancels.add(cancel);
|
|
1931
|
+
|
|
1932
|
+
const decide = (error) => {
|
|
1933
|
+
if (decided) {
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
decided = true;
|
|
1937
|
+
this.___applyVerdict(index, gen, error);
|
|
1938
|
+
stopWaiting(error ? 'fail' : 'ok');
|
|
1939
|
+
};
|
|
1940
|
+
|
|
1794
1941
|
try {
|
|
1795
|
-
|
|
1796
|
-
|
|
1942
|
+
const returned = transport.verify((error) => decide(error || void 0));
|
|
1943
|
+
if (returned && typeof returned.then === 'function') {
|
|
1944
|
+
returned.then(() => decide(), (error) => decide(error || new Error('[mail-time] [verifyTransports] verify() rejected')));
|
|
1945
|
+
} else if (returned === true || returned === false) {
|
|
1946
|
+
decide();
|
|
1947
|
+
}
|
|
1797
1948
|
} catch (error) {
|
|
1798
|
-
|
|
1949
|
+
decide(error || new Error('[mail-time] [verifyTransports] verify() threw'));
|
|
1799
1950
|
}
|
|
1800
|
-
})
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1801
1953
|
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1954
|
+
/**
|
|
1955
|
+
* @internal
|
|
1956
|
+
* @memberOf MailTime
|
|
1957
|
+
* @name ___applyVerdict
|
|
1958
|
+
* @description Apply a probe verdict to the health state. Ignored after `destroy()` and when a newer probe of the same transport has started (generation guard). Success removes the transport from quarantine. Failure quarantines it and fires `onError` once per quarantine episode; a failure for an already quarantined transport releases its re-probe slot and doubles the backoff.
|
|
1959
|
+
* @param {number} index
|
|
1960
|
+
* @param {number} gen
|
|
1961
|
+
* @param {unknown} [error]
|
|
1962
|
+
* @returns {void}
|
|
1963
|
+
*/
|
|
1964
|
+
___applyVerdict(index, gen, error) {
|
|
1965
|
+
if (this.__isDestroyed || this.__verifyGen[index] !== gen) {
|
|
1966
|
+
return;
|
|
1967
|
+
}
|
|
1968
|
+
const quarantined = this.__unhealthyTransports.has(index);
|
|
1969
|
+
if (!error) {
|
|
1970
|
+
if (!quarantined) {
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
this.__unhealthyTransports.delete(index);
|
|
1974
|
+
this.__reprobe.delete(index);
|
|
1975
|
+
if (this.strategy === 'backup' && index < this.transport) {
|
|
1976
|
+
this.transport = index;
|
|
1805
1977
|
}
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
callHook('onError', this.onError, r.error, null, { transportIndex: r.index, phase: 'verify' });
|
|
1978
|
+
console.log(`[mail-time] [verifyTransports] transport #${index} recovered; back in rotation`);
|
|
1979
|
+
return;
|
|
1809
1980
|
}
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1981
|
+
if (quarantined) {
|
|
1982
|
+
const st = this.__reprobe.get(index);
|
|
1983
|
+
// A slot already released by the timeout path keeps its backoff; do not double it twice.
|
|
1984
|
+
if (st && st.inFlight) this.___backoffReprobe(index, st, 'failed');
|
|
1985
|
+
return;
|
|
1986
|
+
}
|
|
1987
|
+
this.__unhealthyTransports.add(index);
|
|
1988
|
+
this.__reprobe.set(index, { count: 0, nextProbeAt: Date.now() + REPROBE_BASE_MS, inFlight: false });
|
|
1989
|
+
if (this.transport === index && this.__unhealthyTransports.size < this.transports.length) {
|
|
1990
|
+
this.transport = this.___nextHealthyTransport(index);
|
|
1813
1991
|
}
|
|
1992
|
+
logError(`[mail-time] [verifyTransports] transport #${index} failed verification`, error);
|
|
1993
|
+
callHook('onError', this.onError, error, null, { transportIndex: index, phase: 'verify' });
|
|
1994
|
+
}
|
|
1814
1995
|
|
|
1815
|
-
|
|
1816
|
-
|
|
1996
|
+
/**
|
|
1997
|
+
* @internal
|
|
1998
|
+
* @memberOf MailTime
|
|
1999
|
+
* @name ___maybeReprobe
|
|
2000
|
+
* @description Start one background `verify()` probe for a quarantined transport when its backoff has elapsed and no probe is in flight. Never awaited by the send path. Backoff starts at 60 s, doubles per failed or timed-out probe and caps at 15 min. A probe that reaches `verifyTimeout` without a verdict releases the slot (one `verify()` may then overlap with the hung one); its late verdict still applies unless a newer probe has started. Failures only reach `__debug`. No probes for `type: 'client'`, `verifyTransports: false`, or after `destroy()`.
|
|
2001
|
+
* @param {number} index
|
|
2002
|
+
* @returns {void}
|
|
2003
|
+
*/
|
|
2004
|
+
___maybeReprobe(index) {
|
|
2005
|
+
if (this.__isDestroyed || this.type !== 'server' || !this.verifyTransports) {
|
|
2006
|
+
return;
|
|
2007
|
+
}
|
|
2008
|
+
const st = this.__reprobe.get(index);
|
|
2009
|
+
const transport = this.transports[index];
|
|
2010
|
+
if (!st || st.inFlight || st.nextProbeAt > Date.now() || !transport || typeof transport.verify !== 'function') {
|
|
2011
|
+
return;
|
|
1817
2012
|
}
|
|
2013
|
+
st.inFlight = true;
|
|
2014
|
+
this.__debug(`[private reprobe] transport #${index}, attempt after ${st.count} failure(s)`);
|
|
2015
|
+
// A verdict releases `inFlight` (see ___applyVerdict). A probe that reaches `verifyTimeout`
|
|
2016
|
+
// without one counts as a failed attempt so a verify() that never settles cannot pin the
|
|
2017
|
+
// transport in quarantine; its late verdict still applies unless a newer probe has started.
|
|
2018
|
+
this.___verifyOne(transport, index).then((outcome) => {
|
|
2019
|
+
// A live verdict always clears `inFlight` in ___applyVerdict. If it is still set, the probe
|
|
2020
|
+
// timed out or its verdict was ignored (superseded generation); either way free the slot.
|
|
2021
|
+
if (st.inFlight && this.__reprobe.get(index) === st) {
|
|
2022
|
+
this.___backoffReprobe(index, st, outcome === 'timeout' ? 'timed out' : 'superseded');
|
|
2023
|
+
}
|
|
2024
|
+
});
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
/**
|
|
2028
|
+
* @internal
|
|
2029
|
+
* @memberOf MailTime
|
|
2030
|
+
* @name ___backoffReprobe
|
|
2031
|
+
* @description Release a re-probe slot after a failed or timed-out probe and double its backoff (cap 15 min).
|
|
2032
|
+
* @param {number} index
|
|
2033
|
+
* @param {{ count: number, nextProbeAt: number, inFlight: boolean }} st
|
|
2034
|
+
* @param {string} why
|
|
2035
|
+
* @returns {void}
|
|
2036
|
+
*/
|
|
2037
|
+
___backoffReprobe(index, st, why) {
|
|
2038
|
+
st.inFlight = false;
|
|
2039
|
+
st.count++;
|
|
2040
|
+
st.nextProbeAt = Date.now() + Math.min(REPROBE_BASE_MS * Math.pow(2, st.count), REPROBE_MAX_MS);
|
|
2041
|
+
this.__debug(`[private reprobe] transport #${index} ${why}, next probe at ${new Date(st.nextProbeAt)}`);
|
|
1818
2042
|
}
|
|
1819
2043
|
|
|
1820
2044
|
/**
|
|
1821
2045
|
* @internal
|
|
1822
2046
|
* @memberOf MailTime
|
|
1823
2047
|
* @name ___isHealthyTransport
|
|
1824
|
-
* @description Return true when the transport at `index`
|
|
2048
|
+
* @description Return true when the transport at `index` is not quarantined. A quarantined transport whose backoff has elapsed triggers a background re-probe; this call still returns false.
|
|
1825
2049
|
* @param {number} index
|
|
1826
2050
|
* @returns {boolean}
|
|
1827
2051
|
*/
|
|
1828
2052
|
___isHealthyTransport(index) {
|
|
1829
|
-
|
|
2053
|
+
if (!this.__unhealthyTransports.has(index)) {
|
|
2054
|
+
return true;
|
|
2055
|
+
}
|
|
2056
|
+
this.___maybeReprobe(index);
|
|
2057
|
+
return false;
|
|
1830
2058
|
}
|
|
1831
2059
|
|
|
1832
2060
|
/**
|