mailery 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/index.cjs +341 -131
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +341 -131
- package/dist/index.js.map +1 -1
- package/dist/{null-OzIqP7A8.d.cts → null-CWw3Gpbl.d.cts} +65 -43
- package/dist/{null-OzIqP7A8.d.ts → null-CWw3Gpbl.d.ts} +65 -43
- package/dist/testing.cjs +346 -136
- package/dist/testing.cjs.map +1 -1
- package/dist/testing.d.cts +3 -3
- package/dist/testing.d.ts +3 -3
- package/dist/testing.js +346 -136
- package/dist/testing.js.map +1 -1
- package/package.json +29 -4
package/dist/index.d.cts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { R as RunnerContext, N as NormalizedEvent, C as ContactAdapter, a as Contact, A as AdapterFilter, M as MailProvider, S as SendArgs, b as SendResult, c as Mailer, T as TemplateDoc, d as SuppressionScope, F as FlowStep, e as SegmentFilter, P as Predicate } from './null-
|
|
2
|
-
export { f as AuditLogDoc, B as BroadcastDoc, g as BroadcastStatus, h as CircuitBreakerThresholds, i as Collections, j as ContactTagDoc, E as EventDoc, k as FlowDoc, l as FlowGoal, m as FlowRunDoc, n as FlowRunStatus, o as FlowVersionDoc, H as HealthDoc, p as HealthStatus, L as LeadDoc, q as MailerConfig, r as NullProvider, O as OutboxDoc, s as RedisOptions, t as SegmentDefinition, u as SendDoc, v as SendStatus, w as SubscriptionDoc, x as SubscriptionStatus, y as SuppressionDoc, z as SuppressionReason, D as TemplateKind, G as TemplateVersionDoc, W as WebhookEventDoc, I as ensureIndexes, J as getCollections } from './null-
|
|
1
|
+
import { R as RunnerContext, N as NormalizedEvent, C as ContactAdapter, a as Contact, A as AdapterFilter, M as MailProvider, S as SendArgs, b as SendResult, c as Mailer, T as TemplateDoc, d as SuppressionScope, F as FlowStep, e as SegmentFilter, P as Predicate } from './null-CWw3Gpbl.cjs';
|
|
2
|
+
export { f as AuditLogDoc, B as BroadcastDoc, g as BroadcastStatus, h as CircuitBreakerThresholds, i as Collections, j as ContactTagDoc, E as EventDoc, k as FlowDoc, l as FlowGoal, m as FlowRunDoc, n as FlowRunStatus, o as FlowVersionDoc, H as HealthDoc, p as HealthStatus, L as LeadDoc, q as MailerConfig, r as NullProvider, O as OutboxDoc, s as RedisOptions, t as SegmentDefinition, u as SendDoc, v as SendStatus, w as SubscriptionDoc, x as SubscriptionStatus, y as SuppressionDoc, z as SuppressionReason, D as TemplateKind, G as TemplateVersionDoc, W as WebhookEventDoc, I as ensureIndexes, J as getCollections } from './null-CWw3Gpbl.cjs';
|
|
3
3
|
import { ObjectId, Db, Filter } from 'mongodb';
|
|
4
4
|
import { Request, Router } from 'express';
|
|
5
5
|
import Handlebars from 'handlebars';
|
|
6
|
-
import 'ioredis';
|
|
7
6
|
import 'zod';
|
|
7
|
+
import 'ioredis';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* The mailer:tick handler. Orchestrates all the periodic work that doesn't
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { R as RunnerContext, N as NormalizedEvent, C as ContactAdapter, a as Contact, A as AdapterFilter, M as MailProvider, S as SendArgs, b as SendResult, c as Mailer, T as TemplateDoc, d as SuppressionScope, F as FlowStep, e as SegmentFilter, P as Predicate } from './null-
|
|
2
|
-
export { f as AuditLogDoc, B as BroadcastDoc, g as BroadcastStatus, h as CircuitBreakerThresholds, i as Collections, j as ContactTagDoc, E as EventDoc, k as FlowDoc, l as FlowGoal, m as FlowRunDoc, n as FlowRunStatus, o as FlowVersionDoc, H as HealthDoc, p as HealthStatus, L as LeadDoc, q as MailerConfig, r as NullProvider, O as OutboxDoc, s as RedisOptions, t as SegmentDefinition, u as SendDoc, v as SendStatus, w as SubscriptionDoc, x as SubscriptionStatus, y as SuppressionDoc, z as SuppressionReason, D as TemplateKind, G as TemplateVersionDoc, W as WebhookEventDoc, I as ensureIndexes, J as getCollections } from './null-
|
|
1
|
+
import { R as RunnerContext, N as NormalizedEvent, C as ContactAdapter, a as Contact, A as AdapterFilter, M as MailProvider, S as SendArgs, b as SendResult, c as Mailer, T as TemplateDoc, d as SuppressionScope, F as FlowStep, e as SegmentFilter, P as Predicate } from './null-CWw3Gpbl.js';
|
|
2
|
+
export { f as AuditLogDoc, B as BroadcastDoc, g as BroadcastStatus, h as CircuitBreakerThresholds, i as Collections, j as ContactTagDoc, E as EventDoc, k as FlowDoc, l as FlowGoal, m as FlowRunDoc, n as FlowRunStatus, o as FlowVersionDoc, H as HealthDoc, p as HealthStatus, L as LeadDoc, q as MailerConfig, r as NullProvider, O as OutboxDoc, s as RedisOptions, t as SegmentDefinition, u as SendDoc, v as SendStatus, w as SubscriptionDoc, x as SubscriptionStatus, y as SuppressionDoc, z as SuppressionReason, D as TemplateKind, G as TemplateVersionDoc, W as WebhookEventDoc, I as ensureIndexes, J as getCollections } from './null-CWw3Gpbl.js';
|
|
3
3
|
import { ObjectId, Db, Filter } from 'mongodb';
|
|
4
4
|
import { Request, Router } from 'express';
|
|
5
5
|
import Handlebars from 'handlebars';
|
|
6
|
-
import 'ioredis';
|
|
7
6
|
import 'zod';
|
|
7
|
+
import 'ioredis';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* The mailer:tick handler. Orchestrates all the periodic work that doesn't
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,6 @@ import { ObjectId } from 'mongodb';
|
|
|
2
2
|
import crypto2 from 'crypto';
|
|
3
3
|
import sgMail from '@sendgrid/mail';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { Queue, Worker } from 'bullmq';
|
|
6
5
|
import IORedis from 'ioredis';
|
|
7
6
|
import Handlebars from 'handlebars';
|
|
8
7
|
import { convert } from 'html-to-text';
|
|
@@ -568,7 +567,8 @@ async function ensureIndexes(db, prefix = "mailer_") {
|
|
|
568
567
|
{ key: { flowRunId: 1 }, sparse: true },
|
|
569
568
|
{ key: { broadcastId: 1 }, sparse: true },
|
|
570
569
|
{ key: { providerMessageId: 1 }, sparse: true },
|
|
571
|
-
{ key: { status: 1, queuedAt: 1 } }
|
|
570
|
+
{ key: { status: 1, queuedAt: 1 } },
|
|
571
|
+
{ key: { status: 1, updatedAt: 1 } }
|
|
572
572
|
]),
|
|
573
573
|
c.suppressions.createIndexes([
|
|
574
574
|
{ key: { email: 1, scope: 1 }, unique: true, partialFilterExpression: { email: { $type: "string" } } },
|
|
@@ -715,6 +715,104 @@ function verifyDoiToken(token, secret, now = /* @__PURE__ */ new Date()) {
|
|
|
715
715
|
if (body.x < now.getTime()) return null;
|
|
716
716
|
return { externalId: body.i, expiresAt: new Date(body.x) };
|
|
717
717
|
}
|
|
718
|
+
var QUEUE_NAMES = {
|
|
719
|
+
tick: "mailer-tick",
|
|
720
|
+
advance: "mailer-advance",
|
|
721
|
+
send: "mailer-send",
|
|
722
|
+
webhook: "mailer-webhook"
|
|
723
|
+
};
|
|
724
|
+
var BullDriver = class _BullDriver {
|
|
725
|
+
queues;
|
|
726
|
+
redis;
|
|
727
|
+
bullQueues;
|
|
728
|
+
workers = null;
|
|
729
|
+
bull;
|
|
730
|
+
static async create(redisConfig) {
|
|
731
|
+
let bull;
|
|
732
|
+
try {
|
|
733
|
+
bull = await import('bullmq');
|
|
734
|
+
} catch {
|
|
735
|
+
throw new Error(
|
|
736
|
+
"mailery: queue driver 'bull' requires the 'bullmq' peer dependency. Run `npm install bullmq ioredis`."
|
|
737
|
+
);
|
|
738
|
+
}
|
|
739
|
+
const redis = isRedisLike(redisConfig) ? redisConfig : connect(redisConfig);
|
|
740
|
+
return new _BullDriver(bull, redis);
|
|
741
|
+
}
|
|
742
|
+
constructor(bull, redis) {
|
|
743
|
+
this.bull = bull;
|
|
744
|
+
this.redis = redis;
|
|
745
|
+
const opts = { connection: redis };
|
|
746
|
+
this.bullQueues = {
|
|
747
|
+
tick: new bull.Queue(QUEUE_NAMES.tick, opts),
|
|
748
|
+
advance: new bull.Queue(QUEUE_NAMES.advance, opts),
|
|
749
|
+
send: new bull.Queue(QUEUE_NAMES.send, opts),
|
|
750
|
+
webhook: new bull.Queue(QUEUE_NAMES.webhook, opts)
|
|
751
|
+
};
|
|
752
|
+
this.queues = {
|
|
753
|
+
tick: adaptBullQueue(this.bullQueues.tick),
|
|
754
|
+
advance: adaptBullQueue(this.bullQueues.advance),
|
|
755
|
+
send: adaptBullQueue(this.bullQueues.send),
|
|
756
|
+
webhook: adaptBullQueue(this.bullQueues.webhook)
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
async scheduleRepeatingTick(intervalSeconds) {
|
|
760
|
+
await this.bullQueues.tick.upsertJobScheduler(
|
|
761
|
+
"mailer-tick-repeat",
|
|
762
|
+
{ every: intervalSeconds * 1e3 },
|
|
763
|
+
{ name: "tick", data: {} }
|
|
764
|
+
);
|
|
765
|
+
}
|
|
766
|
+
async startWorkers(opts) {
|
|
767
|
+
if (this.workers) return;
|
|
768
|
+
const base = { connection: this.redis };
|
|
769
|
+
const { Worker } = this.bull;
|
|
770
|
+
const tick = new Worker(
|
|
771
|
+
QUEUE_NAMES.tick,
|
|
772
|
+
async (job) => opts.handlers.tick(job.data),
|
|
773
|
+
{ ...base, concurrency: 1 }
|
|
774
|
+
);
|
|
775
|
+
const advance = new Worker(
|
|
776
|
+
QUEUE_NAMES.advance,
|
|
777
|
+
async (job) => opts.handlers.advance(job.data),
|
|
778
|
+
{ ...base, concurrency: 10 }
|
|
779
|
+
);
|
|
780
|
+
const send = new Worker(
|
|
781
|
+
QUEUE_NAMES.send,
|
|
782
|
+
async (job) => opts.handlers.send(job.data),
|
|
783
|
+
{
|
|
784
|
+
...base,
|
|
785
|
+
concurrency: opts.concurrency.send,
|
|
786
|
+
limiter: opts.sendRateLimit ? { max: opts.sendRateLimit.max, duration: opts.sendRateLimit.durationMs } : void 0
|
|
787
|
+
}
|
|
788
|
+
);
|
|
789
|
+
const webhook = new Worker(
|
|
790
|
+
QUEUE_NAMES.webhook,
|
|
791
|
+
async (job) => opts.handlers.webhook(job.data),
|
|
792
|
+
{ ...base, concurrency: 4 }
|
|
793
|
+
);
|
|
794
|
+
this.workers = { tick, advance, send, webhook };
|
|
795
|
+
}
|
|
796
|
+
async stopWorkers() {
|
|
797
|
+
if (!this.workers) return;
|
|
798
|
+
await Promise.all([
|
|
799
|
+
this.workers.tick.close(),
|
|
800
|
+
this.workers.advance.close(),
|
|
801
|
+
this.workers.send.close(),
|
|
802
|
+
this.workers.webhook.close()
|
|
803
|
+
]);
|
|
804
|
+
this.workers = null;
|
|
805
|
+
}
|
|
806
|
+
async close() {
|
|
807
|
+
await this.stopWorkers();
|
|
808
|
+
await Promise.all([
|
|
809
|
+
this.bullQueues.tick.close(),
|
|
810
|
+
this.bullQueues.advance.close(),
|
|
811
|
+
this.bullQueues.send.close(),
|
|
812
|
+
this.bullQueues.webhook.close()
|
|
813
|
+
]);
|
|
814
|
+
}
|
|
815
|
+
};
|
|
718
816
|
function adaptBullQueue(q) {
|
|
719
817
|
return {
|
|
720
818
|
add: (name, data, opts) => q.add(name, data, opts),
|
|
@@ -722,39 +820,16 @@ function adaptBullQueue(q) {
|
|
|
722
820
|
close: () => q.close()
|
|
723
821
|
};
|
|
724
822
|
}
|
|
725
|
-
function
|
|
726
|
-
return
|
|
727
|
-
add: async () => void 0,
|
|
728
|
-
getWaitingCount: async () => 0,
|
|
729
|
-
close: async () => void 0
|
|
730
|
-
};
|
|
731
|
-
}
|
|
732
|
-
function noopQueues() {
|
|
733
|
-
return {
|
|
734
|
-
tick: noopQueueAPI(),
|
|
735
|
-
advance: noopQueueAPI(),
|
|
736
|
-
send: noopQueueAPI(),
|
|
737
|
-
webhook: noopQueueAPI()
|
|
738
|
-
};
|
|
739
|
-
}
|
|
740
|
-
function namespacedQueueNames(prefix) {
|
|
741
|
-
return {
|
|
742
|
-
tick: "mailer-tick",
|
|
743
|
-
advance: "mailer-advance",
|
|
744
|
-
send: "mailer-send",
|
|
745
|
-
webhook: "mailer-webhook"
|
|
746
|
-
};
|
|
823
|
+
function isRedisLike(x) {
|
|
824
|
+
return !!x && typeof x === "object" && typeof x.get === "function" && typeof x.set === "function";
|
|
747
825
|
}
|
|
748
|
-
function
|
|
749
|
-
if (isRedisLike(opts)) return opts;
|
|
826
|
+
function connect(opts) {
|
|
750
827
|
const config = {
|
|
751
828
|
maxRetriesPerRequest: null,
|
|
752
829
|
// BullMQ requirement
|
|
753
830
|
enableReadyCheck: false
|
|
754
831
|
};
|
|
755
|
-
if (opts.url)
|
|
756
|
-
return new IORedis(opts.url, config);
|
|
757
|
-
}
|
|
832
|
+
if (opts.url) return new IORedis(opts.url, config);
|
|
758
833
|
return new IORedis({
|
|
759
834
|
...config,
|
|
760
835
|
host: opts.host ?? "127.0.0.1",
|
|
@@ -765,69 +840,198 @@ function makeRedis(opts) {
|
|
|
765
840
|
tls: opts.tls ? {} : void 0
|
|
766
841
|
});
|
|
767
842
|
}
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
843
|
+
|
|
844
|
+
// src/server/queues/agenda.ts
|
|
845
|
+
var QUEUE_NAMES2 = {
|
|
846
|
+
tick: "mailer-tick",
|
|
847
|
+
advance: "mailer-advance",
|
|
848
|
+
send: "mailer-send",
|
|
849
|
+
webhook: "mailer-webhook"
|
|
850
|
+
};
|
|
851
|
+
var AgendaDriver = class _AgendaDriver {
|
|
852
|
+
queues;
|
|
853
|
+
agenda;
|
|
854
|
+
agendaMod;
|
|
855
|
+
sendLimiter = null;
|
|
856
|
+
started = false;
|
|
857
|
+
static async create(opts) {
|
|
858
|
+
let agendaMod;
|
|
859
|
+
let backendMod;
|
|
860
|
+
try {
|
|
861
|
+
agendaMod = await import('agenda');
|
|
862
|
+
backendMod = await import('@agendajs/mongo-backend');
|
|
863
|
+
} catch {
|
|
864
|
+
throw new Error(
|
|
865
|
+
"mailery: queue driver 'agenda' requires the 'agenda' and '@agendajs/mongo-backend' peer dependencies. Run `npm install agenda @agendajs/mongo-backend bottleneck`."
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
const backend = new backendMod.MongoBackend({
|
|
869
|
+
mongo: opts.db,
|
|
870
|
+
collection: opts.collectionName ?? "_mailerJobs"
|
|
871
|
+
});
|
|
872
|
+
const agenda = new agendaMod.Agenda({
|
|
873
|
+
backend,
|
|
874
|
+
processEvery: `${opts.processEverySeconds ?? 5} seconds`,
|
|
875
|
+
defaultLockLifetime: (opts.lockLifetimeSeconds ?? 10 * 60) * 1e3,
|
|
876
|
+
maxConcurrency: 50,
|
|
877
|
+
defaultConcurrency: 5
|
|
878
|
+
});
|
|
879
|
+
return new _AgendaDriver(agenda, agendaMod, opts.db);
|
|
880
|
+
}
|
|
881
|
+
db;
|
|
882
|
+
constructor(agenda, agendaMod, db) {
|
|
883
|
+
this.agenda = agenda;
|
|
884
|
+
this.agendaMod = agendaMod;
|
|
885
|
+
this.db = db;
|
|
886
|
+
this.queues = {
|
|
887
|
+
tick: this.makeQueueAPI(QUEUE_NAMES2.tick),
|
|
888
|
+
advance: this.makeQueueAPI(QUEUE_NAMES2.advance),
|
|
889
|
+
send: this.makeQueueAPI(QUEUE_NAMES2.send),
|
|
890
|
+
webhook: this.makeQueueAPI(QUEUE_NAMES2.webhook)
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
makeQueueAPI(name) {
|
|
894
|
+
return {
|
|
895
|
+
add: async (_jobName, data, opts) => {
|
|
896
|
+
const payload = { ...data };
|
|
897
|
+
if (opts?.jobId) {
|
|
898
|
+
payload.__jobId = opts.jobId;
|
|
899
|
+
if (await this.findPending(name, opts.jobId)) return;
|
|
900
|
+
}
|
|
901
|
+
const job = this.agenda.create(name, payload);
|
|
902
|
+
if (opts?.delay) job.schedule(new Date(Date.now() + opts.delay));
|
|
903
|
+
await job.save();
|
|
904
|
+
},
|
|
905
|
+
getWaitingCount: async () => {
|
|
906
|
+
return this.jobsCollection().countDocuments({
|
|
907
|
+
name,
|
|
908
|
+
$or: [{ lockedAt: null }, { lockedAt: { $exists: false } }],
|
|
909
|
+
nextRunAt: { $lte: /* @__PURE__ */ new Date() }
|
|
910
|
+
});
|
|
911
|
+
},
|
|
912
|
+
close: async () => {
|
|
913
|
+
}
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
/** Direct access to the Mongo collection Agenda persists jobs into. */
|
|
917
|
+
jobsCollection() {
|
|
918
|
+
return this.db.collection(this.collectionName());
|
|
919
|
+
}
|
|
920
|
+
collectionName() {
|
|
921
|
+
return "_mailerJobs";
|
|
922
|
+
}
|
|
923
|
+
async findPending(name, jobId) {
|
|
924
|
+
return this.jobsCollection().findOne({
|
|
925
|
+
name,
|
|
926
|
+
"data.__jobId": jobId,
|
|
927
|
+
$or: [{ lastFinishedAt: null }, { lastFinishedAt: { $exists: false } }]
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
async scheduleRepeatingTick(intervalSeconds) {
|
|
931
|
+
if (!this.started) {
|
|
932
|
+
this.agenda.define(QUEUE_NAMES2.tick, async () => {
|
|
933
|
+
}, { concurrency: 1 });
|
|
934
|
+
await this.agenda.start();
|
|
935
|
+
this.started = true;
|
|
936
|
+
}
|
|
937
|
+
await this.agenda.every(`${intervalSeconds} seconds`, QUEUE_NAMES2.tick);
|
|
938
|
+
}
|
|
939
|
+
async startWorkers(opts) {
|
|
940
|
+
const exp = this.agendaMod.backoffStrategies.exponential;
|
|
941
|
+
if (opts.sendRateLimit) {
|
|
942
|
+
try {
|
|
943
|
+
const Bottleneck = (await import('bottleneck')).default;
|
|
944
|
+
this.sendLimiter = new Bottleneck({
|
|
945
|
+
minTime: Math.ceil(opts.sendRateLimit.durationMs / opts.sendRateLimit.max),
|
|
946
|
+
maxConcurrent: opts.concurrency.send
|
|
947
|
+
});
|
|
948
|
+
} catch {
|
|
949
|
+
throw new Error(
|
|
950
|
+
"mailery: queue driver 'agenda' with sendRateLimit requires the 'bottleneck' peer dependency. Run `npm install bottleneck`."
|
|
951
|
+
);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
const retryBackoff = exp({ delay: 6e4, maxRetries: Math.max(0, opts.retryAttempts - 1), factor: 2 });
|
|
955
|
+
this.agenda.define(QUEUE_NAMES2.tick, async (job) => {
|
|
956
|
+
await opts.handlers.tick(job.attrs.data);
|
|
957
|
+
}, { concurrency: 1 });
|
|
958
|
+
this.agenda.define(QUEUE_NAMES2.advance, async (job) => {
|
|
959
|
+
await opts.handlers.advance(job.attrs.data);
|
|
960
|
+
}, { concurrency: 10, backoff: retryBackoff });
|
|
961
|
+
this.agenda.define(QUEUE_NAMES2.send, async (job) => {
|
|
962
|
+
const data = job.attrs.data;
|
|
963
|
+
if (this.sendLimiter) {
|
|
964
|
+
await this.sendLimiter.schedule(() => opts.handlers.send(data));
|
|
965
|
+
} else {
|
|
966
|
+
await opts.handlers.send(data);
|
|
967
|
+
}
|
|
968
|
+
}, { concurrency: opts.concurrency.send, backoff: retryBackoff });
|
|
969
|
+
this.agenda.define(QUEUE_NAMES2.webhook, async (job) => {
|
|
970
|
+
await opts.handlers.webhook(job.attrs.data);
|
|
971
|
+
}, { concurrency: 4, backoff: retryBackoff });
|
|
972
|
+
if (!this.started) {
|
|
973
|
+
await this.agenda.start();
|
|
974
|
+
this.started = true;
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
async stopWorkers() {
|
|
978
|
+
if (!this.started) return;
|
|
979
|
+
await this.agenda.stop();
|
|
980
|
+
this.started = false;
|
|
981
|
+
if (this.sendLimiter) {
|
|
982
|
+
await this.sendLimiter.stop({ dropWaitingJobs: true }).catch(() => {
|
|
983
|
+
});
|
|
984
|
+
this.sendLimiter = null;
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
async close() {
|
|
988
|
+
await this.stopWorkers();
|
|
989
|
+
}
|
|
990
|
+
};
|
|
991
|
+
|
|
992
|
+
// src/server/queues/noop.ts
|
|
993
|
+
function noopQueueAPI() {
|
|
780
994
|
return {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
send: adaptBullQueue(bullQueues.send),
|
|
785
|
-
webhook: adaptBullQueue(bullQueues.webhook)
|
|
786
|
-
},
|
|
787
|
-
bullQueues
|
|
995
|
+
add: async () => void 0,
|
|
996
|
+
getWaitingCount: async () => 0,
|
|
997
|
+
close: async () => void 0
|
|
788
998
|
};
|
|
789
999
|
}
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}
|
|
797
|
-
function createWorkers(input) {
|
|
798
|
-
const names = namespacedQueueNames();
|
|
799
|
-
const base = { connection: input.redis };
|
|
800
|
-
const tick = new Worker(names.tick, async (job) => input.handlers.tick(job.data), {
|
|
801
|
-
...base,
|
|
802
|
-
concurrency: 1
|
|
803
|
-
// single tick driver per worker process
|
|
804
|
-
});
|
|
805
|
-
const advance = new Worker(
|
|
806
|
-
names.advance,
|
|
807
|
-
async (job) => input.handlers.advance(job.data),
|
|
808
|
-
{ ...base, concurrency: 10 }
|
|
809
|
-
);
|
|
810
|
-
const sendOpts = {
|
|
811
|
-
...base,
|
|
812
|
-
concurrency: input.concurrency.send,
|
|
813
|
-
limiter: input.sendRateLimit ? { max: input.sendRateLimit.max, duration: input.sendRateLimit.durationMs } : void 0
|
|
1000
|
+
var NoopDriver = class {
|
|
1001
|
+
queues = {
|
|
1002
|
+
tick: noopQueueAPI(),
|
|
1003
|
+
advance: noopQueueAPI(),
|
|
1004
|
+
send: noopQueueAPI(),
|
|
1005
|
+
webhook: noopQueueAPI()
|
|
814
1006
|
};
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
async function
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
1007
|
+
async scheduleRepeatingTick(_intervalSeconds) {
|
|
1008
|
+
}
|
|
1009
|
+
async startWorkers(_opts) {
|
|
1010
|
+
}
|
|
1011
|
+
async stopWorkers() {
|
|
1012
|
+
}
|
|
1013
|
+
async close() {
|
|
1014
|
+
}
|
|
1015
|
+
};
|
|
1016
|
+
|
|
1017
|
+
// src/server/queues/index.ts
|
|
1018
|
+
async function createQueueDriver(config, fallbackDb) {
|
|
1019
|
+
switch (config.driver) {
|
|
1020
|
+
case "bull":
|
|
1021
|
+
return BullDriver.create(config.redis);
|
|
1022
|
+
case "agenda":
|
|
1023
|
+
return AgendaDriver.create({
|
|
1024
|
+
db: config.db ?? fallbackDb,
|
|
1025
|
+
processEverySeconds: config.processEverySeconds,
|
|
1026
|
+
lockLifetimeSeconds: config.lockLifetimeSeconds,
|
|
1027
|
+
collectionName: config.collectionName
|
|
1028
|
+
});
|
|
1029
|
+
case "noop":
|
|
1030
|
+
return new NoopDriver();
|
|
1031
|
+
default: {
|
|
1032
|
+
throw new Error(`mailery: unknown queue driver`);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
831
1035
|
}
|
|
832
1036
|
|
|
833
1037
|
// src/server/runner/triggers.ts
|
|
@@ -1320,7 +1524,8 @@ async function dispatchSend(sendId, ctx) {
|
|
|
1320
1524
|
status: "sending",
|
|
1321
1525
|
fromName: rendered.fromName,
|
|
1322
1526
|
fromEmail: rendered.fromEmail,
|
|
1323
|
-
subject: rendered.subject
|
|
1527
|
+
subject: rendered.subject,
|
|
1528
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1324
1529
|
}
|
|
1325
1530
|
}
|
|
1326
1531
|
);
|
|
@@ -1394,6 +1599,7 @@ function buildRenderContext(contact, run, vars, ctx) {
|
|
|
1394
1599
|
};
|
|
1395
1600
|
}
|
|
1396
1601
|
function newSendDoc(input) {
|
|
1602
|
+
const now = /* @__PURE__ */ new Date();
|
|
1397
1603
|
return {
|
|
1398
1604
|
_id: input._id,
|
|
1399
1605
|
dedupeKey: input.dedupeKey,
|
|
@@ -1424,7 +1630,8 @@ function newSendDoc(input) {
|
|
|
1424
1630
|
clickedLinks: [],
|
|
1425
1631
|
unsubscribedAt: null,
|
|
1426
1632
|
complainedAt: null,
|
|
1427
|
-
queuedAt:
|
|
1633
|
+
queuedAt: now,
|
|
1634
|
+
updatedAt: now,
|
|
1428
1635
|
sentAt: null,
|
|
1429
1636
|
deliveredAt: null
|
|
1430
1637
|
};
|
|
@@ -1906,6 +2113,7 @@ async function buildSendDoc(broadcast, template, contact, ctx, scheduledMs, resp
|
|
|
1906
2113
|
unsubscribedAt: null,
|
|
1907
2114
|
complainedAt: null,
|
|
1908
2115
|
queuedAt: /* @__PURE__ */ new Date(),
|
|
2116
|
+
updatedAt: /* @__PURE__ */ new Date(),
|
|
1909
2117
|
sentAt: null,
|
|
1910
2118
|
deliveredAt: null
|
|
1911
2119
|
};
|
|
@@ -1978,6 +2186,7 @@ async function promoteSoftBounces(ctx) {
|
|
|
1978
2186
|
}
|
|
1979
2187
|
|
|
1980
2188
|
// src/server/runner/tick.ts
|
|
2189
|
+
var STRANDED_SEND_THRESHOLD_MS = 5 * 60 * 1e3;
|
|
1981
2190
|
async function runTick(ctx) {
|
|
1982
2191
|
await processNewlyFiredEventTriggers(ctx).catch((err) => {
|
|
1983
2192
|
console.error("mailery: triggers scan failed", err);
|
|
@@ -1985,6 +2194,9 @@ async function runTick(ctx) {
|
|
|
1985
2194
|
await sweepStrandedFlowRuns(ctx).catch((err) => {
|
|
1986
2195
|
console.error("mailery: sweep failed", err);
|
|
1987
2196
|
});
|
|
2197
|
+
await sweepStrandedSends(ctx).catch((err) => {
|
|
2198
|
+
console.error("mailery: stranded-send sweep failed", err);
|
|
2199
|
+
});
|
|
1988
2200
|
await drainOutbox(ctx).catch((err) => {
|
|
1989
2201
|
console.error("mailery: outbox drain failed", err);
|
|
1990
2202
|
});
|
|
@@ -1998,6 +2210,24 @@ async function runTick(ctx) {
|
|
|
1998
2210
|
console.error("mailery: soft-bounce promotion failed", err);
|
|
1999
2211
|
});
|
|
2000
2212
|
}
|
|
2213
|
+
async function sweepStrandedSends(ctx) {
|
|
2214
|
+
const cutoff = new Date(Date.now() - STRANDED_SEND_THRESHOLD_MS);
|
|
2215
|
+
const cursor = ctx.collections.sends.find(
|
|
2216
|
+
{ status: "sending", updatedAt: { $lt: cutoff } },
|
|
2217
|
+
{ projection: { _id: 1 } }
|
|
2218
|
+
).limit(500);
|
|
2219
|
+
for await (const row of cursor) {
|
|
2220
|
+
const reset = await ctx.collections.sends.updateOne(
|
|
2221
|
+
{ _id: row._id, status: "sending", updatedAt: { $lt: cutoff } },
|
|
2222
|
+
{ $set: { status: "queued", updatedAt: /* @__PURE__ */ new Date() } }
|
|
2223
|
+
);
|
|
2224
|
+
if (reset.modifiedCount === 0) continue;
|
|
2225
|
+
await ctx.queues.send.add("send", { sendId: String(row._id) }, {
|
|
2226
|
+
attempts: ctx.config.sendRetryAttempts,
|
|
2227
|
+
backoff: { type: "exponential", delay: 6e4 }
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2001
2231
|
async function drainOutbox(ctx) {
|
|
2002
2232
|
const batch = await ctx.collections.outbox.find({ status: "pending" }).sort({ enqueuedAt: 1 }).limit(200).toArray();
|
|
2003
2233
|
for (const row of batch) {
|
|
@@ -2168,12 +2398,11 @@ var Mailer = class _Mailer {
|
|
|
2168
2398
|
collections;
|
|
2169
2399
|
adapter;
|
|
2170
2400
|
providers;
|
|
2171
|
-
redis;
|
|
2172
2401
|
queues;
|
|
2173
2402
|
config;
|
|
2174
2403
|
events;
|
|
2175
|
-
|
|
2176
|
-
|
|
2404
|
+
queueDriver;
|
|
2405
|
+
workersStarted = false;
|
|
2177
2406
|
runnerContext;
|
|
2178
2407
|
constructor(args) {
|
|
2179
2408
|
this.config = args.config;
|
|
@@ -2181,9 +2410,8 @@ var Mailer = class _Mailer {
|
|
|
2181
2410
|
this.collections = args.collections;
|
|
2182
2411
|
this.adapter = args.adapter;
|
|
2183
2412
|
this.providers = args.providers;
|
|
2184
|
-
this.
|
|
2185
|
-
this.queues = args.queues;
|
|
2186
|
-
this.bullQueues = args.bullQueues;
|
|
2413
|
+
this.queueDriver = args.queueDriver;
|
|
2414
|
+
this.queues = args.queueDriver.queues;
|
|
2187
2415
|
this.events = args.events;
|
|
2188
2416
|
this.runnerContext = {
|
|
2189
2417
|
db: this.db,
|
|
@@ -2248,10 +2476,12 @@ var Mailer = class _Mailer {
|
|
|
2248
2476
|
throw new Error("Mailer.fromEnv: no provider configured (set MAILER_SENDGRID_API_KEY, ...)");
|
|
2249
2477
|
}
|
|
2250
2478
|
const defaultProvider = env.MAILER_DEFAULT_PROVIDER ?? Object.keys(providers)[0];
|
|
2479
|
+
const driverEnv = env.MAILER_QUEUE_DRIVER ?? "bull";
|
|
2480
|
+
const queue = driverEnv === "agenda" ? { driver: "agenda" } : driverEnv === "noop" ? { driver: "noop" } : { driver: "bull", redis: { url: required("MAILER_REDIS_URL") } };
|
|
2251
2481
|
return _Mailer.init({
|
|
2252
2482
|
db,
|
|
2253
2483
|
adapter,
|
|
2254
|
-
|
|
2484
|
+
queue,
|
|
2255
2485
|
providers,
|
|
2256
2486
|
defaultProvider,
|
|
2257
2487
|
publicUrl: required("MAILER_PUBLIC_URL"),
|
|
@@ -2267,19 +2497,9 @@ var Mailer = class _Mailer {
|
|
|
2267
2497
|
}
|
|
2268
2498
|
const collections = getCollections(config.db, config.collectionPrefix);
|
|
2269
2499
|
await ensureIndexes(config.db, config.collectionPrefix);
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
if (config.redis === null) {
|
|
2274
|
-
queues = noopQueues();
|
|
2275
|
-
} else {
|
|
2276
|
-
redis = makeRedis(config.redis);
|
|
2277
|
-
const created = createQueues(redis);
|
|
2278
|
-
queues = created.queues;
|
|
2279
|
-
bullQueues = created.bullQueues;
|
|
2280
|
-
if (!config.workerless) {
|
|
2281
|
-
await scheduleTick(bullQueues, config.tickIntervalSeconds);
|
|
2282
|
-
}
|
|
2500
|
+
const queueDriver = await createQueueDriver(config.queue, config.db);
|
|
2501
|
+
if (!config.workerless && config.queue.driver !== "noop") {
|
|
2502
|
+
await queueDriver.scheduleRepeatingTick(config.tickIntervalSeconds);
|
|
2283
2503
|
}
|
|
2284
2504
|
return new _Mailer({
|
|
2285
2505
|
config,
|
|
@@ -2287,9 +2507,7 @@ var Mailer = class _Mailer {
|
|
|
2287
2507
|
collections,
|
|
2288
2508
|
adapter: config.adapter,
|
|
2289
2509
|
providers: config.providers,
|
|
2290
|
-
|
|
2291
|
-
queues,
|
|
2292
|
-
bullQueues,
|
|
2510
|
+
queueDriver,
|
|
2293
2511
|
events: new EventRegistry()
|
|
2294
2512
|
});
|
|
2295
2513
|
}
|
|
@@ -2597,6 +2815,7 @@ var Mailer = class _Mailer {
|
|
|
2597
2815
|
unsubscribedAt: null,
|
|
2598
2816
|
complainedAt: null,
|
|
2599
2817
|
queuedAt: /* @__PURE__ */ new Date(),
|
|
2818
|
+
updatedAt: /* @__PURE__ */ new Date(),
|
|
2600
2819
|
sentAt: null,
|
|
2601
2820
|
deliveredAt: null
|
|
2602
2821
|
});
|
|
@@ -2628,14 +2847,16 @@ var Mailer = class _Mailer {
|
|
|
2628
2847
|
// Workers
|
|
2629
2848
|
// -------------------------------------------------------------------------
|
|
2630
2849
|
async startWorkers() {
|
|
2631
|
-
if (this.
|
|
2632
|
-
if (
|
|
2850
|
+
if (this.workersStarted) return;
|
|
2851
|
+
if (this.config.queue.driver === "noop") {
|
|
2852
|
+
throw new Error("startWorkers requires a non-noop queue driver");
|
|
2853
|
+
}
|
|
2633
2854
|
const provider = this.providers[this.config.defaultProvider];
|
|
2634
2855
|
const sendRate = provider?.sendRatePerSecond ?? this.config.sendRatePerSecond;
|
|
2635
|
-
this.
|
|
2636
|
-
redis: this.redis,
|
|
2856
|
+
await this.queueDriver.startWorkers({
|
|
2637
2857
|
concurrency: { send: this.config.sendConcurrency },
|
|
2638
2858
|
sendRateLimit: { max: sendRate, durationMs: 1e3 },
|
|
2859
|
+
retryAttempts: this.config.sendRetryAttempts,
|
|
2639
2860
|
handlers: {
|
|
2640
2861
|
tick: async () => {
|
|
2641
2862
|
await runTick(this.runnerContext);
|
|
@@ -2653,6 +2874,7 @@ var Mailer = class _Mailer {
|
|
|
2653
2874
|
}
|
|
2654
2875
|
}
|
|
2655
2876
|
});
|
|
2877
|
+
this.workersStarted = true;
|
|
2656
2878
|
}
|
|
2657
2879
|
/** Process unprocessed webhook events in mailer_webhook_events. */
|
|
2658
2880
|
async processWebhookBacklog() {
|
|
@@ -2679,20 +2901,8 @@ var Mailer = class _Mailer {
|
|
|
2679
2901
|
}
|
|
2680
2902
|
}
|
|
2681
2903
|
async stop() {
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
this.workers = null;
|
|
2685
|
-
}
|
|
2686
|
-
if (this.bullQueues) {
|
|
2687
|
-
await closeBullQueues(this.bullQueues);
|
|
2688
|
-
this.bullQueues = null;
|
|
2689
|
-
} else {
|
|
2690
|
-
await closeQueues(this.queues);
|
|
2691
|
-
}
|
|
2692
|
-
if (this.redis) {
|
|
2693
|
-
await this.redis.quit().catch(() => {
|
|
2694
|
-
});
|
|
2695
|
-
}
|
|
2904
|
+
await this.queueDriver.close();
|
|
2905
|
+
this.workersStarted = false;
|
|
2696
2906
|
}
|
|
2697
2907
|
/** Used internally by the admin router and tests; not part of the public API. */
|
|
2698
2908
|
getRunnerContext() {
|