mailery 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/index.cjs +341 -131
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +341 -131
- package/dist/index.js.map +1 -1
- package/dist/{null-OzIqP7A8.d.cts → null-CWw3Gpbl.d.cts} +65 -43
- package/dist/{null-OzIqP7A8.d.ts → null-CWw3Gpbl.d.ts} +65 -43
- package/dist/testing.cjs +346 -136
- package/dist/testing.cjs.map +1 -1
- package/dist/testing.d.cts +3 -3
- package/dist/testing.d.ts +3 -3
- package/dist/testing.js +346 -136
- package/dist/testing.js.map +1 -1
- package/package.json +29 -4
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ yarn add mailery
|
|
|
14
14
|
npm install mailery
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
-
Peer
|
|
17
|
+
Peer dependencies (host-provided): `express ^4 || ^5` plus one queue driver — either `bullmq` + `ioredis` (default) or `agenda` + `@agendajs/mongo-backend` + `bottleneck`. Runtime deps brought in by mailery: `mongodb`, `mjml`, `handlebars`, `@sendgrid/mail`, `zod`.
|
|
18
18
|
|
|
19
19
|
## Quickstart
|
|
20
20
|
|
|
@@ -44,7 +44,7 @@ const adapter = new MongoContactAdapter({
|
|
|
44
44
|
const mailer = await Mailer.init({
|
|
45
45
|
db,
|
|
46
46
|
adapter,
|
|
47
|
-
redis: { url: process.env.REDIS_URL! },
|
|
47
|
+
queue: { driver: 'bull', redis: { url: process.env.REDIS_URL! } },
|
|
48
48
|
providers: {
|
|
49
49
|
sendgrid: new SendGridProvider({
|
|
50
50
|
apiKey: process.env.SENDGRID_API_KEY!,
|
|
@@ -73,7 +73,7 @@ Run a separate worker process for queue jobs:
|
|
|
73
73
|
|
|
74
74
|
```ts
|
|
75
75
|
const mailer = await Mailer.init({ /* same config */ })
|
|
76
|
-
await mailer.startWorkers() //
|
|
76
|
+
await mailer.startWorkers() // queue consumers (Bull or Agenda, per your queue.driver)
|
|
77
77
|
```
|
|
78
78
|
|
|
79
79
|
See [`examples/express-mongo/`](./examples/express-mongo) for a complete working example.
|
|
@@ -85,7 +85,7 @@ A self-hosted library you `npm install` into your Express + MongoDB app. Fire ev
|
|
|
85
85
|
- **Embedded, not external.** No third-party sync, no per-contact pricing.
|
|
86
86
|
- **Both transactional and marketing in one engine.** Right defaults per kind (suppression scope, sender identity, circuit-breaker behavior).
|
|
87
87
|
- **Provider-agnostic.** SendGrid + NullProvider ship; Postmark / SES / Resend pluggable.
|
|
88
|
-
- **BullMQ + Redis
|
|
88
|
+
- **Pluggable queue** — BullMQ + Redis (default) or Agenda + Mongo (no Redis required).
|
|
89
89
|
- **MJML** templates with click + open tracking, plain-text auto-derivation, scope-aware suppression at send time.
|
|
90
90
|
- **React admin SPA** (Vite-bundled, served as static assets — no build step in your app).
|
|
91
91
|
|
package/dist/index.cjs
CHANGED
|
@@ -4,7 +4,6 @@ var mongodb = require('mongodb');
|
|
|
4
4
|
var crypto2 = require('crypto');
|
|
5
5
|
var sgMail = require('@sendgrid/mail');
|
|
6
6
|
var zod = require('zod');
|
|
7
|
-
var bullmq = require('bullmq');
|
|
8
7
|
var IORedis = require('ioredis');
|
|
9
8
|
var Handlebars = require('handlebars');
|
|
10
9
|
var htmlToText = require('html-to-text');
|
|
@@ -582,7 +581,8 @@ async function ensureIndexes(db, prefix = "mailer_") {
|
|
|
582
581
|
{ key: { flowRunId: 1 }, sparse: true },
|
|
583
582
|
{ key: { broadcastId: 1 }, sparse: true },
|
|
584
583
|
{ key: { providerMessageId: 1 }, sparse: true },
|
|
585
|
-
{ key: { status: 1, queuedAt: 1 } }
|
|
584
|
+
{ key: { status: 1, queuedAt: 1 } },
|
|
585
|
+
{ key: { status: 1, updatedAt: 1 } }
|
|
586
586
|
]),
|
|
587
587
|
c.suppressions.createIndexes([
|
|
588
588
|
{ key: { email: 1, scope: 1 }, unique: true, partialFilterExpression: { email: { $type: "string" } } },
|
|
@@ -729,6 +729,104 @@ function verifyDoiToken(token, secret, now = /* @__PURE__ */ new Date()) {
|
|
|
729
729
|
if (body.x < now.getTime()) return null;
|
|
730
730
|
return { externalId: body.i, expiresAt: new Date(body.x) };
|
|
731
731
|
}
|
|
732
|
+
var QUEUE_NAMES = {
|
|
733
|
+
tick: "mailer-tick",
|
|
734
|
+
advance: "mailer-advance",
|
|
735
|
+
send: "mailer-send",
|
|
736
|
+
webhook: "mailer-webhook"
|
|
737
|
+
};
|
|
738
|
+
var BullDriver = class _BullDriver {
|
|
739
|
+
queues;
|
|
740
|
+
redis;
|
|
741
|
+
bullQueues;
|
|
742
|
+
workers = null;
|
|
743
|
+
bull;
|
|
744
|
+
static async create(redisConfig) {
|
|
745
|
+
let bull;
|
|
746
|
+
try {
|
|
747
|
+
bull = await import('bullmq');
|
|
748
|
+
} catch {
|
|
749
|
+
throw new Error(
|
|
750
|
+
"mailery: queue driver 'bull' requires the 'bullmq' peer dependency. Run `npm install bullmq ioredis`."
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
const redis = isRedisLike(redisConfig) ? redisConfig : connect(redisConfig);
|
|
754
|
+
return new _BullDriver(bull, redis);
|
|
755
|
+
}
|
|
756
|
+
constructor(bull, redis) {
|
|
757
|
+
this.bull = bull;
|
|
758
|
+
this.redis = redis;
|
|
759
|
+
const opts = { connection: redis };
|
|
760
|
+
this.bullQueues = {
|
|
761
|
+
tick: new bull.Queue(QUEUE_NAMES.tick, opts),
|
|
762
|
+
advance: new bull.Queue(QUEUE_NAMES.advance, opts),
|
|
763
|
+
send: new bull.Queue(QUEUE_NAMES.send, opts),
|
|
764
|
+
webhook: new bull.Queue(QUEUE_NAMES.webhook, opts)
|
|
765
|
+
};
|
|
766
|
+
this.queues = {
|
|
767
|
+
tick: adaptBullQueue(this.bullQueues.tick),
|
|
768
|
+
advance: adaptBullQueue(this.bullQueues.advance),
|
|
769
|
+
send: adaptBullQueue(this.bullQueues.send),
|
|
770
|
+
webhook: adaptBullQueue(this.bullQueues.webhook)
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
async scheduleRepeatingTick(intervalSeconds) {
|
|
774
|
+
await this.bullQueues.tick.upsertJobScheduler(
|
|
775
|
+
"mailer-tick-repeat",
|
|
776
|
+
{ every: intervalSeconds * 1e3 },
|
|
777
|
+
{ name: "tick", data: {} }
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
async startWorkers(opts) {
|
|
781
|
+
if (this.workers) return;
|
|
782
|
+
const base = { connection: this.redis };
|
|
783
|
+
const { Worker } = this.bull;
|
|
784
|
+
const tick = new Worker(
|
|
785
|
+
QUEUE_NAMES.tick,
|
|
786
|
+
async (job) => opts.handlers.tick(job.data),
|
|
787
|
+
{ ...base, concurrency: 1 }
|
|
788
|
+
);
|
|
789
|
+
const advance = new Worker(
|
|
790
|
+
QUEUE_NAMES.advance,
|
|
791
|
+
async (job) => opts.handlers.advance(job.data),
|
|
792
|
+
{ ...base, concurrency: 10 }
|
|
793
|
+
);
|
|
794
|
+
const send = new Worker(
|
|
795
|
+
QUEUE_NAMES.send,
|
|
796
|
+
async (job) => opts.handlers.send(job.data),
|
|
797
|
+
{
|
|
798
|
+
...base,
|
|
799
|
+
concurrency: opts.concurrency.send,
|
|
800
|
+
limiter: opts.sendRateLimit ? { max: opts.sendRateLimit.max, duration: opts.sendRateLimit.durationMs } : void 0
|
|
801
|
+
}
|
|
802
|
+
);
|
|
803
|
+
const webhook = new Worker(
|
|
804
|
+
QUEUE_NAMES.webhook,
|
|
805
|
+
async (job) => opts.handlers.webhook(job.data),
|
|
806
|
+
{ ...base, concurrency: 4 }
|
|
807
|
+
);
|
|
808
|
+
this.workers = { tick, advance, send, webhook };
|
|
809
|
+
}
|
|
810
|
+
async stopWorkers() {
|
|
811
|
+
if (!this.workers) return;
|
|
812
|
+
await Promise.all([
|
|
813
|
+
this.workers.tick.close(),
|
|
814
|
+
this.workers.advance.close(),
|
|
815
|
+
this.workers.send.close(),
|
|
816
|
+
this.workers.webhook.close()
|
|
817
|
+
]);
|
|
818
|
+
this.workers = null;
|
|
819
|
+
}
|
|
820
|
+
async close() {
|
|
821
|
+
await this.stopWorkers();
|
|
822
|
+
await Promise.all([
|
|
823
|
+
this.bullQueues.tick.close(),
|
|
824
|
+
this.bullQueues.advance.close(),
|
|
825
|
+
this.bullQueues.send.close(),
|
|
826
|
+
this.bullQueues.webhook.close()
|
|
827
|
+
]);
|
|
828
|
+
}
|
|
829
|
+
};
|
|
732
830
|
function adaptBullQueue(q) {
|
|
733
831
|
return {
|
|
734
832
|
add: (name, data, opts) => q.add(name, data, opts),
|
|
@@ -736,39 +834,16 @@ function adaptBullQueue(q) {
|
|
|
736
834
|
close: () => q.close()
|
|
737
835
|
};
|
|
738
836
|
}
|
|
739
|
-
function
|
|
740
|
-
return
|
|
741
|
-
add: async () => void 0,
|
|
742
|
-
getWaitingCount: async () => 0,
|
|
743
|
-
close: async () => void 0
|
|
744
|
-
};
|
|
745
|
-
}
|
|
746
|
-
function noopQueues() {
|
|
747
|
-
return {
|
|
748
|
-
tick: noopQueueAPI(),
|
|
749
|
-
advance: noopQueueAPI(),
|
|
750
|
-
send: noopQueueAPI(),
|
|
751
|
-
webhook: noopQueueAPI()
|
|
752
|
-
};
|
|
753
|
-
}
|
|
754
|
-
function namespacedQueueNames(prefix) {
|
|
755
|
-
return {
|
|
756
|
-
tick: "mailer-tick",
|
|
757
|
-
advance: "mailer-advance",
|
|
758
|
-
send: "mailer-send",
|
|
759
|
-
webhook: "mailer-webhook"
|
|
760
|
-
};
|
|
837
|
+
function isRedisLike(x) {
|
|
838
|
+
return !!x && typeof x === "object" && typeof x.get === "function" && typeof x.set === "function";
|
|
761
839
|
}
|
|
762
|
-
function
|
|
763
|
-
if (isRedisLike(opts)) return opts;
|
|
840
|
+
function connect(opts) {
|
|
764
841
|
const config = {
|
|
765
842
|
maxRetriesPerRequest: null,
|
|
766
843
|
// BullMQ requirement
|
|
767
844
|
enableReadyCheck: false
|
|
768
845
|
};
|
|
769
|
-
if (opts.url)
|
|
770
|
-
return new IORedis__default.default(opts.url, config);
|
|
771
|
-
}
|
|
846
|
+
if (opts.url) return new IORedis__default.default(opts.url, config);
|
|
772
847
|
return new IORedis__default.default({
|
|
773
848
|
...config,
|
|
774
849
|
host: opts.host ?? "127.0.0.1",
|
|
@@ -779,69 +854,198 @@ function makeRedis(opts) {
|
|
|
779
854
|
tls: opts.tls ? {} : void 0
|
|
780
855
|
});
|
|
781
856
|
}
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
857
|
+
|
|
858
|
+
// src/server/queues/agenda.ts
|
|
859
|
+
var QUEUE_NAMES2 = {
|
|
860
|
+
tick: "mailer-tick",
|
|
861
|
+
advance: "mailer-advance",
|
|
862
|
+
send: "mailer-send",
|
|
863
|
+
webhook: "mailer-webhook"
|
|
864
|
+
};
|
|
865
|
+
var AgendaDriver = class _AgendaDriver {
|
|
866
|
+
queues;
|
|
867
|
+
agenda;
|
|
868
|
+
agendaMod;
|
|
869
|
+
sendLimiter = null;
|
|
870
|
+
started = false;
|
|
871
|
+
static async create(opts) {
|
|
872
|
+
let agendaMod;
|
|
873
|
+
let backendMod;
|
|
874
|
+
try {
|
|
875
|
+
agendaMod = await import('agenda');
|
|
876
|
+
backendMod = await import('@agendajs/mongo-backend');
|
|
877
|
+
} catch {
|
|
878
|
+
throw new Error(
|
|
879
|
+
"mailery: queue driver 'agenda' requires the 'agenda' and '@agendajs/mongo-backend' peer dependencies. Run `npm install agenda @agendajs/mongo-backend bottleneck`."
|
|
880
|
+
);
|
|
881
|
+
}
|
|
882
|
+
const backend = new backendMod.MongoBackend({
|
|
883
|
+
mongo: opts.db,
|
|
884
|
+
collection: opts.collectionName ?? "_mailerJobs"
|
|
885
|
+
});
|
|
886
|
+
const agenda = new agendaMod.Agenda({
|
|
887
|
+
backend,
|
|
888
|
+
processEvery: `${opts.processEverySeconds ?? 5} seconds`,
|
|
889
|
+
defaultLockLifetime: (opts.lockLifetimeSeconds ?? 10 * 60) * 1e3,
|
|
890
|
+
maxConcurrency: 50,
|
|
891
|
+
defaultConcurrency: 5
|
|
892
|
+
});
|
|
893
|
+
return new _AgendaDriver(agenda, agendaMod, opts.db);
|
|
894
|
+
}
|
|
895
|
+
db;
|
|
896
|
+
constructor(agenda, agendaMod, db) {
|
|
897
|
+
this.agenda = agenda;
|
|
898
|
+
this.agendaMod = agendaMod;
|
|
899
|
+
this.db = db;
|
|
900
|
+
this.queues = {
|
|
901
|
+
tick: this.makeQueueAPI(QUEUE_NAMES2.tick),
|
|
902
|
+
advance: this.makeQueueAPI(QUEUE_NAMES2.advance),
|
|
903
|
+
send: this.makeQueueAPI(QUEUE_NAMES2.send),
|
|
904
|
+
webhook: this.makeQueueAPI(QUEUE_NAMES2.webhook)
|
|
905
|
+
};
|
|
906
|
+
}
|
|
907
|
+
makeQueueAPI(name) {
|
|
908
|
+
return {
|
|
909
|
+
add: async (_jobName, data, opts) => {
|
|
910
|
+
const payload = { ...data };
|
|
911
|
+
if (opts?.jobId) {
|
|
912
|
+
payload.__jobId = opts.jobId;
|
|
913
|
+
if (await this.findPending(name, opts.jobId)) return;
|
|
914
|
+
}
|
|
915
|
+
const job = this.agenda.create(name, payload);
|
|
916
|
+
if (opts?.delay) job.schedule(new Date(Date.now() + opts.delay));
|
|
917
|
+
await job.save();
|
|
918
|
+
},
|
|
919
|
+
getWaitingCount: async () => {
|
|
920
|
+
return this.jobsCollection().countDocuments({
|
|
921
|
+
name,
|
|
922
|
+
$or: [{ lockedAt: null }, { lockedAt: { $exists: false } }],
|
|
923
|
+
nextRunAt: { $lte: /* @__PURE__ */ new Date() }
|
|
924
|
+
});
|
|
925
|
+
},
|
|
926
|
+
close: async () => {
|
|
927
|
+
}
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
/** Direct access to the Mongo collection Agenda persists jobs into. */
|
|
931
|
+
jobsCollection() {
|
|
932
|
+
return this.db.collection(this.collectionName());
|
|
933
|
+
}
|
|
934
|
+
collectionName() {
|
|
935
|
+
return "_mailerJobs";
|
|
936
|
+
}
|
|
937
|
+
async findPending(name, jobId) {
|
|
938
|
+
return this.jobsCollection().findOne({
|
|
939
|
+
name,
|
|
940
|
+
"data.__jobId": jobId,
|
|
941
|
+
$or: [{ lastFinishedAt: null }, { lastFinishedAt: { $exists: false } }]
|
|
942
|
+
});
|
|
943
|
+
}
|
|
944
|
+
async scheduleRepeatingTick(intervalSeconds) {
|
|
945
|
+
if (!this.started) {
|
|
946
|
+
this.agenda.define(QUEUE_NAMES2.tick, async () => {
|
|
947
|
+
}, { concurrency: 1 });
|
|
948
|
+
await this.agenda.start();
|
|
949
|
+
this.started = true;
|
|
950
|
+
}
|
|
951
|
+
await this.agenda.every(`${intervalSeconds} seconds`, QUEUE_NAMES2.tick);
|
|
952
|
+
}
|
|
953
|
+
async startWorkers(opts) {
|
|
954
|
+
const exp = this.agendaMod.backoffStrategies.exponential;
|
|
955
|
+
if (opts.sendRateLimit) {
|
|
956
|
+
try {
|
|
957
|
+
const Bottleneck = (await import('bottleneck')).default;
|
|
958
|
+
this.sendLimiter = new Bottleneck({
|
|
959
|
+
minTime: Math.ceil(opts.sendRateLimit.durationMs / opts.sendRateLimit.max),
|
|
960
|
+
maxConcurrent: opts.concurrency.send
|
|
961
|
+
});
|
|
962
|
+
} catch {
|
|
963
|
+
throw new Error(
|
|
964
|
+
"mailery: queue driver 'agenda' with sendRateLimit requires the 'bottleneck' peer dependency. Run `npm install bottleneck`."
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
const retryBackoff = exp({ delay: 6e4, maxRetries: Math.max(0, opts.retryAttempts - 1), factor: 2 });
|
|
969
|
+
this.agenda.define(QUEUE_NAMES2.tick, async (job) => {
|
|
970
|
+
await opts.handlers.tick(job.attrs.data);
|
|
971
|
+
}, { concurrency: 1 });
|
|
972
|
+
this.agenda.define(QUEUE_NAMES2.advance, async (job) => {
|
|
973
|
+
await opts.handlers.advance(job.attrs.data);
|
|
974
|
+
}, { concurrency: 10, backoff: retryBackoff });
|
|
975
|
+
this.agenda.define(QUEUE_NAMES2.send, async (job) => {
|
|
976
|
+
const data = job.attrs.data;
|
|
977
|
+
if (this.sendLimiter) {
|
|
978
|
+
await this.sendLimiter.schedule(() => opts.handlers.send(data));
|
|
979
|
+
} else {
|
|
980
|
+
await opts.handlers.send(data);
|
|
981
|
+
}
|
|
982
|
+
}, { concurrency: opts.concurrency.send, backoff: retryBackoff });
|
|
983
|
+
this.agenda.define(QUEUE_NAMES2.webhook, async (job) => {
|
|
984
|
+
await opts.handlers.webhook(job.attrs.data);
|
|
985
|
+
}, { concurrency: 4, backoff: retryBackoff });
|
|
986
|
+
if (!this.started) {
|
|
987
|
+
await this.agenda.start();
|
|
988
|
+
this.started = true;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
async stopWorkers() {
|
|
992
|
+
if (!this.started) return;
|
|
993
|
+
await this.agenda.stop();
|
|
994
|
+
this.started = false;
|
|
995
|
+
if (this.sendLimiter) {
|
|
996
|
+
await this.sendLimiter.stop({ dropWaitingJobs: true }).catch(() => {
|
|
997
|
+
});
|
|
998
|
+
this.sendLimiter = null;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
async close() {
|
|
1002
|
+
await this.stopWorkers();
|
|
1003
|
+
}
|
|
1004
|
+
};
|
|
1005
|
+
|
|
1006
|
+
// src/server/queues/noop.ts
|
|
1007
|
+
function noopQueueAPI() {
|
|
794
1008
|
return {
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
send: adaptBullQueue(bullQueues.send),
|
|
799
|
-
webhook: adaptBullQueue(bullQueues.webhook)
|
|
800
|
-
},
|
|
801
|
-
bullQueues
|
|
1009
|
+
add: async () => void 0,
|
|
1010
|
+
getWaitingCount: async () => 0,
|
|
1011
|
+
close: async () => void 0
|
|
802
1012
|
};
|
|
803
1013
|
}
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
}
|
|
811
|
-
function createWorkers(input) {
|
|
812
|
-
const names = namespacedQueueNames();
|
|
813
|
-
const base = { connection: input.redis };
|
|
814
|
-
const tick = new bullmq.Worker(names.tick, async (job) => input.handlers.tick(job.data), {
|
|
815
|
-
...base,
|
|
816
|
-
concurrency: 1
|
|
817
|
-
// single tick driver per worker process
|
|
818
|
-
});
|
|
819
|
-
const advance = new bullmq.Worker(
|
|
820
|
-
names.advance,
|
|
821
|
-
async (job) => input.handlers.advance(job.data),
|
|
822
|
-
{ ...base, concurrency: 10 }
|
|
823
|
-
);
|
|
824
|
-
const sendOpts = {
|
|
825
|
-
...base,
|
|
826
|
-
concurrency: input.concurrency.send,
|
|
827
|
-
limiter: input.sendRateLimit ? { max: input.sendRateLimit.max, duration: input.sendRateLimit.durationMs } : void 0
|
|
1014
|
+
var NoopDriver = class {
|
|
1015
|
+
queues = {
|
|
1016
|
+
tick: noopQueueAPI(),
|
|
1017
|
+
advance: noopQueueAPI(),
|
|
1018
|
+
send: noopQueueAPI(),
|
|
1019
|
+
webhook: noopQueueAPI()
|
|
828
1020
|
};
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
async function
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1021
|
+
async scheduleRepeatingTick(_intervalSeconds) {
|
|
1022
|
+
}
|
|
1023
|
+
async startWorkers(_opts) {
|
|
1024
|
+
}
|
|
1025
|
+
async stopWorkers() {
|
|
1026
|
+
}
|
|
1027
|
+
async close() {
|
|
1028
|
+
}
|
|
1029
|
+
};
|
|
1030
|
+
|
|
1031
|
+
// src/server/queues/index.ts
|
|
1032
|
+
async function createQueueDriver(config, fallbackDb) {
|
|
1033
|
+
switch (config.driver) {
|
|
1034
|
+
case "bull":
|
|
1035
|
+
return BullDriver.create(config.redis);
|
|
1036
|
+
case "agenda":
|
|
1037
|
+
return AgendaDriver.create({
|
|
1038
|
+
db: config.db ?? fallbackDb,
|
|
1039
|
+
processEverySeconds: config.processEverySeconds,
|
|
1040
|
+
lockLifetimeSeconds: config.lockLifetimeSeconds,
|
|
1041
|
+
collectionName: config.collectionName
|
|
1042
|
+
});
|
|
1043
|
+
case "noop":
|
|
1044
|
+
return new NoopDriver();
|
|
1045
|
+
default: {
|
|
1046
|
+
throw new Error(`mailery: unknown queue driver`);
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
845
1049
|
}
|
|
846
1050
|
|
|
847
1051
|
// src/server/runner/triggers.ts
|
|
@@ -1334,7 +1538,8 @@ async function dispatchSend(sendId, ctx) {
|
|
|
1334
1538
|
status: "sending",
|
|
1335
1539
|
fromName: rendered.fromName,
|
|
1336
1540
|
fromEmail: rendered.fromEmail,
|
|
1337
|
-
subject: rendered.subject
|
|
1541
|
+
subject: rendered.subject,
|
|
1542
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1338
1543
|
}
|
|
1339
1544
|
}
|
|
1340
1545
|
);
|
|
@@ -1408,6 +1613,7 @@ function buildRenderContext(contact, run, vars, ctx) {
|
|
|
1408
1613
|
};
|
|
1409
1614
|
}
|
|
1410
1615
|
function newSendDoc(input) {
|
|
1616
|
+
const now = /* @__PURE__ */ new Date();
|
|
1411
1617
|
return {
|
|
1412
1618
|
_id: input._id,
|
|
1413
1619
|
dedupeKey: input.dedupeKey,
|
|
@@ -1438,7 +1644,8 @@ function newSendDoc(input) {
|
|
|
1438
1644
|
clickedLinks: [],
|
|
1439
1645
|
unsubscribedAt: null,
|
|
1440
1646
|
complainedAt: null,
|
|
1441
|
-
queuedAt:
|
|
1647
|
+
queuedAt: now,
|
|
1648
|
+
updatedAt: now,
|
|
1442
1649
|
sentAt: null,
|
|
1443
1650
|
deliveredAt: null
|
|
1444
1651
|
};
|
|
@@ -1920,6 +2127,7 @@ async function buildSendDoc(broadcast, template, contact, ctx, scheduledMs, resp
|
|
|
1920
2127
|
unsubscribedAt: null,
|
|
1921
2128
|
complainedAt: null,
|
|
1922
2129
|
queuedAt: /* @__PURE__ */ new Date(),
|
|
2130
|
+
updatedAt: /* @__PURE__ */ new Date(),
|
|
1923
2131
|
sentAt: null,
|
|
1924
2132
|
deliveredAt: null
|
|
1925
2133
|
};
|
|
@@ -1992,6 +2200,7 @@ async function promoteSoftBounces(ctx) {
|
|
|
1992
2200
|
}
|
|
1993
2201
|
|
|
1994
2202
|
// src/server/runner/tick.ts
|
|
2203
|
+
var STRANDED_SEND_THRESHOLD_MS = 5 * 60 * 1e3;
|
|
1995
2204
|
async function runTick(ctx) {
|
|
1996
2205
|
await processNewlyFiredEventTriggers(ctx).catch((err) => {
|
|
1997
2206
|
console.error("mailery: triggers scan failed", err);
|
|
@@ -1999,6 +2208,9 @@ async function runTick(ctx) {
|
|
|
1999
2208
|
await sweepStrandedFlowRuns(ctx).catch((err) => {
|
|
2000
2209
|
console.error("mailery: sweep failed", err);
|
|
2001
2210
|
});
|
|
2211
|
+
await sweepStrandedSends(ctx).catch((err) => {
|
|
2212
|
+
console.error("mailery: stranded-send sweep failed", err);
|
|
2213
|
+
});
|
|
2002
2214
|
await drainOutbox(ctx).catch((err) => {
|
|
2003
2215
|
console.error("mailery: outbox drain failed", err);
|
|
2004
2216
|
});
|
|
@@ -2012,6 +2224,24 @@ async function runTick(ctx) {
|
|
|
2012
2224
|
console.error("mailery: soft-bounce promotion failed", err);
|
|
2013
2225
|
});
|
|
2014
2226
|
}
|
|
2227
|
+
async function sweepStrandedSends(ctx) {
|
|
2228
|
+
const cutoff = new Date(Date.now() - STRANDED_SEND_THRESHOLD_MS);
|
|
2229
|
+
const cursor = ctx.collections.sends.find(
|
|
2230
|
+
{ status: "sending", updatedAt: { $lt: cutoff } },
|
|
2231
|
+
{ projection: { _id: 1 } }
|
|
2232
|
+
).limit(500);
|
|
2233
|
+
for await (const row of cursor) {
|
|
2234
|
+
const reset = await ctx.collections.sends.updateOne(
|
|
2235
|
+
{ _id: row._id, status: "sending", updatedAt: { $lt: cutoff } },
|
|
2236
|
+
{ $set: { status: "queued", updatedAt: /* @__PURE__ */ new Date() } }
|
|
2237
|
+
);
|
|
2238
|
+
if (reset.modifiedCount === 0) continue;
|
|
2239
|
+
await ctx.queues.send.add("send", { sendId: String(row._id) }, {
|
|
2240
|
+
attempts: ctx.config.sendRetryAttempts,
|
|
2241
|
+
backoff: { type: "exponential", delay: 6e4 }
|
|
2242
|
+
});
|
|
2243
|
+
}
|
|
2244
|
+
}
|
|
2015
2245
|
async function drainOutbox(ctx) {
|
|
2016
2246
|
const batch = await ctx.collections.outbox.find({ status: "pending" }).sort({ enqueuedAt: 1 }).limit(200).toArray();
|
|
2017
2247
|
for (const row of batch) {
|
|
@@ -2182,12 +2412,11 @@ var Mailer = class _Mailer {
|
|
|
2182
2412
|
collections;
|
|
2183
2413
|
adapter;
|
|
2184
2414
|
providers;
|
|
2185
|
-
redis;
|
|
2186
2415
|
queues;
|
|
2187
2416
|
config;
|
|
2188
2417
|
events;
|
|
2189
|
-
|
|
2190
|
-
|
|
2418
|
+
queueDriver;
|
|
2419
|
+
workersStarted = false;
|
|
2191
2420
|
runnerContext;
|
|
2192
2421
|
constructor(args) {
|
|
2193
2422
|
this.config = args.config;
|
|
@@ -2195,9 +2424,8 @@ var Mailer = class _Mailer {
|
|
|
2195
2424
|
this.collections = args.collections;
|
|
2196
2425
|
this.adapter = args.adapter;
|
|
2197
2426
|
this.providers = args.providers;
|
|
2198
|
-
this.
|
|
2199
|
-
this.queues = args.queues;
|
|
2200
|
-
this.bullQueues = args.bullQueues;
|
|
2427
|
+
this.queueDriver = args.queueDriver;
|
|
2428
|
+
this.queues = args.queueDriver.queues;
|
|
2201
2429
|
this.events = args.events;
|
|
2202
2430
|
this.runnerContext = {
|
|
2203
2431
|
db: this.db,
|
|
@@ -2262,10 +2490,12 @@ var Mailer = class _Mailer {
|
|
|
2262
2490
|
throw new Error("Mailer.fromEnv: no provider configured (set MAILER_SENDGRID_API_KEY, ...)");
|
|
2263
2491
|
}
|
|
2264
2492
|
const defaultProvider = env.MAILER_DEFAULT_PROVIDER ?? Object.keys(providers)[0];
|
|
2493
|
+
const driverEnv = env.MAILER_QUEUE_DRIVER ?? "bull";
|
|
2494
|
+
const queue = driverEnv === "agenda" ? { driver: "agenda" } : driverEnv === "noop" ? { driver: "noop" } : { driver: "bull", redis: { url: required("MAILER_REDIS_URL") } };
|
|
2265
2495
|
return _Mailer.init({
|
|
2266
2496
|
db,
|
|
2267
2497
|
adapter,
|
|
2268
|
-
|
|
2498
|
+
queue,
|
|
2269
2499
|
providers,
|
|
2270
2500
|
defaultProvider,
|
|
2271
2501
|
publicUrl: required("MAILER_PUBLIC_URL"),
|
|
@@ -2281,19 +2511,9 @@ var Mailer = class _Mailer {
|
|
|
2281
2511
|
}
|
|
2282
2512
|
const collections = getCollections(config.db, config.collectionPrefix);
|
|
2283
2513
|
await ensureIndexes(config.db, config.collectionPrefix);
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
if (config.redis === null) {
|
|
2288
|
-
queues = noopQueues();
|
|
2289
|
-
} else {
|
|
2290
|
-
redis = makeRedis(config.redis);
|
|
2291
|
-
const created = createQueues(redis);
|
|
2292
|
-
queues = created.queues;
|
|
2293
|
-
bullQueues = created.bullQueues;
|
|
2294
|
-
if (!config.workerless) {
|
|
2295
|
-
await scheduleTick(bullQueues, config.tickIntervalSeconds);
|
|
2296
|
-
}
|
|
2514
|
+
const queueDriver = await createQueueDriver(config.queue, config.db);
|
|
2515
|
+
if (!config.workerless && config.queue.driver !== "noop") {
|
|
2516
|
+
await queueDriver.scheduleRepeatingTick(config.tickIntervalSeconds);
|
|
2297
2517
|
}
|
|
2298
2518
|
return new _Mailer({
|
|
2299
2519
|
config,
|
|
@@ -2301,9 +2521,7 @@ var Mailer = class _Mailer {
|
|
|
2301
2521
|
collections,
|
|
2302
2522
|
adapter: config.adapter,
|
|
2303
2523
|
providers: config.providers,
|
|
2304
|
-
|
|
2305
|
-
queues,
|
|
2306
|
-
bullQueues,
|
|
2524
|
+
queueDriver,
|
|
2307
2525
|
events: new EventRegistry()
|
|
2308
2526
|
});
|
|
2309
2527
|
}
|
|
@@ -2611,6 +2829,7 @@ var Mailer = class _Mailer {
|
|
|
2611
2829
|
unsubscribedAt: null,
|
|
2612
2830
|
complainedAt: null,
|
|
2613
2831
|
queuedAt: /* @__PURE__ */ new Date(),
|
|
2832
|
+
updatedAt: /* @__PURE__ */ new Date(),
|
|
2614
2833
|
sentAt: null,
|
|
2615
2834
|
deliveredAt: null
|
|
2616
2835
|
});
|
|
@@ -2642,14 +2861,16 @@ var Mailer = class _Mailer {
|
|
|
2642
2861
|
// Workers
|
|
2643
2862
|
// -------------------------------------------------------------------------
|
|
2644
2863
|
async startWorkers() {
|
|
2645
|
-
if (this.
|
|
2646
|
-
if (
|
|
2864
|
+
if (this.workersStarted) return;
|
|
2865
|
+
if (this.config.queue.driver === "noop") {
|
|
2866
|
+
throw new Error("startWorkers requires a non-noop queue driver");
|
|
2867
|
+
}
|
|
2647
2868
|
const provider = this.providers[this.config.defaultProvider];
|
|
2648
2869
|
const sendRate = provider?.sendRatePerSecond ?? this.config.sendRatePerSecond;
|
|
2649
|
-
this.
|
|
2650
|
-
redis: this.redis,
|
|
2870
|
+
await this.queueDriver.startWorkers({
|
|
2651
2871
|
concurrency: { send: this.config.sendConcurrency },
|
|
2652
2872
|
sendRateLimit: { max: sendRate, durationMs: 1e3 },
|
|
2873
|
+
retryAttempts: this.config.sendRetryAttempts,
|
|
2653
2874
|
handlers: {
|
|
2654
2875
|
tick: async () => {
|
|
2655
2876
|
await runTick(this.runnerContext);
|
|
@@ -2667,6 +2888,7 @@ var Mailer = class _Mailer {
|
|
|
2667
2888
|
}
|
|
2668
2889
|
}
|
|
2669
2890
|
});
|
|
2891
|
+
this.workersStarted = true;
|
|
2670
2892
|
}
|
|
2671
2893
|
/** Process unprocessed webhook events in mailer_webhook_events. */
|
|
2672
2894
|
async processWebhookBacklog() {
|
|
@@ -2693,20 +2915,8 @@ var Mailer = class _Mailer {
|
|
|
2693
2915
|
}
|
|
2694
2916
|
}
|
|
2695
2917
|
async stop() {
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
this.workers = null;
|
|
2699
|
-
}
|
|
2700
|
-
if (this.bullQueues) {
|
|
2701
|
-
await closeBullQueues(this.bullQueues);
|
|
2702
|
-
this.bullQueues = null;
|
|
2703
|
-
} else {
|
|
2704
|
-
await closeQueues(this.queues);
|
|
2705
|
-
}
|
|
2706
|
-
if (this.redis) {
|
|
2707
|
-
await this.redis.quit().catch(() => {
|
|
2708
|
-
});
|
|
2709
|
-
}
|
|
2918
|
+
await this.queueDriver.close();
|
|
2919
|
+
this.workersStarted = false;
|
|
2710
2920
|
}
|
|
2711
2921
|
/** Used internally by the admin router and tests; not part of the public API. */
|
|
2712
2922
|
getRunnerContext() {
|