notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { loadEnv, parseConfig, baseConfigSchema } from "@/index.js";
|
|
2
|
-
import {
|
|
3
|
-
import { RedisClient,
|
|
2
|
+
import type { Logger } from "@/index.js";
|
|
3
|
+
import type { RedisClient, Redis } from "@/index.js";
|
|
4
4
|
import {
|
|
5
|
-
StreamConsumer,
|
|
6
|
-
PendingMessageScanner,
|
|
5
|
+
type StreamConsumer,
|
|
6
|
+
type PendingMessageScanner,
|
|
7
7
|
type StreamMessage,
|
|
8
8
|
StreamProducer,
|
|
9
9
|
} from "@/index.js";
|
|
10
|
-
import { BaseWorker } from "@/index.js";
|
|
10
|
+
import { BaseWorker, type ProcessResult } from "@/index.js";
|
|
11
11
|
import {
|
|
12
12
|
STREAMS,
|
|
13
13
|
OUTBOUND_STREAMS,
|
|
@@ -15,16 +15,27 @@ import {
|
|
|
15
15
|
registry,
|
|
16
16
|
buildStreamEvent,
|
|
17
17
|
type NotificationDispatchedPayload,
|
|
18
|
+
type DispatchedTaskPayload,
|
|
19
|
+
type RenderedContent,
|
|
20
|
+
PUBSUB_CHANNELS,
|
|
18
21
|
} from "@/index.js";
|
|
19
|
-
import {
|
|
20
|
-
import { createDatabase } from "@/db/index.js";
|
|
22
|
+
import { upsertOutboxProviderIds } from "@/db/bulk.js";
|
|
21
23
|
import { deliveryOutbox, scheduledPayloads } from "@/db/schema.js";
|
|
22
|
-
import {
|
|
24
|
+
import { sql as drizzleSql } from "drizzle-orm";
|
|
25
|
+
import { ContactRepository, IdempotencyGuard, TemplateRepository } from "@/index.js";
|
|
26
|
+
import { renderWithTemplate, TemplateCache } from "@/templates/index.js";
|
|
23
27
|
import { transportRegistry } from "@/index.js";
|
|
24
28
|
import { globalEmitter, getPriorityBucket, type WorkerOptions } from "@/shared/index.js";
|
|
25
29
|
import { startHealthReporter, NonRetryableError } from "@/workers/index.js";
|
|
30
|
+
import {
|
|
31
|
+
createPriorityProducers,
|
|
32
|
+
createStreamConsumer,
|
|
33
|
+
createWorkerRuntime,
|
|
34
|
+
shutdownWorker,
|
|
35
|
+
uniqueConsumerId,
|
|
36
|
+
} from "@/workers/bootstrap.js";
|
|
26
37
|
import { BatchProcessor, CircuitBreaker } from "@/shared/index.js";
|
|
27
|
-
import {
|
|
38
|
+
import { ProviderThrottle } from "./throttle.js";
|
|
28
39
|
import { metrics } from "@/metrics/index.js";
|
|
29
40
|
|
|
30
41
|
// ─── App-specific config ────────────────────────────────────────────────────
|
|
@@ -36,7 +47,7 @@ const deliveryConfigSchema = baseConfigSchema.extend({});
|
|
|
36
47
|
loadEnv();
|
|
37
48
|
const config = parseConfig(deliveryConfigSchema, process.env);
|
|
38
49
|
|
|
39
|
-
let logger:
|
|
50
|
+
let logger: Logger;
|
|
40
51
|
let redis: RedisClient;
|
|
41
52
|
let sql: any;
|
|
42
53
|
let db: any;
|
|
@@ -47,6 +58,7 @@ let worker: BaseWorker;
|
|
|
47
58
|
let scheduledProducer: StreamProducer;
|
|
48
59
|
let enrichedProducers: Record<string, StreamProducer>;
|
|
49
60
|
let healthInterval: NodeJS.Timeout | null = null;
|
|
61
|
+
let subscriber: Redis | null = null;
|
|
50
62
|
|
|
51
63
|
(global as any)._telemetry = (global as any)._telemetry || {
|
|
52
64
|
count: 0,
|
|
@@ -59,6 +71,48 @@ let healthInterval: NodeJS.Timeout | null = null;
|
|
|
59
71
|
flushCount: 0,
|
|
60
72
|
};
|
|
61
73
|
|
|
74
|
+
/** How long a send may run before its dispatched log entry is published alone. */
|
|
75
|
+
const DISPATCH_LOG_DELAY_MS = 1_000;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Wait before re-trying a send that failed at the provider, by prior failures.
|
|
79
|
+
* The first step matches the circuit breaker's reset timeout, so a retry of a
|
|
80
|
+
* task the open breaker turned away reaches the provider's recovery probe
|
|
81
|
+
* rather than the same closed door.
|
|
82
|
+
*/
|
|
83
|
+
const RETRY_BACKOFF_MS = [30_000, 120_000, 600_000];
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* How many times a send may be put back for want of provider capacity before
|
|
87
|
+
* it is failed. Separate from `maxAttempts`, which counts sends the provider
|
|
88
|
+
* refused: waiting in line is not a failed attempt, and a campaign several
|
|
89
|
+
* windows deeper than the provider's limit has to be able to wait that long.
|
|
90
|
+
*/
|
|
91
|
+
export const MAX_THROTTLE_DEFERRALS = 50;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Names the provider at `transports[i]` for its rate limit and circuit breaker.
|
|
95
|
+
* The rate limit's budget lives in Redis and is shared by every delivery
|
|
96
|
+
* process, so the name must come out the same everywhere: the class name,
|
|
97
|
+
* numbered when a channel has two providers of one class.
|
|
98
|
+
*/
|
|
99
|
+
function providerKey(channel: string, transports: any[], i: number): string {
|
|
100
|
+
const name = transports[i].constructor?.name ?? "provider";
|
|
101
|
+
const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
|
|
102
|
+
return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A provider's refusal, keeping whether it said another attempt could work. */
|
|
106
|
+
function providerFailure(res: { error?: string; retryable?: boolean }): Error {
|
|
107
|
+
const err = new Error(res.error ?? "Transport failed");
|
|
108
|
+
(err as any).retryable = res.retryable;
|
|
109
|
+
return err;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function retryDelayMs(priorFailures: number): number {
|
|
113
|
+
return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)]!;
|
|
114
|
+
}
|
|
115
|
+
|
|
62
116
|
export interface DeliveryWorkerOptions extends WorkerOptions {
|
|
63
117
|
transportRegistry: any;
|
|
64
118
|
idempotency: any;
|
|
@@ -69,12 +123,14 @@ export interface DeliveryWorkerOptions extends WorkerOptions {
|
|
|
69
123
|
eventsProducer: any;
|
|
70
124
|
globalEmitter: any;
|
|
71
125
|
db: any;
|
|
126
|
+
providerThrottle?: ProviderThrottle;
|
|
127
|
+
/** Renders tasks that arrive without `renderedContent`. */
|
|
128
|
+
templateCache?: TemplateCache;
|
|
72
129
|
}
|
|
73
130
|
|
|
74
131
|
export class DeliveryWorker extends BaseWorker {
|
|
75
132
|
private readonly transportRegistry: any;
|
|
76
133
|
private readonly idempotency: any;
|
|
77
|
-
private readonly redisCli: Redis;
|
|
78
134
|
private readonly scheduledProducer: any;
|
|
79
135
|
private readonly enrichedProducers: any;
|
|
80
136
|
private readonly contactRepo: any;
|
|
@@ -86,49 +142,63 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
86
142
|
private outboxUpdateProcessor: BatchProcessor<any, void>;
|
|
87
143
|
private outboxInsertProcessor: BatchProcessor<any, boolean>;
|
|
88
144
|
private breakers = new Map<string, CircuitBreaker>();
|
|
145
|
+
private readonly providerThrottle: ProviderThrottle;
|
|
146
|
+
private readonly templateCache?: TemplateCache;
|
|
89
147
|
|
|
90
148
|
constructor(options: DeliveryWorkerOptions) {
|
|
91
149
|
super(options);
|
|
92
150
|
this.transportRegistry = options.transportRegistry;
|
|
93
151
|
this.idempotency = options.idempotency;
|
|
94
|
-
this.redisCli = options.redis;
|
|
95
152
|
this.scheduledProducer = options.scheduledProducer;
|
|
96
153
|
this.enrichedProducers = options.enrichedProducers;
|
|
97
154
|
this.contactRepo = options.contactRepo;
|
|
98
155
|
this.eventsProducer = options.eventsProducer;
|
|
99
156
|
this.globalEmitter = options.globalEmitter;
|
|
100
157
|
this.db = options.db;
|
|
158
|
+
this.providerThrottle =
|
|
159
|
+
options.providerThrottle ?? new ProviderThrottle(options.redis, this.logger);
|
|
160
|
+
this.templateCache = options.templateCache;
|
|
161
|
+
|
|
162
|
+
// Every message waits on these flushes before it is acked, so they bound
|
|
163
|
+
// the worker's throughput: allow a few in flight and let batches grow with
|
|
164
|
+
// the configured concurrency.
|
|
165
|
+
const batchSize = Math.max(10, Math.min(options.concurrency ?? 50, 500));
|
|
166
|
+
const flushConcurrency = 4;
|
|
167
|
+
|
|
168
|
+
this.eventProcessor = new BatchProcessor<any, void>(
|
|
169
|
+
batchSize,
|
|
170
|
+
5,
|
|
171
|
+
async (events) => {
|
|
172
|
+
await this.eventsProducer.publishBatch(events);
|
|
173
|
+
return events.map(() => undefined as void);
|
|
174
|
+
},
|
|
175
|
+
flushConcurrency,
|
|
176
|
+
);
|
|
177
|
+
|
|
178
|
+
this.outboxUpdateProcessor = new BatchProcessor<any, void>(
|
|
179
|
+
batchSize,
|
|
180
|
+
5,
|
|
181
|
+
async (updates) => {
|
|
182
|
+
const values = updates.map((update) => ({
|
|
183
|
+
taskId: update.taskId,
|
|
184
|
+
channel: update.channel,
|
|
185
|
+
destination: update.destination,
|
|
186
|
+
providerMessageId: update.providerMessageId,
|
|
187
|
+
}));
|
|
188
|
+
|
|
189
|
+
const tDbUpdateStart = Date.now();
|
|
190
|
+
await upsertOutboxProviderIds(this.db, values).catch((e: any) =>
|
|
191
|
+
this.logger.error({ err: e }, "background update failed"),
|
|
192
|
+
);
|
|
101
193
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const { sql } = await import("drizzle-orm");
|
|
109
|
-
const values = updates.map((update) => ({
|
|
110
|
-
taskId: update.taskId,
|
|
111
|
-
channel: update.channel,
|
|
112
|
-
destination: update.destination,
|
|
113
|
-
providerMessageId: update.providerMessageId,
|
|
114
|
-
}));
|
|
115
|
-
|
|
116
|
-
const tDbUpdateStart = Date.now();
|
|
117
|
-
await this.db
|
|
118
|
-
.insert(deliveryOutbox)
|
|
119
|
-
.values(values)
|
|
120
|
-
.onConflictDoUpdate({
|
|
121
|
-
target: [deliveryOutbox.taskId, deliveryOutbox.channel, deliveryOutbox.destination],
|
|
122
|
-
set: { providerMessageId: sql`EXCLUDED.provider_message_id` },
|
|
123
|
-
})
|
|
124
|
-
.catch((e: any) => this.logger.error({ err: e }, "background update failed"));
|
|
125
|
-
|
|
126
|
-
(global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
|
|
127
|
-
(global as any)._telemetry.flushCount++;
|
|
128
|
-
return updates.map(() => undefined as void);
|
|
129
|
-
});
|
|
194
|
+
(global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
|
|
195
|
+
(global as any)._telemetry.flushCount++;
|
|
196
|
+
return updates.map(() => undefined as void);
|
|
197
|
+
},
|
|
198
|
+
flushConcurrency,
|
|
199
|
+
);
|
|
130
200
|
|
|
131
|
-
this.outboxInsertProcessor = new BatchProcessor(
|
|
201
|
+
this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
|
|
132
202
|
const values = tasks.map((task: any) => ({
|
|
133
203
|
taskId: task.taskId,
|
|
134
204
|
channel: task.channel,
|
|
@@ -149,6 +219,68 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
149
219
|
await super.stop();
|
|
150
220
|
}
|
|
151
221
|
|
|
222
|
+
/**
|
|
223
|
+
* Renders a task the engine dispatched unrendered. Done here, after the
|
|
224
|
+
* throttle and the lease, so the body is built once per actual send and never
|
|
225
|
+
* travels through the outbound stream.
|
|
226
|
+
*/
|
|
227
|
+
private async render(task: NotificationDispatchedPayload): Promise<RenderedContent> {
|
|
228
|
+
if (!task.templateId) return renderWithTemplate(null, task.templateVariables);
|
|
229
|
+
const template = this.templateCache
|
|
230
|
+
? await this.templateCache.getCachedTemplate(task.projectId, task.templateId)
|
|
231
|
+
: null;
|
|
232
|
+
if (!template) {
|
|
233
|
+
// The engine checked it existed; it was deleted since. Retrying won't help.
|
|
234
|
+
throw new NonRetryableError(`template "${task.templateId}" not found at send time`);
|
|
235
|
+
}
|
|
236
|
+
return renderWithTemplate(template, task.templateVariables);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Parks a task in Postgres and the scheduler until `sendAt`. The payload
|
|
241
|
+
* carries the send time, so the scheduler can rebuild it if Redis loses its
|
|
242
|
+
* data.
|
|
243
|
+
*/
|
|
244
|
+
private async reschedule(
|
|
245
|
+
task: DispatchedTaskPayload,
|
|
246
|
+
sendAt: Date,
|
|
247
|
+
reason: string,
|
|
248
|
+
): Promise<void> {
|
|
249
|
+
const scheduledAt = sendAt.toISOString();
|
|
250
|
+
await this.db
|
|
251
|
+
.insert(scheduledPayloads)
|
|
252
|
+
.values({ taskId: task.taskId, payload: { ...task, scheduledAt } })
|
|
253
|
+
.onConflictDoUpdate({
|
|
254
|
+
target: scheduledPayloads.taskId,
|
|
255
|
+
set: { payload: drizzleSql`EXCLUDED.payload` },
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
await this.scheduledProducer.publish(
|
|
259
|
+
buildStreamEvent(
|
|
260
|
+
"notification.scheduled",
|
|
261
|
+
{
|
|
262
|
+
projectId: task.projectId,
|
|
263
|
+
enrichedEventId: task.enrichedEventId,
|
|
264
|
+
taskId: task.taskId,
|
|
265
|
+
scheduledAt,
|
|
266
|
+
},
|
|
267
|
+
"delivery",
|
|
268
|
+
`${task.taskId}:${reason}:${Date.now()}`,
|
|
269
|
+
),
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Takes a slot from the provider's rate limit, if it has one. */
|
|
274
|
+
private async providerGate(
|
|
275
|
+
channel: string,
|
|
276
|
+
transports: any[],
|
|
277
|
+
i: number,
|
|
278
|
+
): Promise<{ allowed: boolean; retryAfterMs: number }> {
|
|
279
|
+
const limits = transports[i].limits;
|
|
280
|
+
if (!limits) return { allowed: true, retryAfterMs: 0 };
|
|
281
|
+
return this.providerThrottle.check(providerKey(channel, transports, i), limits);
|
|
282
|
+
}
|
|
283
|
+
|
|
152
284
|
private getBreaker(name: string): CircuitBreaker {
|
|
153
285
|
let breaker = this.breakers.get(name);
|
|
154
286
|
if (!breaker) {
|
|
@@ -158,7 +290,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
158
290
|
return breaker;
|
|
159
291
|
}
|
|
160
292
|
|
|
161
|
-
async process(message: StreamMessage, attempt: number = 1): Promise<
|
|
293
|
+
async process(message: StreamMessage, attempt: number = 1): Promise<ProcessResult> {
|
|
162
294
|
const { event } = message;
|
|
163
295
|
const publishPromises: Promise<void>[] = [];
|
|
164
296
|
|
|
@@ -172,6 +304,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
172
304
|
}
|
|
173
305
|
|
|
174
306
|
const task = payloadResult.data as NotificationDispatchedPayload;
|
|
307
|
+
// Attempts made on earlier trips through the scheduler, plus this one.
|
|
308
|
+
const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
|
|
309
|
+
// What gets parked for a retry: the task as it arrived, before rendering
|
|
310
|
+
// and the transport's abort signal were attached to it.
|
|
311
|
+
const original: DispatchedTaskPayload = { ...task };
|
|
175
312
|
|
|
176
313
|
const fallbackToNextChannel = async (reason: string) => {
|
|
177
314
|
if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
|
|
@@ -191,6 +328,9 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
191
328
|
recipient: task.recipient,
|
|
192
329
|
scheduledAt: undefined,
|
|
193
330
|
fallbackChain: remainingChain.length > 0 ? remainingChain : undefined,
|
|
331
|
+
campaignId: task.campaignId,
|
|
332
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
333
|
+
throttleCounted: true,
|
|
194
334
|
};
|
|
195
335
|
|
|
196
336
|
const p = getPriorityBucket(task.priority);
|
|
@@ -241,10 +381,9 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
241
381
|
failureReason: "no transport registered for channel",
|
|
242
382
|
failureCode: "no_transport",
|
|
243
383
|
retryable: false,
|
|
244
|
-
attempt,
|
|
384
|
+
attempt: attemptNo,
|
|
245
385
|
templateId: task.templateId,
|
|
246
|
-
workflowInstanceId:
|
|
247
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
386
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
248
387
|
campaignId: task.campaignId,
|
|
249
388
|
},
|
|
250
389
|
"delivery",
|
|
@@ -255,137 +394,216 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
255
394
|
return;
|
|
256
395
|
}
|
|
257
396
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
)
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
397
|
+
// Each provider has its own budget: start at the first with room left. The
|
|
398
|
+
// channel is out of capacity only once every provider for it is.
|
|
399
|
+
let startAt = -1;
|
|
400
|
+
let retryAfterMs = Infinity;
|
|
401
|
+
let windowMs = Infinity;
|
|
402
|
+
for (let i = 0; i < transports.length; i++) {
|
|
403
|
+
const gate = await this.providerGate(task.channel, transports, i);
|
|
404
|
+
if (gate.allowed) {
|
|
405
|
+
startAt = i;
|
|
406
|
+
break;
|
|
407
|
+
}
|
|
408
|
+
retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs);
|
|
409
|
+
windowMs = Math.min(windowMs, (transports[i].limits?.windowSeconds ?? 0) * 1000);
|
|
410
|
+
}
|
|
411
|
+
if (startAt === -1) {
|
|
412
|
+
task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
|
|
269
413
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
414
|
+
if (task.throttleAttemptCount > MAX_THROTTLE_DEFERRALS) {
|
|
415
|
+
this.logger.warn(
|
|
416
|
+
{ messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount },
|
|
417
|
+
"provider rate limit max attempts exceeded",
|
|
418
|
+
);
|
|
419
|
+
const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
|
|
420
|
+
if (!fallbackTriggered) {
|
|
421
|
+
this.globalEmitter.emit(
|
|
422
|
+
"delivery:failed",
|
|
423
|
+
task.taskId,
|
|
424
|
+
"provider throttle exceeded",
|
|
425
|
+
task.channel,
|
|
426
|
+
task.projectId,
|
|
427
|
+
);
|
|
428
|
+
await this.eventProcessor.add(
|
|
429
|
+
buildStreamEvent(
|
|
430
|
+
"notification.failed",
|
|
431
|
+
{
|
|
432
|
+
projectId: task.projectId,
|
|
433
|
+
taskId: task.taskId,
|
|
434
|
+
enrichedEventId: task.enrichedEventId,
|
|
435
|
+
recipientId: task.recipientId,
|
|
436
|
+
channel: task.channel,
|
|
437
|
+
failureReason: "provider rate limit max attempts exceeded",
|
|
438
|
+
failureCode: "provider_throttle_exceeded",
|
|
439
|
+
retryable: false,
|
|
440
|
+
attempt: attemptNo,
|
|
441
|
+
templateId: task.templateId,
|
|
442
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
443
|
+
campaignId: task.campaignId,
|
|
444
|
+
},
|
|
445
|
+
"delivery",
|
|
446
|
+
event.metadata.traceId,
|
|
447
|
+
),
|
|
274
448
|
);
|
|
275
|
-
const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
|
|
276
|
-
if (!fallbackTriggered) {
|
|
277
|
-
this.globalEmitter.emit(
|
|
278
|
-
"delivery:failed",
|
|
279
|
-
task.taskId,
|
|
280
|
-
"provider throttle exceeded",
|
|
281
|
-
task.channel,
|
|
282
|
-
task.projectId,
|
|
283
|
-
);
|
|
284
|
-
await this.eventProcessor.add(
|
|
285
|
-
buildStreamEvent(
|
|
286
|
-
"notification.failed",
|
|
287
|
-
{
|
|
288
|
-
projectId: task.projectId,
|
|
289
|
-
taskId: task.taskId,
|
|
290
|
-
enrichedEventId: task.enrichedEventId,
|
|
291
|
-
recipientId: task.recipientId,
|
|
292
|
-
channel: task.channel,
|
|
293
|
-
failureReason: "provider rate limit max attempts exceeded",
|
|
294
|
-
failureCode: "provider_throttle_exceeded",
|
|
295
|
-
retryable: false,
|
|
296
|
-
attempt,
|
|
297
|
-
templateId: task.templateId,
|
|
298
|
-
workflowInstanceId:
|
|
299
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
300
|
-
campaignId: task.campaignId,
|
|
301
|
-
},
|
|
302
|
-
"delivery",
|
|
303
|
-
event.metadata.traceId,
|
|
304
|
-
),
|
|
305
|
-
);
|
|
306
|
-
}
|
|
307
|
-
return;
|
|
308
449
|
}
|
|
309
|
-
|
|
310
|
-
const { sql } = await import("drizzle-orm");
|
|
311
|
-
await this.db
|
|
312
|
-
.insert(scheduledPayloads)
|
|
313
|
-
.values({
|
|
314
|
-
taskId: task.taskId,
|
|
315
|
-
payload: task,
|
|
316
|
-
})
|
|
317
|
-
.onConflictDoUpdate({
|
|
318
|
-
target: scheduledPayloads.taskId,
|
|
319
|
-
set: { payload: sql`EXCLUDED.payload` },
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
await this.scheduledProducer.publish(
|
|
323
|
-
buildStreamEvent(
|
|
324
|
-
"notification.scheduled",
|
|
325
|
-
{
|
|
326
|
-
projectId: task.projectId,
|
|
327
|
-
enrichedEventId: task.enrichedEventId,
|
|
328
|
-
taskId: task.taskId,
|
|
329
|
-
scheduledAt: new Date(Date.now() + retryAfterMs).toISOString(),
|
|
330
|
-
throttleAttemptCount: task.throttleAttemptCount,
|
|
331
|
-
},
|
|
332
|
-
"delivery",
|
|
333
|
-
`${task.taskId}:throttle:${Date.now()}`,
|
|
334
|
-
),
|
|
335
|
-
);
|
|
336
|
-
|
|
337
450
|
return;
|
|
338
451
|
}
|
|
452
|
+
|
|
453
|
+
// Spread over more windows the longer a task has waited. Every task turned
|
|
454
|
+
// away in one window used to come back at the same boundary, where the
|
|
455
|
+
// limit again let one window's worth through and bounced the rest.
|
|
456
|
+
const spreadMs = Number.isFinite(windowMs)
|
|
457
|
+
? Math.random() * windowMs * Math.min(task.throttleAttemptCount, 10)
|
|
458
|
+
: 0;
|
|
459
|
+
await this.reschedule(
|
|
460
|
+
{ ...original, throttleAttemptCount: task.throttleAttemptCount },
|
|
461
|
+
new Date(Date.now() + retryAfterMs + spreadMs),
|
|
462
|
+
"throttle",
|
|
463
|
+
);
|
|
464
|
+
return;
|
|
339
465
|
}
|
|
340
466
|
|
|
341
|
-
const tInsertStart = Date.now();
|
|
342
467
|
const idempotencyKey = task.taskId;
|
|
343
|
-
|
|
468
|
+
const lease = this.idempotency.acquireLease
|
|
469
|
+
? await this.idempotency.acquireLease(idempotencyKey, 60)
|
|
470
|
+
: (await this.idempotency.checkAndMark(idempotencyKey, 60))
|
|
471
|
+
? "acquired"
|
|
472
|
+
: "locked";
|
|
473
|
+
|
|
474
|
+
if (lease === "completed") {
|
|
344
475
|
this.logger.info(
|
|
345
|
-
{ messageId: message.id, taskId: task.taskId, channel: task.channel
|
|
346
|
-
"duplicate delivery —
|
|
476
|
+
{ messageId: message.id, taskId: task.taskId, channel: task.channel },
|
|
477
|
+
"duplicate delivery — already completed, acking message",
|
|
347
478
|
);
|
|
348
479
|
return;
|
|
349
480
|
}
|
|
350
481
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
482
|
+
if (lease === "locked") {
|
|
483
|
+
this.logger.info(
|
|
484
|
+
{ messageId: message.id, taskId: task.taskId, channel: task.channel, attempt },
|
|
485
|
+
"duplicate delivery — skipping and leaving pending in stream",
|
|
486
|
+
);
|
|
487
|
+
return { ack: false };
|
|
488
|
+
}
|
|
354
489
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
490
|
+
const tDispatchStart = Date.now();
|
|
491
|
+
const MAX_DISPATCH_BUDGET_MS = 25_000; // Strictly shorter than lock lease (30s)
|
|
492
|
+
|
|
493
|
+
// The attempt's `dispatched` log entry normally rides on its outcome event,
|
|
494
|
+
// one stream entry instead of two. A send still running after
|
|
495
|
+
// DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
|
|
496
|
+
// visible while it is in flight, as it always was.
|
|
497
|
+
let dispatchStarted = false;
|
|
498
|
+
let dispatchLogged = false;
|
|
499
|
+
let dispatchedAt = "";
|
|
500
|
+
let dispatchLogTimer: NodeJS.Timeout | undefined;
|
|
501
|
+
const logDispatched = (): Promise<void> | undefined => {
|
|
502
|
+
clearTimeout(dispatchLogTimer);
|
|
503
|
+
if (!dispatchStarted || dispatchLogged) return undefined;
|
|
504
|
+
dispatchLogged = true;
|
|
505
|
+
const logged = this.eventProcessor.add(
|
|
506
|
+
buildStreamEvent(
|
|
507
|
+
"notification.dispatched",
|
|
508
|
+
{
|
|
509
|
+
projectId: task.projectId,
|
|
510
|
+
taskId: task.taskId,
|
|
511
|
+
enrichedEventId: task.enrichedEventId,
|
|
512
|
+
recipientId: task.recipientId,
|
|
513
|
+
channel: task.channel,
|
|
514
|
+
templateId: task.templateId,
|
|
515
|
+
attempt: attemptNo,
|
|
516
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
517
|
+
campaignId: task.campaignId,
|
|
518
|
+
},
|
|
519
|
+
"delivery",
|
|
520
|
+
event.metadata.traceId,
|
|
374
521
|
),
|
|
375
522
|
);
|
|
523
|
+
publishPromises.push(logged);
|
|
524
|
+
return logged;
|
|
525
|
+
};
|
|
526
|
+
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
527
|
+
const outcome = <T extends Record<string, unknown>>(payload: T): T => {
|
|
528
|
+
clearTimeout(dispatchLogTimer);
|
|
529
|
+
if (dispatchLogged) return payload;
|
|
530
|
+
dispatchLogged = true;
|
|
531
|
+
return { ...payload, attempt: attemptNo, dispatchedAt };
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
let retryScheduled = false;
|
|
535
|
+
/**
|
|
536
|
+
* Re-queues a send the provider turned away, unless it said the failure is
|
|
537
|
+
* permanent or the task is out of attempts. An outage, a timeout or an open
|
|
538
|
+
* circuit is exactly what a later attempt can get past; failing the task
|
|
539
|
+
* outright turned every blip into lost mail. The attempt stays in the log as
|
|
540
|
+
* dispatched, and the task's outcome is whatever its last attempt comes to.
|
|
541
|
+
*/
|
|
542
|
+
const retryLater = async (failure: { retryable?: boolean; error?: string }) => {
|
|
543
|
+
const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
|
|
544
|
+
if (failure.retryable === false || attemptNo >= maxAttempts) return false;
|
|
545
|
+
const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0);
|
|
546
|
+
await this.reschedule(
|
|
547
|
+
{ ...original, deliveryAttemptCount: attemptNo },
|
|
548
|
+
new Date(Date.now() + delayMs),
|
|
549
|
+
"retry",
|
|
550
|
+
);
|
|
551
|
+
retryScheduled = true;
|
|
552
|
+
metrics.deliveryFailed.inc({ channel: task.channel, reason: "retry_scheduled" });
|
|
553
|
+
this.logger.warn(
|
|
554
|
+
{
|
|
555
|
+
taskId: task.taskId,
|
|
556
|
+
channel: task.channel,
|
|
557
|
+
attempt: attemptNo,
|
|
558
|
+
maxAttempts,
|
|
559
|
+
delayMs,
|
|
560
|
+
error: failure.error,
|
|
561
|
+
},
|
|
562
|
+
"provider send failed — retrying later",
|
|
563
|
+
);
|
|
564
|
+
return true;
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
try {
|
|
568
|
+
if (!task.renderedContent) {
|
|
569
|
+
task.renderedContent = await this.render(task);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
dispatchStarted = true;
|
|
573
|
+
dispatchedAt = new Date().toISOString();
|
|
574
|
+
dispatchLogTimer = setTimeout(() => {
|
|
575
|
+
void logDispatched();
|
|
576
|
+
}, DISPATCH_LOG_DELAY_MS);
|
|
376
577
|
|
|
377
578
|
// For push: we just need to send to the pre-resolved destination,
|
|
378
579
|
// but if it's invalid, we deactivate it.
|
|
379
580
|
if (task.channel === "push") {
|
|
380
581
|
let lastResult: any = { success: false, error: "No transports" };
|
|
381
582
|
|
|
382
|
-
for (
|
|
583
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
584
|
+
const transport = transports[i];
|
|
585
|
+
// The starting provider's slot was taken above. Failing over to a
|
|
586
|
+
// later one spends that provider's own budget, or skips it if spent.
|
|
587
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
591
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
592
|
+
if (remainingBudgetMs <= 0) {
|
|
593
|
+
this.logger.warn(
|
|
594
|
+
{ taskId: task.taskId, channel: task.channel },
|
|
595
|
+
"dispatch budget exhausted before transport call — aborting to protect lock lease",
|
|
596
|
+
);
|
|
597
|
+
break;
|
|
598
|
+
}
|
|
599
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
600
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
601
|
+
|
|
383
602
|
try {
|
|
384
603
|
const tProv = Date.now();
|
|
385
|
-
const breaker = this.getBreaker(
|
|
604
|
+
const breaker = this.getBreaker(providerKey(task.channel, transports, i));
|
|
386
605
|
|
|
387
606
|
lastResult = await breaker.execute(async () => {
|
|
388
|
-
const timeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
389
607
|
const controller = new AbortController();
|
|
390
608
|
const timeout = setTimeout(() => {
|
|
391
609
|
controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
@@ -403,7 +621,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
403
621
|
}),
|
|
404
622
|
]);
|
|
405
623
|
if (!res.success && !res.invalidToken) {
|
|
406
|
-
throw
|
|
624
|
+
throw providerFailure(res);
|
|
407
625
|
}
|
|
408
626
|
return res;
|
|
409
627
|
} finally {
|
|
@@ -414,11 +632,19 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
414
632
|
(global as any)._telemetry.provider += Date.now() - tProv;
|
|
415
633
|
if (lastResult.success || lastResult.invalidToken) break;
|
|
416
634
|
} catch (err: any) {
|
|
417
|
-
lastResult = { success: false, error: err.message };
|
|
635
|
+
lastResult = { success: false, error: err.message, retryable: err.retryable };
|
|
418
636
|
}
|
|
419
637
|
}
|
|
420
638
|
|
|
421
639
|
if (lastResult.invalidToken) {
|
|
640
|
+
publishPromises.push(
|
|
641
|
+
this.outboxUpdateProcessor.add({
|
|
642
|
+
taskId: task.taskId,
|
|
643
|
+
channel: task.channel,
|
|
644
|
+
destination: task.destination,
|
|
645
|
+
providerMessageId: null,
|
|
646
|
+
}),
|
|
647
|
+
);
|
|
422
648
|
await this.contactRepo.deactivate(
|
|
423
649
|
task.projectId,
|
|
424
650
|
task.recipientId,
|
|
@@ -442,7 +668,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
442
668
|
this.eventProcessor.add(
|
|
443
669
|
buildStreamEvent(
|
|
444
670
|
"notification.failed",
|
|
445
|
-
{
|
|
671
|
+
outcome({
|
|
446
672
|
projectId: task.projectId,
|
|
447
673
|
taskId: task.taskId,
|
|
448
674
|
enrichedEventId: task.enrichedEventId,
|
|
@@ -451,12 +677,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
451
677
|
failureReason: "invalid_token",
|
|
452
678
|
failureCode: "push_failure",
|
|
453
679
|
retryable: false,
|
|
454
|
-
attempt,
|
|
680
|
+
attempt: attemptNo,
|
|
455
681
|
templateId: task.templateId,
|
|
456
|
-
workflowInstanceId:
|
|
457
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
682
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
458
683
|
campaignId: task.campaignId,
|
|
459
|
-
},
|
|
684
|
+
}),
|
|
460
685
|
"delivery",
|
|
461
686
|
event.metadata.traceId,
|
|
462
687
|
),
|
|
@@ -497,7 +722,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
497
722
|
this.eventProcessor.add(
|
|
498
723
|
buildStreamEvent(
|
|
499
724
|
"notification.delivered",
|
|
500
|
-
{
|
|
725
|
+
outcome({
|
|
501
726
|
projectId: task.projectId,
|
|
502
727
|
taskId: task.taskId,
|
|
503
728
|
enrichedEventId: task.enrichedEventId,
|
|
@@ -505,16 +730,15 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
505
730
|
deliveredAt: new Date().toISOString(),
|
|
506
731
|
providerMessageId,
|
|
507
732
|
templateId: task.templateId,
|
|
508
|
-
workflowInstanceId:
|
|
509
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
733
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
510
734
|
campaignId: task.campaignId,
|
|
511
|
-
},
|
|
735
|
+
}),
|
|
512
736
|
"delivery",
|
|
513
737
|
event.metadata.traceId,
|
|
514
738
|
),
|
|
515
739
|
),
|
|
516
740
|
);
|
|
517
|
-
} else {
|
|
741
|
+
} else if (!(await retryLater(lastResult))) {
|
|
518
742
|
this.logger.warn(
|
|
519
743
|
{ taskId: task.taskId, error: lastResult.error },
|
|
520
744
|
"push delivery failed",
|
|
@@ -528,11 +752,20 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
528
752
|
);
|
|
529
753
|
metrics.deliveryFailed.inc({ channel: task.channel, reason: "push_error" });
|
|
530
754
|
|
|
755
|
+
publishPromises.push(
|
|
756
|
+
this.outboxUpdateProcessor.add({
|
|
757
|
+
taskId: task.taskId,
|
|
758
|
+
channel: task.channel,
|
|
759
|
+
destination: task.destination,
|
|
760
|
+
providerMessageId: null,
|
|
761
|
+
}),
|
|
762
|
+
);
|
|
763
|
+
|
|
531
764
|
publishPromises.push(
|
|
532
765
|
this.eventProcessor.add(
|
|
533
766
|
buildStreamEvent(
|
|
534
767
|
"notification.failed",
|
|
535
|
-
{
|
|
768
|
+
outcome({
|
|
536
769
|
projectId: task.projectId,
|
|
537
770
|
taskId: task.taskId,
|
|
538
771
|
enrichedEventId: task.enrichedEventId,
|
|
@@ -541,12 +774,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
541
774
|
failureReason: lastResult.error ?? "push delivery failed",
|
|
542
775
|
failureCode: "push_failure",
|
|
543
776
|
retryable: false,
|
|
544
|
-
attempt,
|
|
777
|
+
attempt: attemptNo,
|
|
545
778
|
templateId: task.templateId,
|
|
546
|
-
workflowInstanceId:
|
|
547
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
779
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
548
780
|
campaignId: task.campaignId,
|
|
549
|
-
},
|
|
781
|
+
}),
|
|
550
782
|
"delivery",
|
|
551
783
|
event.metadata.traceId,
|
|
552
784
|
),
|
|
@@ -561,13 +793,30 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
561
793
|
} else {
|
|
562
794
|
// Other channels (email, sms, webhook, whatsapp) use the pre-resolved destination
|
|
563
795
|
let result: any = { success: false, error: "No transports" };
|
|
564
|
-
for (
|
|
796
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
797
|
+
const transport = transports[i];
|
|
798
|
+
// The starting provider's slot was taken above. Failing over to a
|
|
799
|
+
// later one spends that provider's own budget, or skips it if spent.
|
|
800
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
|
|
801
|
+
continue;
|
|
802
|
+
}
|
|
803
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
804
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
805
|
+
if (remainingBudgetMs <= 0) {
|
|
806
|
+
this.logger.warn(
|
|
807
|
+
{ taskId: task.taskId, channel: task.channel },
|
|
808
|
+
"dispatch budget exhausted before transport call — aborting to protect lock lease",
|
|
809
|
+
);
|
|
810
|
+
break;
|
|
811
|
+
}
|
|
812
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
813
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
814
|
+
|
|
565
815
|
try {
|
|
566
816
|
const tProv = Date.now();
|
|
567
|
-
const breaker = this.getBreaker(
|
|
817
|
+
const breaker = this.getBreaker(providerKey(task.channel, transports, i));
|
|
568
818
|
|
|
569
819
|
result = await breaker.execute(async () => {
|
|
570
|
-
const timeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
571
820
|
const controller = new AbortController();
|
|
572
821
|
const timeout = setTimeout(() => {
|
|
573
822
|
controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
@@ -585,7 +834,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
585
834
|
}),
|
|
586
835
|
]);
|
|
587
836
|
if (!res.success) {
|
|
588
|
-
throw
|
|
837
|
+
throw providerFailure(res);
|
|
589
838
|
}
|
|
590
839
|
return res;
|
|
591
840
|
} finally {
|
|
@@ -596,7 +845,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
596
845
|
(global as any)._telemetry.provider += Date.now() - tProv;
|
|
597
846
|
if (result.success) break;
|
|
598
847
|
} catch (err: any) {
|
|
599
|
-
result = { success: false, error: err.message };
|
|
848
|
+
result = { success: false, error: err.message, retryable: err.retryable };
|
|
600
849
|
}
|
|
601
850
|
}
|
|
602
851
|
|
|
@@ -626,7 +875,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
626
875
|
this.eventProcessor.add(
|
|
627
876
|
buildStreamEvent(
|
|
628
877
|
"notification.delivered",
|
|
629
|
-
{
|
|
878
|
+
outcome({
|
|
630
879
|
projectId: task.projectId,
|
|
631
880
|
taskId: task.taskId,
|
|
632
881
|
enrichedEventId: task.enrichedEventId,
|
|
@@ -634,21 +883,29 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
634
883
|
deliveredAt: new Date().toISOString(),
|
|
635
884
|
providerMessageId,
|
|
636
885
|
templateId: task.templateId,
|
|
637
|
-
workflowInstanceId:
|
|
638
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
886
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
639
887
|
campaignId: task.campaignId,
|
|
640
|
-
},
|
|
888
|
+
}),
|
|
641
889
|
"delivery",
|
|
642
890
|
event.metadata.traceId,
|
|
643
891
|
),
|
|
644
892
|
),
|
|
645
893
|
);
|
|
646
|
-
} else {
|
|
894
|
+
} else if (!(await retryLater(result))) {
|
|
895
|
+
publishPromises.push(
|
|
896
|
+
this.outboxUpdateProcessor.add({
|
|
897
|
+
taskId: task.taskId,
|
|
898
|
+
channel: task.channel,
|
|
899
|
+
destination: task.destination,
|
|
900
|
+
providerMessageId: null,
|
|
901
|
+
}),
|
|
902
|
+
);
|
|
903
|
+
|
|
647
904
|
publishPromises.push(
|
|
648
905
|
this.eventProcessor.add(
|
|
649
906
|
buildStreamEvent(
|
|
650
907
|
"notification.failed",
|
|
651
|
-
{
|
|
908
|
+
outcome({
|
|
652
909
|
projectId: task.projectId,
|
|
653
910
|
taskId: task.taskId,
|
|
654
911
|
enrichedEventId: task.enrichedEventId,
|
|
@@ -656,12 +913,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
656
913
|
failureReason: result.error ?? "delivery failed completely",
|
|
657
914
|
failureCode: "provider_error",
|
|
658
915
|
retryable: false,
|
|
659
|
-
attempt,
|
|
916
|
+
attempt: attemptNo,
|
|
660
917
|
templateId: task.templateId,
|
|
661
|
-
workflowInstanceId:
|
|
662
|
-
event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
|
|
918
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
663
919
|
campaignId: task.campaignId,
|
|
664
|
-
},
|
|
920
|
+
}),
|
|
665
921
|
"delivery",
|
|
666
922
|
event.metadata.traceId,
|
|
667
923
|
),
|
|
@@ -669,7 +925,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
669
925
|
);
|
|
670
926
|
}
|
|
671
927
|
|
|
672
|
-
if (!result.success) {
|
|
928
|
+
if (!result.success && !retryScheduled) {
|
|
673
929
|
this.logger.warn(
|
|
674
930
|
{ taskId: task.taskId, channel: task.channel, error: result.error },
|
|
675
931
|
"delivery failed completely across providers",
|
|
@@ -687,14 +943,17 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
687
943
|
if (!fallbackTriggered) {
|
|
688
944
|
throw new NonRetryableError(result.error ?? "delivery failed");
|
|
689
945
|
}
|
|
690
|
-
} else {
|
|
691
|
-
this.logger.
|
|
946
|
+
} else if (result.success) {
|
|
947
|
+
this.logger.debug(
|
|
692
948
|
{ taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
|
|
693
949
|
"notification delivered",
|
|
694
950
|
);
|
|
695
951
|
}
|
|
696
952
|
}
|
|
697
953
|
|
|
954
|
+
// Every branch above publishes an outcome; this only guards one that doesn't.
|
|
955
|
+
void logDispatched();
|
|
956
|
+
|
|
698
957
|
const tFlushStart = Date.now();
|
|
699
958
|
await Promise.all(publishPromises).catch((err) => {
|
|
700
959
|
this.logger.error(
|
|
@@ -704,7 +963,14 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
704
963
|
});
|
|
705
964
|
const flushTime = Date.now() - tFlushStart;
|
|
706
965
|
|
|
707
|
-
|
|
966
|
+
if (retryScheduled) {
|
|
967
|
+
// Released so the re-queued attempt can take the lease.
|
|
968
|
+
await this.idempotency.unmark(idempotencyKey).catch(() => {});
|
|
969
|
+
} else {
|
|
970
|
+
void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
|
|
971
|
+
this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
|
|
972
|
+
});
|
|
973
|
+
}
|
|
708
974
|
|
|
709
975
|
const t = ((global as any)._telemetry = (global as any)._telemetry || {
|
|
710
976
|
count: 0,
|
|
@@ -714,7 +980,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
714
980
|
ack: 0,
|
|
715
981
|
});
|
|
716
982
|
t.count++;
|
|
717
|
-
t.insert +=
|
|
983
|
+
t.insert += 0;
|
|
718
984
|
t.flush += flushTime;
|
|
719
985
|
|
|
720
986
|
if (t.count % 1000 === 0) {
|
|
@@ -731,6 +997,8 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
731
997
|
t.flushCount = 0;
|
|
732
998
|
}
|
|
733
999
|
} catch (err) {
|
|
1000
|
+
// The attempt happened even though it did not finish; keep it in the log.
|
|
1001
|
+
logDispatched()?.catch(() => {});
|
|
734
1002
|
await this.idempotency.unmark(idempotencyKey).catch(() => {});
|
|
735
1003
|
throw err;
|
|
736
1004
|
}
|
|
@@ -738,21 +1006,22 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
738
1006
|
}
|
|
739
1007
|
|
|
740
1008
|
export async function startDeliveryWorker() {
|
|
741
|
-
logger =
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
const
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
1009
|
+
({ logger, redis, sql, db } = createWorkerRuntime(config, "delivery"));
|
|
1010
|
+
// Delivery spends most of its time waiting on providers, so it needs far
|
|
1011
|
+
// more messages in flight than the CPU-bound stages do.
|
|
1012
|
+
const deliveryConcurrency = config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY;
|
|
1013
|
+
const consumerId = uniqueConsumerId("delivery");
|
|
1014
|
+
({ consumer, pendingScanner } = createStreamConsumer(
|
|
1015
|
+
{ redis, logger },
|
|
1016
|
+
{
|
|
1017
|
+
stream: OUTBOUND_STREAMS,
|
|
1018
|
+
group: CONSUMER_GROUPS.DELIVERY,
|
|
1019
|
+
consumer: consumerId,
|
|
1020
|
+
batchSize: deliveryConcurrency,
|
|
1021
|
+
bufferAcks: true,
|
|
1022
|
+
coalesceMs: 2,
|
|
1023
|
+
},
|
|
1024
|
+
));
|
|
756
1025
|
|
|
757
1026
|
scheduledProducer = new StreamProducer({
|
|
758
1027
|
redis: redis.native,
|
|
@@ -760,30 +1029,15 @@ export async function startDeliveryWorker() {
|
|
|
760
1029
|
logger,
|
|
761
1030
|
});
|
|
762
1031
|
|
|
763
|
-
|
|
764
|
-
redis: redis.native,
|
|
765
|
-
stream: OUTBOUND_STREAMS as unknown as StreamName[],
|
|
766
|
-
group: CONSUMER_GROUPS.DELIVERY,
|
|
767
|
-
consumer: `delivery-${process.pid}`,
|
|
768
|
-
logger,
|
|
769
|
-
});
|
|
770
|
-
|
|
771
|
-
enrichedProducers = {
|
|
772
|
-
critical: new StreamProducer({
|
|
773
|
-
redis: redis.native,
|
|
774
|
-
stream: STREAMS.ENRICHED_CRITICAL,
|
|
775
|
-
logger,
|
|
776
|
-
}),
|
|
777
|
-
normal: new StreamProducer({ redis: redis.native, stream: STREAMS.ENRICHED_NORMAL, logger }),
|
|
778
|
-
low: new StreamProducer({ redis: redis.native, stream: STREAMS.ENRICHED_LOW, logger }),
|
|
779
|
-
};
|
|
1032
|
+
enrichedProducers = createPriorityProducers(redis.native, logger, "ENRICHED");
|
|
780
1033
|
|
|
781
1034
|
const contactRepo = new ContactRepository(db);
|
|
1035
|
+
const templateCache = new TemplateCache(new TemplateRepository(db));
|
|
782
1036
|
|
|
783
1037
|
const idempotency = new IdempotencyGuard({
|
|
784
1038
|
redis: redis.native,
|
|
785
1039
|
keyPrefix: "notif:processed:delivery",
|
|
786
|
-
ttlSeconds:
|
|
1040
|
+
ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
|
|
787
1041
|
});
|
|
788
1042
|
|
|
789
1043
|
// ─── Stage 3: Delivery Worker ───────────────────────────────────────────────
|
|
@@ -804,7 +1058,7 @@ export async function startDeliveryWorker() {
|
|
|
804
1058
|
consumer,
|
|
805
1059
|
pendingScanner,
|
|
806
1060
|
logger,
|
|
807
|
-
concurrency:
|
|
1061
|
+
concurrency: deliveryConcurrency,
|
|
808
1062
|
transportRegistry,
|
|
809
1063
|
idempotency,
|
|
810
1064
|
redis: redis.native,
|
|
@@ -814,8 +1068,14 @@ export async function startDeliveryWorker() {
|
|
|
814
1068
|
eventsProducer,
|
|
815
1069
|
globalEmitter,
|
|
816
1070
|
db,
|
|
1071
|
+
templateCache,
|
|
817
1072
|
});
|
|
818
1073
|
|
|
1074
|
+
// Delivery renders templates now, so it has to hear about edits too.
|
|
1075
|
+
subscriber = redis.native.duplicate({ enableAutoPipelining: false });
|
|
1076
|
+
await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
|
|
1077
|
+
subscriber.on("message", (_channel: string, key: string) => templateCache.invalidateKey(key));
|
|
1078
|
+
|
|
819
1079
|
// ─── Health check interval ──────────────────────────────────────────────────
|
|
820
1080
|
|
|
821
1081
|
healthInterval = startHealthReporter("delivery", worker, redis, logger);
|
|
@@ -827,16 +1087,21 @@ export async function startDeliveryWorker() {
|
|
|
827
1087
|
await worker.start();
|
|
828
1088
|
}
|
|
829
1089
|
|
|
1090
|
+
export function getDeliveryWorker(): DeliveryWorker | undefined {
|
|
1091
|
+
return worker as DeliveryWorker | undefined;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
830
1094
|
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
831
1095
|
|
|
832
1096
|
export async function stopDeliveryWorker(): Promise<void> {
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
1097
|
+
await shutdownWorker("delivery", {
|
|
1098
|
+
logger,
|
|
1099
|
+
timers: [healthInterval],
|
|
1100
|
+
subscriber,
|
|
1101
|
+
worker,
|
|
1102
|
+
sql,
|
|
1103
|
+
redis,
|
|
1104
|
+
});
|
|
1105
|
+
healthInterval = null;
|
|
1106
|
+
subscriber = null;
|
|
842
1107
|
}
|