notifkit 0.1.0

package/dist/main-ClEeP5qw.mjs

@@ -0,0 +1,629 @@
+import { J as PendingMessageScanner, Jt as buildStreamEvent, L as CircuitBreaker, Q as metrics, Qt as OUTBOUND_STREAMS, R as BatchProcessor, W as getPriorityBucket, X as StreamProducer, Y as StreamConsumer, Yt as CONSUMER_GROUPS, an as registry, c as BaseWorker, dn as baseConfigSchema, en as STREAMS, et as createLogger, it as createDatabase, k as RedisClient, l as NonRetryableError, mn as parseConfig, ot as deliveryOutbox, pn as loadEnv, q as globalEmitter, rt as IdempotencyGuard, u as startHealthReporter, ut as scheduledPayloads, x as ContactRepository } from "./src-DrSN2wCg.mjs";
+import { transportRegistry } from "./index.mjs";
+//#region src/services/delivery/throttle.ts
+const LUA_THROTTLE = `
+  local key = KEYS[1]
+  local now = tonumber(ARGV[1])
+  local windowSeconds = tonumber(ARGV[2])
+  local limit = tonumber(ARGV[3])
+  local member = ARGV[4]
+
+  local clearBefore = now - (windowSeconds * 1000)
+  
+  -- Cleanup expired scores
+  redis.call('ZREMRANGEBYSCORE', key, 0, clearBefore)
+  
+  -- Get current count
+  local count = redis.call('ZCARD', key)
+  
+  if count >= limit then
+    -- Find the oldest score
+    local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES')
+    if oldest and oldest[2] then
+      return {0, tonumber(oldest[2])}
+    end
+    return {0, now}
+  end
+  
+  -- Add new request
+  redis.call('ZADD', key, now, member)
+  redis.call('EXPIRE', key, windowSeconds * 2)
+  return {1, 0}
+`;
+async function throttleProvider(redis, channel, config, logger) {
+	const key = `rate-limit:provider:${channel}`;
+	const now = Date.now();
+	const zmember = `${now}:${Math.random()}`;
+	const result = await redis.eval(LUA_THROTTLE, 1, key, now.toString(), config.windowSeconds.toString(), config.limit.toString(), zmember);
+	const allowed = result[0] === 1;
+	const oldestTimestamp = result[1];
+	let retryAfterMs = 0;
+	if (!allowed) {
+		retryAfterMs = Math.max(0, oldestTimestamp + config.windowSeconds * 1e3 - now);
+		logger.warn({
+			channel,
+			limit: config.limit,
+			windowSeconds: config.windowSeconds,
+			retryAfterMs
+		}, "Provider rate limit hit — task must be rescheduled");
+	}
+	return {
+		allowed,
+		retryAfterMs
+	};
+}
+//#endregion
+//#region src/services/delivery/main.ts
+const deliveryConfigSchema = baseConfigSchema.extend({});
+loadEnv();
+const config = parseConfig(deliveryConfigSchema, process.env);
+let logger;
+let redis;
+let sql;
+let db;
+let consumer;
+let pendingScanner;
+let worker;
+let scheduledProducer;
+let enrichedProducers;
+let healthInterval = null;
+global._telemetry = global._telemetry || {
+	count: 0,
+	insert: 0,
+	provider: 0,
+	flush: 0,
+	ack: 0,
+	dequeue: 0,
+	dbupdate: 0,
+	flushCount: 0
+};
+var DeliveryWorker = class extends BaseWorker {
+	transportRegistry;
+	idempotency;
+	redisCli;
+	scheduledProducer;
+	enrichedProducers;
+	contactRepo;
+	eventsProducer;
+	globalEmitter;
+	db;
+	eventProcessor;
+	outboxUpdateProcessor;
+	outboxInsertProcessor;
+	breakers = /* @__PURE__ */ new Map();
+	constructor(options) {
+		super(options);
+		this.transportRegistry = options.transportRegistry;
+		this.idempotency = options.idempotency;
+		this.redisCli = options.redis;
+		this.scheduledProducer = options.scheduledProducer;
+		this.enrichedProducers = options.enrichedProducers;
+		this.contactRepo = options.contactRepo;
+		this.eventsProducer = options.eventsProducer;
+		this.globalEmitter = options.globalEmitter;
+		this.db = options.db;
+		this.eventProcessor = new BatchProcessor(1e3, 100, async (events) => {
+			await this.eventsProducer.publishBatch(events);
+			return events.map(() => void 0);
+		});
+		this.outboxUpdateProcessor = new BatchProcessor(500, 100, async (updates) => {
+			const { sql } = await import("drizzle-orm");
+			const values = updates.map((update) => ({
+				taskId: update.taskId,
+				channel: update.channel,
+				destination: update.destination,
+				providerMessageId: update.providerMessageId
+			}));
+			const tDbUpdateStart = Date.now();
+			await this.db.insert(deliveryOutbox).values(values).onConflictDoUpdate({
+				target: [
+					deliveryOutbox.taskId,
+					deliveryOutbox.channel,
+					deliveryOutbox.destination
+				],
+				set: { providerMessageId: sql`EXCLUDED.provider_message_id` }
+			}).catch((e) => this.logger.error({ err: e }, "background update failed"));
+			global._telemetry.dbupdate += Date.now() - tDbUpdateStart;
+			global._telemetry.flushCount++;
+			return updates.map(() => void 0);
+		});
+		this.outboxInsertProcessor = new BatchProcessor(500, 10, async (tasks) => {
+			const values = tasks.map((task) => ({
+				taskId: task.taskId,
+				channel: task.channel,
+				destination: task.destination
+			}));
+			await this.db.insert(deliveryOutbox).values(values).onConflictDoNothing();
+			return tasks.map(() => true);
+		});
+	}
+	async stop() {
+		await Promise.all([
+			this.eventProcessor.flush(),
+			this.outboxUpdateProcessor.flush(),
+			this.outboxInsertProcessor.flush()
+		]);
+		await super.stop();
+	}
+	getBreaker(name) {
+		let breaker = this.breakers.get(name);
+		if (!breaker) {
+			breaker = new CircuitBreaker({
+				failureThreshold: 5,
+				resetTimeoutMs: 3e4
+			});
+			this.breakers.set(name, breaker);
+		}
+		return breaker;
+	}
+	async process(message, attempt = 1) {
+		const { event } = message;
+		const publishPromises = [];
+		const payloadResult = registry.safeParsePayload("notification.dispatched", event.payload);
+		if (!payloadResult.success) {
+			this.logger.warn({
+				messageId: message.id,
+				issues: payloadResult.error.issues
+			}, "invalid notification.dispatched payload — skipping");
+			return;
+		}
+		const task = payloadResult.data;
+		const fallbackToNextChannel = async (reason) => {
+			if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
+				const nextChannel = task.fallbackChain[0];
+				const remainingChain = task.fallbackChain.slice(1);
+				const fallbackPayload = {
+					projectId: task.projectId,
+					rawEventId: task.enrichedEventId,
+					recipientId: task.recipientId,
+					channel: nextChannel,
+					priority: task.priority,
+					templateId: task.templateId,
+					templateVariables: task.templateVariables,
+					aiPrompts: task.aiPrompts,
+					recipient: task.recipient,
+					scheduledAt: void 0,
+					fallbackChain: remainingChain.length > 0 ? remainingChain : void 0
+				};
+				const p = getPriorityBucket(task.priority);
+				await (this.enrichedProducers[p] ?? this.enrichedProducers["normal"]).publish(buildStreamEvent("notification.enriched", fallbackPayload, "delivery", event.metadata.traceId));
+				this.logger.info({
+					taskId: task.taskId,
+					reason,
+					nextChannel,
+					traceId: event.metadata.traceId
+				}, "Delivery failed completely, rolling over to next channel in fallback chain");
+				return true;
+			}
+			return false;
+		};
+		const transports = this.transportRegistry.getAll(task.channel);
+		if (transports.length === 0) {
+			this.logger.warn({
+				taskId: task.taskId,
+				channel: task.channel
+			}, "no transport registered for channel — dropping");
+			if (!await fallbackToNextChannel("no_transport")) {
+				this.globalEmitter.emit("delivery:failed", task.taskId, "no transport", task.channel, task.projectId);
+				await this.eventProcessor.add(buildStreamEvent("notification.failed", {
+					projectId: task.projectId,
+					taskId: task.taskId,
+					enrichedEventId: task.enrichedEventId,
+					recipientId: task.recipientId,
+					channel: task.channel,
+					failureReason: "no transport registered for channel",
+					failureCode: "no_transport",
+					retryable: false,
+					attempt,
+					templateId: task.templateId,
+					workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+					campaignId: task.campaignId
+				}, "delivery", event.metadata.traceId));
+			}
+			return;
+		}
+		const limitConfig = transports[0].limits;
+		if (limitConfig) {
+			const { allowed, retryAfterMs } = await throttleProvider(this.redisCli, task.channel, limitConfig, this.logger);
+			if (!allowed) {
+				task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
+				const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
+				if (task.throttleAttemptCount > maxAttempts) {
+					this.logger.warn({
+						messageId: message.id,
+						taskId: task.taskId,
+						attempts: task.throttleAttemptCount
+					}, "provider rate limit max attempts exceeded");
+					if (!await fallbackToNextChannel("provider_throttle_exceeded")) {
+						this.globalEmitter.emit("delivery:failed", task.taskId, "provider throttle exceeded", task.channel, task.projectId);
+						await this.eventProcessor.add(buildStreamEvent("notification.failed", {
+							projectId: task.projectId,
+							taskId: task.taskId,
+							enrichedEventId: task.enrichedEventId,
+							recipientId: task.recipientId,
+							channel: task.channel,
+							failureReason: "provider rate limit max attempts exceeded",
+							failureCode: "provider_throttle_exceeded",
+							retryable: false,
+							attempt,
+							templateId: task.templateId,
+							workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+							campaignId: task.campaignId
+						}, "delivery", event.metadata.traceId));
+					}
+					return;
+				}
+				const { sql } = await import("drizzle-orm");
+				await this.db.insert(scheduledPayloads).values({
+					taskId: task.taskId,
+					payload: task
+				}).onConflictDoUpdate({
+					target: scheduledPayloads.taskId,
+					set: { payload: sql`EXCLUDED.payload` }
+				});
+				await this.scheduledProducer.publish(buildStreamEvent("notification.scheduled", {
+					projectId: task.projectId,
+					enrichedEventId: task.enrichedEventId,
+					taskId: task.taskId,
+					scheduledAt: new Date(Date.now() + retryAfterMs).toISOString(),
+					throttleAttemptCount: task.throttleAttemptCount
+				}, "delivery", `${task.taskId}:throttle:${Date.now()}`));
+				return;
+			}
+		}
+		const tInsertStart = Date.now();
+		const idempotencyKey = task.taskId;
+		if (!await this.idempotency.checkAndMark(idempotencyKey, 60)) {
+			this.logger.info({
+				messageId: message.id,
+				taskId: task.taskId,
+				channel: task.channel,
+				attempt
+			}, "duplicate delivery — skipping");
+			return;
+		}
+		try {
+			await this.outboxInsertProcessor.add(task);
+			const insertTime = Date.now() - tInsertStart;
+			publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.dispatched", {
+				projectId: task.projectId,
+				taskId: task.taskId,
+				enrichedEventId: task.enrichedEventId,
+				recipientId: task.recipientId,
+				channel: task.channel,
+				templateId: task.templateId,
+				attempt,
+				workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+				campaignId: task.campaignId
+			}, "delivery", event.metadata.traceId)));
+			if (task.channel === "push") {
+				let lastResult = {
+					success: false,
+					error: "No transports"
+				};
+				for (const transport of transports) try {
+					const tProv = Date.now();
+					lastResult = await this.getBreaker(`${task.channel}:${transport.constructor.name}`).execute(async () => {
+						const timeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
+						const controller = new AbortController();
+						const timeout = setTimeout(() => {
+							controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
+						}, timeoutMs);
+						try {
+							task.signal = controller.signal;
+							const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
+								if (controller.signal.aborted) return reject(controller.signal.reason);
+								controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
+							})]);
+							if (!res.success && !res.invalidToken) throw new Error(res.error ?? "Transport failed");
+							return res;
+						} finally {
+							clearTimeout(timeout);
+						}
+					});
+					global._telemetry.provider += Date.now() - tProv;
+					if (lastResult.success || lastResult.invalidToken) break;
+				} catch (err) {
+					lastResult = {
+						success: false,
+						error: err.message
+					};
+				}
+				if (lastResult.invalidToken) {
+					await this.contactRepo.deactivate(task.projectId, task.recipientId, "push", task.destination);
+					this.logger.info({ token: task.destination }, "deactivated invalid push token");
+					this.globalEmitter.emit("delivery:failed", task.taskId, "invalidToken", task.channel, task.projectId);
+					metrics.deliveryFailed.inc({
+						channel: task.channel,
+						reason: "invalid_token"
+					});
+					publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", {
+						projectId: task.projectId,
+						taskId: task.taskId,
+						enrichedEventId: task.enrichedEventId,
+						recipientId: task.recipientId,
+						channel: task.channel,
+						failureReason: "invalid_token",
+						failureCode: "push_failure",
+						retryable: false,
+						attempt,
+						templateId: task.templateId,
+						workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+						campaignId: task.campaignId
+					}, "delivery", event.metadata.traceId)));
+					if (!await fallbackToNextChannel("invalid_token")) throw new NonRetryableError("Push delivery failed: invalid token");
+				} else if (lastResult.success) {
+					const providerMessageId = lastResult.providerMessageId || "push-success";
+					publishPromises.push(this.outboxUpdateProcessor.add({
+						taskId: task.taskId,
+						channel: task.channel,
+						destination: task.destination,
+						providerMessageId
+					}));
+					this.logger.debug({
+						taskId: task.taskId,
+						messageId: providerMessageId
+					}, "push delivered");
+					this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
+					metrics.deliverySuccess.inc({ channel: task.channel });
+					publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", {
+						projectId: task.projectId,
+						taskId: task.taskId,
+						enrichedEventId: task.enrichedEventId,
+						channel: task.channel,
+						deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
+						providerMessageId,
+						templateId: task.templateId,
+						workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+						campaignId: task.campaignId
+					}, "delivery", event.metadata.traceId)));
+				} else {
+					this.logger.warn({
+						taskId: task.taskId,
+						error: lastResult.error
+					}, "push delivery failed");
+					this.globalEmitter.emit("delivery:failed", task.taskId, lastResult.error ?? "push delivery failed", task.channel, task.projectId);
+					metrics.deliveryFailed.inc({
+						channel: task.channel,
+						reason: "push_error"
+					});
+					publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", {
+						projectId: task.projectId,
+						taskId: task.taskId,
+						enrichedEventId: task.enrichedEventId,
+						recipientId: task.recipientId,
+						channel: task.channel,
+						failureReason: lastResult.error ?? "push delivery failed",
+						failureCode: "push_failure",
+						retryable: false,
+						attempt,
+						templateId: task.templateId,
+						workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+						campaignId: task.campaignId
+					}, "delivery", event.metadata.traceId)));
+					if (!await fallbackToNextChannel("push_delivery_failed")) throw new NonRetryableError(lastResult.error ?? "Push delivery failed");
+				}
+			} else {
+				let result = {
+					success: false,
+					error: "No transports"
+				};
+				for (const transport of transports) try {
+					const tProv = Date.now();
+					result = await this.getBreaker(`${task.channel}:${transport.constructor.name}`).execute(async () => {
+						const timeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
+						const controller = new AbortController();
+						const timeout = setTimeout(() => {
+							controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
+						}, timeoutMs);
+						try {
+							task.signal = controller.signal;
+							const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
+								if (controller.signal.aborted) return reject(controller.signal.reason);
+								controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
+							})]);
+							if (!res.success) throw new Error(res.error ?? "Transport failed");
+							return res;
+						} finally {
+							clearTimeout(timeout);
+						}
+					});
+					global._telemetry.provider += Date.now() - tProv;
+					if (result.success) break;
+				} catch (err) {
+					result = {
+						success: false,
+						error: err.message
+					};
+				}
+				if (result.success) {
+					const providerMessageId = result.providerMessageId || "success";
+					publishPromises.push(this.outboxUpdateProcessor.add({
+						taskId: task.taskId,
+						channel: task.channel,
+						destination: task.destination,
+						providerMessageId
+					}));
+					this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
+					metrics.deliverySuccess.inc({ channel: task.channel });
+					publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", {
+						projectId: task.projectId,
+						taskId: task.taskId,
+						enrichedEventId: task.enrichedEventId,
+						channel: task.channel,
+						deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
+						providerMessageId,
+						templateId: task.templateId,
+						workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+						campaignId: task.campaignId
+					}, "delivery", event.metadata.traceId)));
+				} else publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", {
+					projectId: task.projectId,
+					taskId: task.taskId,
+					enrichedEventId: task.enrichedEventId,
+					channel: task.channel,
+					failureReason: result.error ?? "delivery failed completely",
+					failureCode: "provider_error",
+					retryable: false,
+					attempt,
+					templateId: task.templateId,
+					workflowInstanceId: event.metadata.source === "workflow" ? event.metadata.traceId : void 0,
+					campaignId: task.campaignId
+				}, "delivery", event.metadata.traceId)));
+				if (!result.success) {
+					this.logger.warn({
+						taskId: task.taskId,
+						channel: task.channel,
+						error: result.error
+					}, "delivery failed completely across providers");
+					this.globalEmitter.emit("delivery:failed", task.taskId, result.error ?? "delivery failed completely", task.channel, task.projectId);
+					metrics.deliveryFailed.inc({
+						channel: task.channel,
+						reason: "provider_error"
+					});
+					if (!await fallbackToNextChannel("all_providers_failed")) throw new NonRetryableError(result.error ?? "delivery failed");
+				} else this.logger.info({
+					taskId: task.taskId,
+					channel: task.channel,
+					messageId: result.providerMessageId
+				}, "notification delivered");
+			}
+			const tFlushStart = Date.now();
+			await Promise.all(publishPromises).catch((err) => {
+				this.logger.error({
+					err,
+					taskId: task.taskId
+				}, "failed to publish post-dispatch events, swallowing error to prevent duplicate delivery");
+			});
+			const flushTime = Date.now() - tFlushStart;
+			await this.idempotency.markProcessed(idempotencyKey);
+			const t = global._telemetry = global._telemetry || {
+				count: 0,
+				insert: 0,
+				provider: 0,
+				flush: 0,
+				ack: 0
+			};
+			t.count++;
+			t.insert += insertTime;
+			t.flush += flushTime;
+			if (t.count % 1e3 === 0) {
+				this.logger.debug(`[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1e3}ms, DB Insert: ${t.insert / 1e3}ms, Provider: ${t.provider / 1e3}ms, Wait for Flush: ${t.flush / 1e3}ms, Ack: ${t.ack / 1e3}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`);
+				t.count = 0;
+				t.insert = 0;
+				t.provider = 0;
+				t.flush = 0;
+				t.dequeue = 0;
+				t.ack = 0;
+				t.dbupdate = 0;
+				t.flushCount = 0;
+			}
+		} catch (err) {
+			await this.idempotency.unmark(idempotencyKey).catch(() => {});
+			throw err;
+		}
+	}
+};
+async function startDeliveryWorker() {
+	logger = createLogger({
+		name: "delivery",
+		level: config.LOG_LEVEL
+	});
+	redis = new RedisClient({
+		url: config.REDIS_URL,
+		name: "delivery",
+		logger
+	});
+	const dbData = createDatabase({
+		url: config.DATABASE_URL,
+		applicationName: "delivery",
+		logger
+	});
+	sql = dbData.sql;
+	db = dbData.db;
+	consumer = new StreamConsumer({
+		redis: redis.native,
+		stream: OUTBOUND_STREAMS,
+		group: CONSUMER_GROUPS.DELIVERY,
+		consumer: `delivery-${process.pid}`,
+		dlqStream: STREAMS.DEAD_LETTER,
+		batchSize: config.WORKER_CONCURRENCY,
+		logger
+	});
+	scheduledProducer = new StreamProducer({
+		redis: redis.native,
+		stream: STREAMS.SCHEDULED,
+		logger
+	});
+	pendingScanner = new PendingMessageScanner({
+		redis: redis.native,
+		stream: OUTBOUND_STREAMS,
+		group: CONSUMER_GROUPS.DELIVERY,
+		consumer: `delivery-${process.pid}`,
+		logger
+	});
+	enrichedProducers = {
+		critical: new StreamProducer({
+			redis: redis.native,
+			stream: STREAMS.ENRICHED_CRITICAL,
+			logger
+		}),
+		normal: new StreamProducer({
+			redis: redis.native,
+			stream: STREAMS.ENRICHED_NORMAL,
+			logger
+		}),
+		low: new StreamProducer({
+			redis: redis.native,
+			stream: STREAMS.ENRICHED_LOW,
+			logger
+		})
+	};
+	const contactRepo = new ContactRepository(db);
+	const idempotency = new IdempotencyGuard({
+		redis: redis.native,
+		keyPrefix: "notif:processed:delivery",
+		ttlSeconds: 86400
+	});
+	const eventsProducer = new StreamProducer({
+		redis: redis.native,
+		stream: STREAMS.EVENTS_INBOUND,
+		logger
+	});
+	worker = new DeliveryWorker({
+		consumer,
+		pendingScanner,
+		logger,
+		concurrency: config.WORKER_CONCURRENCY,
+		transportRegistry,
+		idempotency,
+		redis: redis.native,
+		scheduledProducer,
+		enrichedProducers,
+		contactRepo,
+		eventsProducer,
+		globalEmitter,
+		db
+	});
+	healthInterval = startHealthReporter("delivery", worker, redis, logger);
+	logger.info({
+		env: config.NODE_ENV,
+		channels: transportRegistry.registeredChannels()
+	}, "delivery starting");
+	await worker.start();
+}
+async function stopDeliveryWorker() {
+	logger?.info("shutdown initiated");
+	if (healthInterval) {
+		clearInterval(healthInterval);
+		healthInterval = null;
+	}
+	if (worker) await worker.stop();
+	if (sql) await sql.end();
+	if (redis) await redis.disconnect();
+	logger?.info("delivery stopped");
+}
+//#endregion
+export { startDeliveryWorker, stopDeliveryWorker };
+
+//# sourceMappingURL=main-ClEeP5qw.mjs.map