notifkit 0.1.3 → 0.1.4

package/src/services/events/main.ts

@@ -0,0 +1,365 @@
+import { loadEnv, readBaseConfig } from "@/index.js";
+import { createLogger } from "@/index.js";
+import { RedisClient } from "@/index.js";
+import {
+  StreamConsumer,
+  PendingMessageScanner,
+  StreamProducer,
+  type StreamMessage,
+} from "@/index.js";
+import { BaseWorker } from "@/index.js";
+import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/index.js";
+import { type StreamName } from "@/contracts/streams.js";
+import { createDatabase } from "@/db/index.js";
+import { workflowWaiters, workflowSteps, workflowInstances, messageLogs } from "@/db/schema.js";
+import { eq, and, or, isNull, gt } from "drizzle-orm";
+import { type WorkerOptions } from "@/shared/index.js";
+import { startHealthReporter } from "@/workers/index.js";
+
+loadEnv();
+const config = readBaseConfig();
+let logger: ReturnType<typeof createLogger>;
+let redis: RedisClient;
+let sql: any;
+let db: any;
+
+let consumer: StreamConsumer;
+let pendingScanner: PendingMessageScanner;
+let worker: BaseWorker;
+let healthInterval: NodeJS.Timeout | null = null;
+
+let workflowProducer: StreamProducer;
+
+export interface EventWorkerOptions extends WorkerOptions {
+  db: any;
+  workflowProducer: any;
+}
+
+export class EventWorker extends BaseWorker {
+  private readonly dbConn: any;
+  private readonly workflowProducer: any;
+  private messageLogBuffer: { log: any; resolve: () => void; reject: (err: any) => void }[] = [];
+  private eventBuffer: {
+    producer: any;
+    event: any;
+    resolve: () => void;
+    reject: (err: any) => void;
+  }[] = [];
+  private flushInterval: NodeJS.Timeout | null = null;
+  private isFlushing = false;
+
+  constructor(options: EventWorkerOptions) {
+    super(options);
+    this.dbConn = options.db;
+    this.workflowProducer = options.workflowProducer;
+    this.startFlushInterval();
+  }
+
+  private startFlushInterval() {
+    this.flushInterval = setInterval(() => {
+      void this.flushLogs();
+    }, config.LOG_FLUSH_INTERVAL_MS);
+  }
+
+  private async flushLogs() {
+    if (this.isFlushing || (this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0))
+      return;
+    this.isFlushing = true;
+
+    const startTime = Date.now();
+    const batch = this.messageLogBuffer;
+    this.messageLogBuffer = [];
+
+    const events = this.eventBuffer;
+    this.eventBuffer = [];
+
+    try {
+      if (batch.length > 0) {
+        await this.dbConn
+          .insert(messageLogs)
+          .values(batch.map((b) => b.log))
+          .onConflictDoNothing(); // Idempotent insert
+        for (const b of batch) b.resolve();
+
+        this.logger.info(
+          {
+            batchSize: batch.length,
+            latencyMs: Date.now() - startTime,
+          },
+          "flushed delivery logs to database",
+        );
+      }
+
+      if (events.length > 0) {
+        const byProducer = new Map<any, typeof events>();
+        for (const item of events) {
+          if (!byProducer.has(item.producer)) byProducer.set(item.producer, []);
+          byProducer.get(item.producer)!.push(item);
+        }
+        for (const [producer, evts] of byProducer) {
+          await producer.publishBatch(evts.map((e) => e.event));
+          for (const e of evts) e.resolve();
+        }
+      }
+    } catch (error: any) {
+      this.logger.error(
+        {
+          batchSize: batch.length,
+          error: error.message,
+        },
+        "failed to flush delivery logs — dropping batch",
+      );
+      for (const b of batch) b.reject(error);
+      for (const e of events) e.reject(error);
+    } finally {
+      this.isFlushing = false;
+    }
+  }
+
+  override async stop(): Promise<void> {
+    if (this.flushInterval) clearInterval(this.flushInterval);
+    await super.stop();
+    await this.flushLogs();
+  }
+
+  override async process(message: StreamMessage): Promise<void> {
+    const { event } = message;
+    const publishPromises: Promise<void>[] = [];
+
+    if (
+      event.type === "notification.dispatched" ||
+      event.type === "notification.delivered" ||
+      event.type === "notification.failed"
+    ) {
+      const payload = event.payload as any;
+      if (this.messageLogBuffer.length >= config.LOG_BUFFER_MAX_SIZE) {
+        this.logger.warn(
+          { dropped: 1, bufferSize: this.messageLogBuffer.length },
+          "log buffer full — dropping oldest delivery log",
+        );
+        const dropped = this.messageLogBuffer.shift(); // Drop oldest to apply back-pressure/prevent OOM
+        if (dropped) dropped.reject(new Error("Dropped due to buffer overflow"));
+      }
+
+      const status =
+        event.type === "notification.dispatched"
+          ? "dispatched"
+          : event.type === "notification.delivered"
+            ? "delivered"
+            : "failed";
+
+      const kind = event.type === "notification.dispatched" ? "dispatched" : "attempt";
+
+      publishPromises.push(
+        new Promise((resolve, reject) => {
+          this.messageLogBuffer.push({
+            log: {
+              projectId: payload.projectId,
+              taskId: payload.taskId,
+              providerMessageId: payload.providerMessageId || null,
+              channel: payload.channel,
+              attempt: payload.attempt || 1,
+              kind,
+              status,
+              templateId: payload.templateId || null,
+              workflowInstanceId: payload.workflowInstanceId || null,
+              campaignId: payload.campaignId || null,
+            },
+            resolve,
+            reject,
+          });
+        }),
+      );
+      await Promise.all(publishPromises);
+      return;
+    }
+
+    if (event.type !== "event.received") {
+      return;
+    }
+
+    const payload = event.payload as any;
+    const eventName = payload.eventName;
+    const eventData = payload.payload;
+    const projectId = payload.projectId;
+
+    if (!eventName || !projectId) {
+      this.logger.warn("Missing eventName or projectId in payload");
+      return;
+    }
+
+    // Find matching waiters
+    const now = new Date();
+    const waiters = await this.dbConn
+      .select()
+      .from(workflowWaiters)
+      .where(
+        and(
+          eq(workflowWaiters.eventName, eventName),
+          eq(workflowWaiters.projectId, projectId),
+          or(isNull(workflowWaiters.expiresAt), gt(workflowWaiters.expiresAt, now)),
+        ),
+      );
+
+    const data: Record<string, any> = eventData && typeof eventData === "object" ? eventData : {};
+
+    for (const waiter of waiters) {
+      // Check match criteria
+      let matched = true;
+      const criteria = waiter.matchCriteria as Record<string, any>;
+      if (criteria && typeof criteria === "object") {
+        for (const [k, v] of Object.entries(criteria)) {
+          // simple dot notation check for a flat object or 1 level deep
+          let dataValue = data[k];
+          if (k.includes(".")) {
+            const parts = k.split(".");
+            dataValue = data;
+            for (const p of parts) {
+              if (dataValue) dataValue = dataValue[p];
+            }
+          }
+          if (dataValue !== v) {
+            matched = false;
+            break;
+          }
+        }
+      }
+
+      if (matched) {
+        this.logger.info(
+          { instanceId: waiter.instanceId, eventName },
+          "Event matched a sleeping workflow",
+        );
+
+        // The resume event must carry everything WorkflowWorker.process requires
+        // (name, instanceId, projectId) or it is dropped and the instance — whose
+        // waiter we are about to delete — is stranded forever.
+        const instance = (
+          await this.dbConn
+            .select()
+            .from(workflowInstances)
+            .where(eq(workflowInstances.id, waiter.instanceId))
+            .limit(1)
+        )[0];
+
+        if (!instance) {
+          this.logger.warn(
+            { instanceId: waiter.instanceId },
+            "waiter references a missing workflow instance — dropping waiter",
+          );
+          await this.dbConn.delete(workflowWaiters).where(eq(workflowWaiters.id, waiter.id));
+          continue;
+        }
+
+        // Find the step that is waiting
+        const steps = await this.dbConn
+          .select()
+          .from(workflowSteps)
+          .where(
+            and(
+              eq(workflowSteps.instanceId, waiter.instanceId),
+              eq(workflowSteps.action, "waitForEvent"),
+            ),
+          );
+
+        // Get the latest one that has no output
+        const pendingStep = steps.find((s: any) => s.output === null);
+
+        if (pendingStep) {
+          // Update the step with the event payload
+          await this.dbConn
+            .update(workflowSteps)
+            .set({ output: data })
+            .where(eq(workflowSteps.id, pendingStep.id));
+        }
+
+        // Wake up workflow. Publish BEFORE deleting the waiter: a crash after the
+        // delete but before the publish would leave the instance unrecoverable,
+        // whereas a duplicate resume is idempotent (the instance lock and the
+        // replayed step outputs both absorb it).
+        publishPromises.push(
+          new Promise((resolve, reject) => {
+            this.eventBuffer.push({
+              producer: this.workflowProducer,
+              event: buildStreamEvent(
+                "workflow.resumed",
+                {
+                  projectId: instance.projectId,
+                  instanceId: waiter.instanceId,
+                  name: instance.name,
+                  input: instance.input ?? {},
+                  reason: "event_matched",
+                },
+                "events",
+                event.metadata.traceId,
+              ),
+              resolve,
+              reject,
+            });
+          }),
+        );
+
+        // Delete the waiter
+        await this.dbConn.delete(workflowWaiters).where(eq(workflowWaiters.id, waiter.id));
+
+        // The timer ZSET entry is left to expire: the poll loop re-checks the
+        // instance status and drops the entry if it is no longer pending.
+      }
+    }
+
+    await Promise.all(publishPromises);
+  }
+}
+
+export async function startEventWorker() {
+  logger = createLogger({ name: "event-worker", level: config.LOG_LEVEL });
+  redis = new RedisClient({ url: config.REDIS_URL, name: "events", logger });
+  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "events", logger });
+  sql = dbData.sql;
+  db = dbData.db;
+  workflowProducer = new StreamProducer({
+    redis: redis.native,
+    stream: STREAMS.WORKFLOW_INBOUND,
+    logger,
+  });
+  consumer = new StreamConsumer({
+    redis: redis.native,
+    stream: STREAMS.EVENTS_INBOUND as StreamName,
+    group: CONSUMER_GROUPS.EVENTS as any,
+    consumer: `events-${process.pid}`,
+    dlqStream: STREAMS.DEAD_LETTER,
+    batchSize: config.WORKER_CONCURRENCY,
+    logger,
+  });
+
+  pendingScanner = new PendingMessageScanner({
+    redis: redis.native,
+    stream: STREAMS.EVENTS_INBOUND as StreamName,
+    group: CONSUMER_GROUPS.EVENTS as any,
+    consumer: `events-${process.pid}`,
+    logger,
+  });
+
+  worker = new EventWorker({
+    consumer,
+    pendingScanner,
+    logger,
+    concurrency: config.WORKER_CONCURRENCY,
+    db,
+    workflowProducer,
+  });
+
+  healthInterval = startHealthReporter("events", worker, redis, logger);
+
+  logger.info("event worker starting");
+  await worker.start();
+}
+
+export async function stopEventWorker(): Promise<void> {
+  logger?.info("shutdown initiated");
+  if (healthInterval) clearInterval(healthInterval);
+  if (worker) await worker.stop();
+  if (sql) await sql.end();
+  if (redis) await redis.disconnect();
+  logger?.info("event worker stopped");
+}

package/src/services/scheduler/main.ts

@@ -0,0 +1,319 @@
+import { randomUUID } from "node:crypto";
+import { loadEnv, readBaseConfig } from "@/index.js";
+import { createLogger } from "@/index.js";
+import { RedisClient, type Redis } from "@/index.js";
+import {
+  StreamConsumer,
+  PendingMessageScanner,
+  StreamProducer,
+  type StreamMessage,
+} from "@/index.js";
+import { BaseWorker } from "@/index.js";
+import {
+  STREAMS,
+  CONSUMER_GROUPS,
+  registry,
+  buildStreamEvent,
+  type NotificationScheduledPayload,
+} from "@/index.js";
+import { getPriorityBucket, type WorkerOptions, LUA_SCHEDULER_POLL } from "@/shared/index.js";
+import { createDatabase } from "@/db/index.js";
+import { scheduledPayloads } from "@/db/schema.js";
+import { inArray } from "drizzle-orm";
+import { startHealthReporter } from "@/workers/index.js";
+// ─── Bootstrap ─────────────────────────────────────────────────────────────
+
+loadEnv();
+const config = readBaseConfig();
+
+let logger: ReturnType<typeof createLogger>;
+let redis: RedisClient;
+let sql: any;
+let db: any;
+
+let consumer: StreamConsumer;
+let pendingScanner: PendingMessageScanner;
+let worker: BaseWorker;
+let healthInterval: NodeJS.Timeout | null = null;
+let pollTimeout: ReturnType<typeof setTimeout> | null = null;
+let isPolling = false;
+
+export interface SchedulerWorkerOptions extends WorkerOptions {
+  registry: any;
+  redis: Redis;
+}
+
+export class SchedulerWorker extends BaseWorker {
+  private readonly registry: any;
+  private readonly redisCli: Redis;
+
+  constructor(options: SchedulerWorkerOptions) {
+    super(options);
+    this.registry = options.registry;
+    this.redisCli = options.redis;
+  }
+  async process(message: StreamMessage): Promise<void> {
+    const { event } = message;
+
+    const payloadResult = this.registry.safeParsePayload("notification.scheduled", event.payload);
+    if (!payloadResult.success) {
+      this.logger.warn(
+        { messageId: message.id, issues: payloadResult.error.issues },
+        "invalid notification.scheduled payload — skipping",
+      );
+      return;
+    }
+
+    const scheduled = payloadResult.data as NotificationScheduledPayload;
+    const scheduledAt = new Date(scheduled.scheduledAt).getTime();
+
+    const taskData = JSON.stringify({
+      taskId: scheduled.taskId,
+      enrichedEventId: scheduled.enrichedEventId,
+      traceId: event.metadata.traceId,
+    });
+
+    const shard = parseInt(scheduled.taskId.slice(-1), 16) || 0;
+    const zsetKey = `notif:scheduled:zset:${shard}`;
+    await this.redisCli.zadd(zsetKey, scheduledAt, taskData);
+    this.logger.debug(
+      { taskId: scheduled.taskId, scheduledAt: scheduled.scheduledAt, shard },
+      "scheduled task queued in ZSET",
+    );
+  }
+}
+
+export async function executeSchedulerPoll(
+  redis: Redis,
+  outboundProducers: any,
+  logger: any,
+  db: any,
+): Promise<boolean> {
+  const lockKey = "notif:lock:scheduler:poll";
+  const lockOwner = randomUUID();
+  let acquired = false;
+  try {
+    const lockAcquired = await redis.set(lockKey, lockOwner, "EX", 30, "NX");
+    if (lockAcquired !== "OK") {
+      return false; // Another instance is currently polling
+    }
+    acquired = true;
+
+    const now = Date.now();
+    const visibilityTimeout = 60000; // 60 seconds
+    const perShardLimit = 100;
+    const pipeline = redis.pipeline();
+
+    for (let i = 0; i < 16; i++) {
+      pipeline.eval(
+        LUA_SCHEDULER_POLL,
+        1,
+        `notif:scheduled:zset:${i}`,
+        now,
+        perShardLimit,
+        visibilityTimeout,
+      );
+    }
+
+    const results = await pipeline.exec();
+    const perShard = results?.map((res: any) => (res[0] ? [] : ((res[1] ?? []) as string[]))) ?? [];
+    const tasks = perShard.flat();
+    // A shard that came back full may still have due tasks behind it, which is
+    // what tells the caller to poll again rather than wait for the next tick.
+    const anyShardFull = perShard.some((shard) => shard.length >= perShardLimit);
+
+    if (tasks.length === 0) return false;
+
+    const parsedTasks = tasks
+      .map((t) => {
+        try {
+          return { taskStr: t, ...JSON.parse(t) };
+        } catch (err) {
+          logger.error({ taskStr: t, err }, "failed to parse scheduled task JSON");
+          return null;
+        }
+      })
+      .filter(Boolean) as any[];
+    if (parsedTasks.length === 0) return false;
+
+    const taskIds = parsedTasks.map((t) => t.taskId);
+    const dbPayloads = await db
+      .select()
+      .from(scheduledPayloads)
+      .where(inArray(scheduledPayloads.taskId, taskIds));
+    const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload]));
+
+    const batchedEvents: Record<
+      "critical" | "high" | "normal" | "low",
+      Omit<any, "id" | "timestamp">[]
+    > = {
+      critical: [],
+      high: [],
+      normal: [],
+      low: [],
+    };
+
+    const cleanupPipeline = redis.pipeline();
+    const dbCleanupIds: string[] = [];
+
+    for (let i = 0; i < parsedTasks.length; i++) {
+      const { taskStr, taskId, traceId } = parsedTasks[i];
+      const payload = payloadMap.get(taskId);
+
+      if (!payload) {
+        logger.warn({ taskId }, "scheduled payload not found in Postgres — skipping release");
+        const shard = parseInt(taskId.slice(-1), 16) || 0;
+        cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
+        continue;
+      }
+
+      const dispatchPayload = payload as any;
+      const p = getPriorityBucket(dispatchPayload.priority);
+
+      batchedEvents[p].push(
+        buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId),
+      );
+
+      dbCleanupIds.push(taskId);
+      const shard = parseInt(taskId.slice(-1), 16) || 0;
+      cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
+    }
+
+    // Every bucket, not a hardcoded subset: a task binned into one that is not
+    // published here has already been added to the cleanup list, so it would be
+    // deleted from Redis and Postgres without ever being sent.
+    for (const p of Object.keys(batchedEvents) as (keyof typeof batchedEvents)[]) {
+      if (batchedEvents[p].length > 0) {
+        const producer = outboundProducers[p] ?? outboundProducers["normal"]!;
+        await producer!.publishBatch(batchedEvents[p]);
+      }
+    }
+
+    if (dbCleanupIds.length > 0) {
+      try {
+        await db.delete(scheduledPayloads).where(inArray(scheduledPayloads.taskId, dbCleanupIds));
+      } catch (err) {
+        logger.error(
+          { err },
+          "failed to delete scheduled payloads from Postgres, continuing with Redis cleanup",
+        );
+      }
+    }
+    await cleanupPipeline.exec();
+
+    logger.info({ count: parsedTasks.length }, "scheduled tasks released to outbound");
+
+    // "Poll again immediately" — the caller loops while this is true. The old
+    // test was `tasks.length === 100` against a total drawn from 16 shards, so
+    // a real backlog reported "nothing more" and waited for the next tick.
+    return anyShardFull;
+  } catch (err) {
+    logger.error({ err }, "error in scheduler polling loop");
+    return false;
+  } finally {
+    if (acquired) {
+      // Delete the lock only if we still own it
+      const lua = `
+        if redis.call("get", KEYS[1]) == ARGV[1] then
+          return redis.call("del", KEYS[1])
+        else
+          return 0
+        end
+      `;
+      await redis.eval(lua, 1, lockKey, lockOwner).catch(() => {});
+    }
+  }
+}
+
+export async function startSchedulerWorker() {
+  logger = createLogger({ name: "scheduler", level: config.LOG_LEVEL });
+  redis = new RedisClient({ url: config.REDIS_URL, name: "scheduler", logger });
+  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "scheduler", logger });
+  sql = dbData.sql;
+  db = dbData.db;
+  consumer = new StreamConsumer({
+    redis: redis.native,
+    stream: STREAMS.SCHEDULED,
+    group: CONSUMER_GROUPS.SCHEDULER,
+    consumer: `scheduler-${process.pid}`,
+    dlqStream: STREAMS.DEAD_LETTER,
+    batchSize: config.WORKER_CONCURRENCY,
+    logger,
+  });
+
+  pendingScanner = new PendingMessageScanner({
+    redis: redis.native,
+    stream: STREAMS.SCHEDULED,
+    group: CONSUMER_GROUPS.SCHEDULER,
+    consumer: `scheduler-${process.pid}`,
+    logger,
+  });
+
+  const outboundProducers = {
+    critical: new StreamProducer({
+      redis: redis.native,
+      stream: STREAMS.OUTBOUND_CRITICAL,
+      logger,
+    }),
+    normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),
+    low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),
+  };
+
+  // ─── Stage 4a: Scheduled notification processor ─────────────────────────────
+  //
+  // Reads from SCHEDULED stream, parses, and queues tasks in a Redis ZSET.
+  // A background polling interval releases tasks when they are due.
+
+  worker = new SchedulerWorker({
+    consumer,
+    pendingScanner,
+    logger,
+    concurrency: config.WORKER_CONCURRENCY,
+    registry,
+    redis: redis.native,
+  });
+
+  // ─── Polling Loop ─────────────────────────────────────────────────────────
+  isPolling = true;
+  const pollLoop = async (): Promise<void> => {
+    if (!isPolling) return;
+    try {
+      const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
+      if (isPolling) {
+        pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5000);
+      }
+    } catch (err) {
+      logger.error({ err }, "scheduler poll loop error");
+      if (isPolling) {
+        pollTimeout = setTimeout(() => void pollLoop(), 5000);
+      }
+    }
+  };
+  void pollLoop();
+
+  // ─── Health check interval ──────────────────────────────────────────────────
+
+  healthInterval = startHealthReporter("scheduler", worker, redis, logger);
+
+  logger.info({ env: config.NODE_ENV }, "scheduler starting");
+  await worker.start();
+}
+
+// ─── Shutdown ──────────────────────────────────────────────────────────────
+
+export async function stopSchedulerWorker(): Promise<void> {
+  logger?.info("shutdown initiated");
+  if (healthInterval) {
+    clearInterval(healthInterval);
+    healthInterval = null;
+  }
+  if (pollTimeout) {
+    clearTimeout(pollTimeout);
+    pollTimeout = null;
+  }
+  isPolling = false;
+  if (worker) await worker.stop();
+  if (sql) await sql.end();
+  if (redis) await redis.disconnect();
+  logger?.info("scheduler stopped");
+}