@l-etabli/events 0.6.0 → 0.7.0

package/package.json

@@ -3,7 +3,7 @@
   "description": "The purpose of this repository is to make it easy to setup event driven architecture using outbox pattern",
   "module": "src/index.ts",
   "type": "module",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "files": [
@@ -39,10 +39,11 @@
     "@commitlint/config-conventional": "^20.0.0",
     "@types/bun": "latest",
     "@types/pg": "^8.16.0",
+    "effect": "4.0.0-beta.43",
     "kysely": "^0.28.2",
     "lefthook": "^1.13.6",
     "pg": "^8.16.3",
-    "semantic-release": "^24.2.9",
+    "semantic-release": "^25.0.3",
     "tsup": "^8.5.0",
     "typescript": "^5.9.3"
   },
@@ -56,14 +57,28 @@
       "types": "./dist/adapters/kysely/index.d.ts",
       "import": "./dist/adapters/kysely/index.mjs",
       "require": "./dist/adapters/kysely/index.cjs"
+    },
+    "./effect": {
+      "types": "./dist/effect/index.d.ts",
+      "import": "./dist/effect/index.mjs",
+      "require": "./dist/effect/index.cjs"
+    },
+    "./effect-kysely": {
+      "types": "./dist/adapters/effect-kysely/index.d.ts",
+      "import": "./dist/adapters/effect-kysely/index.mjs",
+      "require": "./dist/adapters/effect-kysely/index.cjs"
     }
   },
   "peerDependencies": {
-    "kysely": "latest"
+    "kysely": ">=0.28.0",
+    "effect": ">=4.0.0-beta.0"
   },
   "peerDependenciesMeta": {
     "kysely": {
       "optional": true
+    },
+    "effect": {
+      "optional": true
     }
   }
 }

package/src/adapters/effect-kysely/EffectKyselyEventQueries.ts

@@ -0,0 +1,45 @@
+import { Effect } from "effect";
+import type { Kysely, SqlBool } from "kysely";
+import { sql } from "kysely";
+import type { EventQueries } from "../../effect/ports/EffectEventQueries.ts";
+import type { DefaultContext, GenericEvent } from "../../types.ts";
+import { mapEventRow } from "../kysely/mapEventRow.ts";
+import type { EventsTable } from "../kysely/types.ts";
+
+export const createEffectKyselyEventQueries = <
+  Event extends GenericEvent<string, unknown, DefaultContext>,
+  DB extends EventsTable = EventsTable,
+>(
+  db: Kysely<DB>,
+): EventQueries<Event> => {
+  const eventsDb = db as unknown as Kysely<EventsTable>;
+  return {
+    getEvents: ({ filters, limit }) =>
+      Effect.gen(function* () {
+        let query = eventsDb
+          .selectFrom("events")
+          .selectAll()
+          .where("status", "in", filters.statuses)
+          .limit(limit);
+
+        if (filters.context) {
+          for (const [key, value] of Object.entries(filters.context)) {
+            query = query.where(sql<SqlBool>`context->>${key} = ${value}`);
+          }
+        }
+
+        if (filters.occurredAt?.from) {
+          query = query.where("occurredAt", ">=", filters.occurredAt.from);
+        }
+
+        if (filters.occurredAt?.to) {
+          query = query.where("occurredAt", "<=", filters.occurredAt.to);
+        }
+
+        const rows = yield* Effect.promise(() => query.execute());
+        return rows.map((row: EventsTable["events"]) =>
+          mapEventRow<Event>(row),
+        );
+      }),
+  };
+};

package/src/adapters/effect-kysely/EffectKyselyEventRepository.ts

@@ -0,0 +1,90 @@
+import { Effect } from "effect";
+import type { Kysely } from "kysely";
+import type { EventRepository } from "../../effect/ports/EffectEventRepository.ts";
+import type { DefaultContext, GenericEvent } from "../../types.ts";
+import { jsonb } from "../kysely/jsonb.ts";
+import type { EventsTable } from "../kysely/types.ts";
+
+export const createEffectKyselyEventRepository = <
+  Event extends GenericEvent<string, unknown, DefaultContext>,
+  DB extends EventsTable = EventsTable,
+>(
+  db: Kysely<DB>,
+): EventRepository<Event> => {
+  const eventsDb = db as unknown as Kysely<EventsTable>;
+  return {
+    save: (event) =>
+      Effect.promise(() =>
+        eventsDb
+          .insertInto("events")
+          .values({
+            ...event,
+            payload: jsonb(event.payload),
+            triggeredByActor: jsonb(event.triggeredByActor),
+            context: jsonb(event.context),
+            publications: jsonb(event.publications),
+          })
+          .onConflict((oc) =>
+            oc.column("id").doUpdateSet({
+              topic: event.topic,
+              payload: jsonb(event.payload),
+              triggeredByActor: jsonb(event.triggeredByActor),
+              context: jsonb(event.context),
+              status: event.status,
+              flowId: event.flowId,
+              causedByEventId: event.causedByEventId,
+              occurredAt: event.occurredAt,
+              publications: jsonb(event.publications),
+              priority: event.priority,
+            }),
+          )
+          .execute(),
+      ).pipe(Effect.asVoid),
+
+    saveNewEventsBatch: (events) => {
+      if (events.length === 0) return Effect.void;
+      return Effect.promise(() =>
+        eventsDb
+          .insertInto("events")
+          .values(
+            events.map((event) => ({
+              ...event,
+              payload: jsonb(event.payload),
+              triggeredByActor: jsonb(event.triggeredByActor),
+              context: jsonb(event.context),
+              publications: jsonb(event.publications),
+            })),
+          )
+          .execute(),
+      ).pipe(Effect.asVoid);
+    },
+
+    markEventsAsInProcess: (events) => {
+      if (events.length === 0) return Effect.void;
+      const ids = events.map((e) => e.id);
+
+      return Effect.gen(function* () {
+        const lockedRows = yield* Effect.promise(() =>
+          eventsDb
+            .selectFrom("events")
+            .select("id")
+            .where("id", "in", ids)
+            .forUpdate()
+            .skipLocked()
+            .execute(),
+        );
+
+        if (lockedRows.length === 0) return;
+        const lockedIds = lockedRows.map((r) => r.id);
+
+        yield* Effect.promise(() =>
+          eventsDb
+            .updateTable("events")
+            .set({ status: "in-process" })
+            .where("id", "in", lockedIds)
+            .execute(),
+        ).pipe(Effect.asVoid);
+      });
+    },
+  };
+};

package/src/adapters/effect-kysely/index.ts

@@ -0,0 +1,3 @@
+export type { EventsTable, TypedEventsTable } from "../kysely/types.ts";
+export { createEffectKyselyEventQueries } from "./EffectKyselyEventQueries.ts";
+export { createEffectKyselyEventRepository } from "./EffectKyselyEventRepository.ts";

package/src/adapters/in-memory/InMemoryEventBus.ts

@@ -7,6 +7,7 @@ import type {
   EventDefinitions,
   InferEventsFromDefinitions,
 } from "../../eventDefinitions.ts";
+import { getSubscriptionIdsToPublish } from "../../getSubscriptionIdsToPublish.ts";
 import type { EventBus } from "../../ports/EventBus.ts";
 import type { WithEventsUow } from "../../ports/EventRepository.ts";
 import {
@@ -113,28 +114,6 @@ export function createInMemoryEventBus<
     }
   };
 
-  const getSubscriptionIdsToPublish = (
-    event: Event,
-    callbacksBySubscriptionId: SubscriptionsForTopic,
-  ): string[] => {
-    const allSubscriptionIds = Object.keys(callbacksBySubscriptionId);
-
-    if (event.publications.length === 0 || event.status === "to-republish") {
-      return allSubscriptionIds;
-    }
-
-    const lastPublication = event.publications.reduce((latest, current) =>
-      current.publishedAt > latest.publishedAt ? current : latest,
-    );
-    const failedSubscriptionIds = (lastPublication.failures ?? []).map(
-      (failure) => failure.subscriptionId,
-    );
-
-    return allSubscriptionIds.filter((id) =>
-      failedSubscriptionIds.includes(id),
-    );
-  };
-
   const eventBus: EventBus<Event> = {
     publish: async (event) => {
       const publishedAt = new Date();
@@ -156,7 +135,7 @@ export function createInMemoryEventBus<
 
       const subscriptionIdsToPublish = getSubscriptionIdsToPublish(
         event,
-        callbacksBySubscriptionSlug,
+        Object.keys(callbacksBySubscriptionSlug),
       );
 
       const failuresOrUndefined = await Promise.all(

package/src/adapters/in-memory/InMemoryEventQueries.ts

@@ -1,3 +1,4 @@
+import { filterEvents } from "../../filterEvents.ts";
 import type { EventQueries } from "../../ports/EventQueries.ts";
 import type { DefaultContext, GenericEvent } from "../../types.ts";
 import type { InMemoryEventRepositoryHelpers } from "./InMemoryEventRepository.ts";
@@ -8,37 +9,6 @@ export const createInMemoryEventQueries = <
   helpers: InMemoryEventRepositoryHelpers<Event>,
 ): { eventQueries: EventQueries<Event> } => ({
   eventQueries: {
-    getEvents: async ({ filters, limit }) => {
-      const matchesContext = (event: Event): boolean => {
-        if (!filters.context) return true;
-        if (!event.context) return false;
-
-        return Object.entries(filters.context).every(
-          ([key, value]) => event.context?.[key] === value,
-        );
-      };
-
-      const matchesOccurredAt = (event: Event): boolean => {
-        if (!filters.occurredAt) return true;
-
-        const { from, to } = filters.occurredAt;
-        const eventTime = event.occurredAt.getTime();
-
-        if (from && eventTime < from.getTime()) return false;
-        if (to && eventTime > to.getTime()) return false;
-
-        return true;
-      };
-
-      return helpers
-        .getAllEvents()
-        .filter(
-          (event) =>
-            filters.statuses.includes(event.status) &&
-            matchesContext(event) &&
-            matchesOccurredAt(event),
-        )
-        .slice(0, limit);
-    },
+    getEvents: async (params) => filterEvents(helpers.getAllEvents(), params),
   },
 });

package/src/adapters/kysely/KyselyEventQueries.ts

@@ -2,44 +2,40 @@ import type { Kysely, SqlBool } from "kysely";
 import { sql } from "kysely";
 import type { EventQueries } from "../../ports/EventQueries.ts";
 import type { DefaultContext, GenericEvent } from "../../types.ts";
+import { mapEventRow } from "./mapEventRow.ts";
 import type { EventsTable } from "./types.ts";
 
 export const createKyselyEventQueries = <
   Event extends GenericEvent<string, unknown, DefaultContext>,
+  DB extends EventsTable = EventsTable,
 >(
-  db: Kysely<EventsTable>,
-): EventQueries<Event> => ({
-  getEvents: async ({ filters, limit }) => {
-    let query = db
-      .selectFrom("events")
-      .selectAll()
-      .where("status", "in", filters.statuses)
-      .limit(limit);
+  db: Kysely<DB>,
+): EventQueries<Event> => {
+  const eventsDb = db as unknown as Kysely<EventsTable>;
+  return {
+    getEvents: async ({ filters, limit }) => {
+      let query = eventsDb
+        .selectFrom("events")
+        .selectAll()
+        .where("status", "in", filters.statuses)
+        .limit(limit);
 
-    if (filters.context) {
-      for (const [key, value] of Object.entries(filters.context)) {
-        query = query.where(sql<SqlBool>`context->>${key} = ${value}`);
+      if (filters.context) {
+        for (const [key, value] of Object.entries(filters.context)) {
+          query = query.where(sql<SqlBool>`context->>${key} = ${value}`);
+        }
       }
-    }
 
-    if (filters.occurredAt?.from) {
-      query = query.where("occurredAt", ">=", filters.occurredAt.from);
-    }
+      if (filters.occurredAt?.from) {
+        query = query.where("occurredAt", ">=", filters.occurredAt.from);
+      }
 
-    if (filters.occurredAt?.to) {
-      query = query.where("occurredAt", "<=", filters.occurredAt.to);
-    }
+      if (filters.occurredAt?.to) {
+        query = query.where("occurredAt", "<=", filters.occurredAt.to);
+      }
 
-    const rows = await query.execute();
-    return rows.map(
-      (row: EventsTable["events"]) =>
-        ({
-          ...row,
-          context: row.context ?? undefined,
-          flowId: row.flowId ?? undefined,
-          causedByEventId: row.causedByEventId ?? undefined,
-          priority: row.priority ?? undefined,
-        }) as Event,
-    );
-  },
-});
+      const rows = await query.execute();
+      return rows.map((row: EventsTable["events"]) => mapEventRow<Event>(row));
+    },
+  };
+};

package/src/adapters/kysely/KyselyEventRepository.ts

@@ -1,80 +1,82 @@
-import { type Kysely, type RawBuilder, sql } from "kysely";
+import type { Kysely } from "kysely";
 import type { EventRepository } from "../../ports/EventRepository.ts";
 import type { DefaultContext, GenericEvent } from "../../types.ts";
+import { jsonb } from "./jsonb.ts";
 import type { EventsTable } from "./types.ts";
 
-const jsonb = <T>(value: T): RawBuilder<T> =>
-  sql`${JSON.stringify(value)}::jsonb`;
-
 export const createKyselyEventRepository = <
   Event extends GenericEvent<string, unknown, DefaultContext>,
+  DB extends EventsTable = EventsTable,
 >(
-  db: Kysely<EventsTable>,
-): EventRepository<Event> => ({
-  save: async (event) => {
-    await db
-      .insertInto("events")
-      .values({
-        ...event,
-        payload: jsonb(event.payload),
-        triggeredByActor: jsonb(event.triggeredByActor),
-        context: jsonb(event.context),
-        publications: jsonb(event.publications),
-      })
-      .onConflict((oc) =>
-        oc.column("id").doUpdateSet({
-          topic: event.topic,
-          payload: jsonb(event.payload),
-          triggeredByActor: jsonb(event.triggeredByActor),
-          context: jsonb(event.context),
-          status: event.status,
-          flowId: event.flowId,
-          causedByEventId: event.causedByEventId,
-          occurredAt: event.occurredAt,
-          publications: jsonb(event.publications),
-          priority: event.priority,
-        }),
-      )
-      .execute();
-  },
-
-  saveNewEventsBatch: async (events) => {
-    if (events.length === 0) return;
-    await db
-      .insertInto("events")
-      .values(
-        events.map((event) => ({
+  db: Kysely<DB>,
+): EventRepository<Event> => {
+  const eventsDb = db as unknown as Kysely<EventsTable>;
+  return {
+    save: async (event) => {
+      await eventsDb
+        .insertInto("events")
+        .values({
           ...event,
           payload: jsonb(event.payload),
           triggeredByActor: jsonb(event.triggeredByActor),
           context: jsonb(event.context),
           publications: jsonb(event.publications),
-        })),
-      )
-      .execute();
-  },
+        })
+        .onConflict((oc) =>
+          oc.column("id").doUpdateSet({
+            topic: event.topic,
+            payload: jsonb(event.payload),
+            triggeredByActor: jsonb(event.triggeredByActor),
+            context: jsonb(event.context),
+            status: event.status,
+            flowId: event.flowId,
+            causedByEventId: event.causedByEventId,
+            occurredAt: event.occurredAt,
+            publications: jsonb(event.publications),
+            priority: event.priority,
+          }),
+        )
+        .execute();
+    },
+
+    saveNewEventsBatch: async (events) => {
+      if (events.length === 0) return;
+      await eventsDb
+        .insertInto("events")
+        .values(
+          events.map((event) => ({
+            ...event,
+            payload: jsonb(event.payload),
+            triggeredByActor: jsonb(event.triggeredByActor),
+            context: jsonb(event.context),
+            publications: jsonb(event.publications),
+          })),
+        )
+        .execute();
+    },
 
-  markEventsAsInProcess: async (events) => {
-    if (events.length === 0) return;
-    const ids = events.map((e) => e.id);
+    markEventsAsInProcess: async (events) => {
+      if (events.length === 0) return;
+      const ids = events.map((e) => e.id);
 
-    // Lock the rows to prevent concurrent processing
-    const lockedRows = await db
-      .selectFrom("events")
-      .select("id")
-      .where("id", "in", ids)
-      .forUpdate()
-      .skipLocked()
-      .execute();
+      // Lock the rows to prevent concurrent processing
+      const lockedRows = await eventsDb
+        .selectFrom("events")
+        .select("id")
+        .where("id", "in", ids)
+        .forUpdate()
+        .skipLocked()
+        .execute();
 
-    if (lockedRows.length === 0) return;
-    const lockedIds = lockedRows.map((r) => r.id);
+      if (lockedRows.length === 0) return;
+      const lockedIds = lockedRows.map((r) => r.id);
 
-    // Update status to in-process (only for locked rows)
-    await db
-      .updateTable("events")
-      .set({ status: "in-process" })
-      .where("id", "in", lockedIds)
-      .execute();
-  },
-});
+      // Update status to in-process (only for locked rows)
+      await eventsDb
+        .updateTable("events")
+        .set({ status: "in-process" })
+        .where("id", "in", lockedIds)
+        .execute();
+    },
+  };
+};

package/src/adapters/kysely/jsonb.ts

@@ -0,0 +1,4 @@
+import { type RawBuilder, sql } from "kysely";
+
+export const jsonb = <T>(value: T): RawBuilder<T> =>
+  sql`${JSON.stringify(value)}::jsonb`;

package/src/adapters/kysely/mapEventRow.ts

@@ -0,0 +1,15 @@
+import type { DefaultContext, GenericEvent } from "../../types.ts";
+import type { EventsTable } from "./types.ts";
+
+export const mapEventRow = <
+  Event extends GenericEvent<string, unknown, DefaultContext>,
+>(
+  row: EventsTable["events"],
+): Event =>
+  ({
+    ...row,
+    context: row.context ?? undefined,
+    flowId: row.flowId ?? undefined,
+    causedByEventId: row.causedByEventId ?? undefined,
+    priority: row.priority ?? undefined,
+  }) as Event;

package/src/createEventCrawler.ts

@@ -1,6 +1,7 @@
 import type { EventBus } from "./ports/EventBus.ts";
 import type { EventQueries } from "./ports/EventQueries.ts";
 import type { WithEventsUow } from "./ports/EventRepository.ts";
+import { splitIntoChunks } from "./splitIntoChunks.ts";
 import type { DefaultContext, GenericEvent } from "./types.ts";
 
 /** Configuration options for the event crawler. */
@@ -15,14 +16,6 @@ type CreateEventCrawlerOptions = {
   failedEventsIntervalMs?: number;
 };
 
-const splitIntoChunks = <T>(array: T[], chunkSize: number): T[][] => {
-  const chunks: T[][] = [];
-  for (let i = 0; i < array.length; i += chunkSize) {
-    chunks.push(array.slice(i, i + chunkSize));
-  }
-  return chunks;
-};
-
 /**
  * Creates a background event crawler that processes and publishes events.
  *

package/src/effect/EffectEventCrawler.ts

@@ -0,0 +1,124 @@
+import { Cause, Effect, Exit } from "effect";
+import type { DefaultContext, GenericEvent } from "../types.ts";
+import type { EventBus } from "./ports/EffectEventBus.ts";
+import type { EventQueries } from "./ports/EffectEventQueries.ts";
+import type { WithEventsUow } from "./ports/EffectEventRepository.ts";
+
+type CreateEffectEventCrawlerOptions = {
+  batchSize?: number;
+  maxParallelProcessing?: number;
+  newEventsIntervalMs?: number;
+  failedEventsIntervalMs?: number;
+};
+
+export const createEffectEventCrawler = <
+  Event extends GenericEvent<string, unknown, DefaultContext>,
+>({
+  withUow,
+  eventQueries,
+  eventBus,
+  options = {},
+}: {
+  withUow: WithEventsUow<Event>;
+  eventQueries: EventQueries<Event>;
+  eventBus: EventBus<Event>;
+  options?: CreateEffectEventCrawlerOptions;
+}) => {
+  const batchSize = options.batchSize ?? 100;
+  const maxParallelProcessing = options.maxParallelProcessing ?? 1;
+  const newEventsIntervalMs = options.newEventsIntervalMs ?? 10_000;
+  const failedEventsIntervalMs = options.failedEventsIntervalMs ?? 60_000;
+
+  const publishEventsInParallel = (events: Event[]): Effect.Effect<void> =>
+    Effect.all(
+      events.map((event) => eventBus.publish(event)),
+      { concurrency: maxParallelProcessing, discard: true },
+    );
+
+  const processNewEvents = (): Effect.Effect<void> =>
+    Effect.gen(function* () {
+      const events = yield* eventQueries.getEvents({
+        filters: { statuses: ["never-published"] },
+        limit: batchSize,
+      });
+
+      if (events.length === 0) return;
+
+      yield* withUow((uow) =>
+        uow.eventRepository.markEventsAsInProcess(events),
+      );
+
+      yield* publishEventsInParallel(events);
+    });
+
+  const retryFailedEvents = (): Effect.Effect<void> =>
+    Effect.gen(function* () {
+      const oneMinuteAgo = new Date(Date.now() - 60_000);
+
+      const events = yield* eventQueries.getEvents({
+        filters: {
+          statuses: ["to-republish", "failed-but-will-retry"],
+          occurredAt: { to: oneMinuteAgo },
+        },
+        limit: batchSize,
+      });
+
+      if (events.length === 0) return;
+
+      yield* publishEventsInParallel(events);
+    });
+
+  const triggerProcessing = (): Effect.Effect<void> =>
+    Effect.gen(function* () {
+      const results = yield* Effect.all(
+        [Effect.exit(processNewEvents()), Effect.exit(retryFailedEvents())],
+        { concurrency: "unbounded" },
+      );
+
+      const errors = results
+        .filter(Exit.isFailure)
+        .map((exit) => Cause.squash(exit.cause));
+
+      if (errors.length > 0) {
+        yield* Effect.die(
+          new AggregateError(errors, "Event processing failed"),
+        );
+      }
+    });
+
+  const start = () => {
+    const scheduleProcessNewEvents = () => {
+      setTimeout(async () => {
+        try {
+          await Effect.runPromise(processNewEvents());
+        } catch (error) {
+          console.error("Error processing new events:", error);
+        } finally {
+          scheduleProcessNewEvents();
+        }
+      }, newEventsIntervalMs);
+    };
+
+    const scheduleRetryFailedEvents = () => {
+      setTimeout(async () => {
+        try {
+          await Effect.runPromise(retryFailedEvents());
+        } catch (error) {
+          console.error("Error retrying failed events:", error);
+        } finally {
+          scheduleRetryFailedEvents();
+        }
+      }, failedEventsIntervalMs);
+    };
+
+    scheduleProcessNewEvents();
+    scheduleRetryFailedEvents();
+  };
+
+  return {
+    processNewEvents,
+    retryFailedEvents,
+    triggerProcessing,
+    start,
+  };
+};