@wataruoguchi/emmett-event-store-kysely 1.1.3 → 2.0.0

package/README.md

@@ -4,210 +4,122 @@ A Kysely-based event store implementation for [Emmett](https://github.com/event-
 
 ## Features
 
-- **Event Store**: Emmett event store implementation with Kysely
-- **Projections**: Read model projections with automatic event processing
+- **Event Store** - Full event sourcing with Kysely and PostgreSQL
+- **Snapshot Projections** - Recommended approach for read models
+- **Event Consumer** - Continuous background event processing
+- **Type Safety** - Full TypeScript support with discriminated unions
+- **Multi-Tenancy** - Built-in partition support
 
 ## Installation
 
 ```bash
-npm install @wataruoguchi/emmett-event-store-kysely @event-driven-io/emmett kysely
+npm install @wataruoguchi/emmett-event-store-kysely @event-driven-io/emmett kysely pg
 ```
 
-## Database Setup
+## Quick Start
 
-First, you need to set up the event store tables in your PostgreSQL database. You can achieve this with [the Kysely migration file](https://github.com/wataruoguchi/poc-emmett/blob/main/package/1758758113676_event_sourcing_migration_example.ts)
+### 1. Database Setup
 
-## Basic Usage
+Set up the required PostgreSQL tables using [our migration example](./database/migrations/1758758113676_event_sourcing_migration_example.ts):
 
-You can find [the complete working example](https://github.com/wataruoguchi/poc-emmett/tree/main/example). The following are some snippets.
+```typescript
+import { Kysely } from "kysely";
+
+// Required tables: messages, streams, subscriptions
+// See docs/database-setup.md for details
+```
 
-### 1. Setting up the Event Store
+### 2. Create Event Store
 
 ```typescript
-import { createEventStore } from "@wataruoguchi/emmett-event-store-kysely";
+import { getKyselyEventStore } from "@wataruoguchi/emmett-event-store-kysely";
 import { Kysely, PostgresDialect } from "kysely";
-import { Pool } from "pg";
 
-// Set up your Kysely database connection
-const db = new Kysely<YourDatabaseSchema>({
+const db = new Kysely({
   dialect: new PostgresDialect({
-    pool: new Pool({
-      connectionString: process.env.DATABASE_URL,
-    }),
+    pool: new Pool({ connectionString: process.env.DATABASE_URL }),
   }),
 });
 
-// Create the event store
-const eventStore = createEventStore({ db, logger });
+const eventStore = getKyselyEventStore({ 
+  db, 
+  logger: console,
+});
 ```
 
-### 2. Using the Event Store with Emmett
+### 3. Write Events & Commands & Business Logic & State
 
-```typescript
-import { DeciderCommandHandler } from "@event-driven-io/emmett";
-import type { EventStore } from "@wataruoguchi/emmett-event-store-kysely";
-
-// Define your domain events and commands
-type CreateCartCommand = {
-  type: "CreateCart";
-  data: { tenantId: string; cartId: string; currency: string };
-};
-
-type CartCreatedEvent = {
-  type: "CartCreated";
-  data: { tenantId: string; cartId: string; currency: string };
-};
-
-// Create your event handler
-export function cartEventHandler({
-  eventStore,
-  getContext,
-}: {
-  eventStore: EventStore;
-  getContext: () => AppContext;
-}) {
-  const handler = DeciderCommandHandler({
-    decide: createDecide(getContext),
-    evolve: createEvolve(),
-    initialState,
-  });
-
-  return {
-    create: (cartId: string, data: CreateCartCommand["data"]) =>
-      handler(
-        eventStore,
-        cartId,
-        { type: "CreateCart", data },
-        { partition: data.tenantId, streamType: "cart" }
-      ),
-  };
-}
-
-// Use in your service
-const cartService = createCartService({
-  eventStore,
-  getContext,
-});
-```
+Please read <https://event-driven-io.github.io/emmett/getting-started.html>
 
-## Projections (Read Models)
+- [Events](https://event-driven-io.github.io/emmett/getting-started.html#events)
+- [Commands](https://event-driven-io.github.io/emmett/getting-started.html#commands)
+- [Business logic and decisions](https://event-driven-io.github.io/emmett/getting-started.html#business-logic-and-decisions)
+- [Building state from events](https://event-driven-io.github.io/emmett/getting-started.html#building-state-from-events)
 
-### 1. Creating a Projection
+### 4. Build Read Models
+
+This package supports "Snapshot Projections".
 
 ```typescript
-import type {
-  ProjectionEvent,
-  ProjectionRegistry,
+import { 
+  createSnapshotProjectionRegistry 
 } from "@wataruoguchi/emmett-event-store-kysely/projections";
 
-export function cartsProjection(): ProjectionRegistry<DatabaseExecutor> {
-  return {
-    CartCreated: async (db, event) => {
-      await db
-        .insertInto("carts")
-        .values({
-          stream_id: event.metadata.streamId,
-          tenant_id: event.data.tenantId,
-          cart_id: event.data.cartId,
-          currency: event.data.currency,
-          items: JSON.stringify([]),
-          total: 0,
-          last_stream_position: event.metadata.streamPosition,
-        })
-        .execute();
-    },
-    ItemAddedToCart: async (db, event) => {
-      // Update cart with new item
-      await db
-        .updateTable("carts")
-        .set({
-          items: JSON.stringify([...existingItems, event.data.item]),
-          total: newTotal,
-          last_stream_position: event.metadata.streamPosition,
-        })
-        .where("stream_id", "=", event.metadata.streamId)
-        .execute();
-    },
-  };
-}
+// Reuse your write model's evolve function!
+const registry = createSnapshotProjectionRegistry(
+  ["CartCreated", "ItemAdded", "CartCheckedOut"],
+  {
+    tableName: "carts",
+    primaryKeys: ["tenant_id", "cart_id", "partition"],
+    extractKeys: (event, partition) => ({
+      tenant_id: event.data.eventMeta.tenantId,
+      cart_id: event.data.eventMeta.cartId,
+      partition,
+    }),
+    evolve: domainEvolve,      // Reuse from write model!
+    initialState,
+    mapToColumns: (state) => ({ // Optional: denormalize for queries
+      currency: state.currency,
+      total: state.status === "checkedOut" ? state.total : null,
+    }),
+  }
+);
 ```
 
-### 2. Running Projections
+### 5. Process Events and Update Read Model
 
 ```typescript
-import {
-  createProjectionRegistry,
-  createProjectionRunner,
-} from "@wataruoguchi/emmett-event-store-kysely/projections";
-import { createReadStream } from "@wataruoguchi/emmett-event-store-kysely";
-
-// Set up projection runner
-const readStream = createReadStream({ db, logger });
-const registry = createProjectionRegistry(cartsProjection());
-const runner = createProjectionRunner({
-  db,
-  readStream,
+import { createProjectionRunner } from "@wataruoguchi/emmett-event-store-kysely/projections";
+
+const runner = createProjectionRunner({ 
+  db, 
+  readStream: eventStore.readStream, 
   registry,
 });
 
-// Project events for a specific stream
-await runner.projectEvents("carts-read-model", "cart-123", {
-  partition: "tenant-456",
-  batchSize: 100,
+await runner.projectEvents("subscription-id", "cart-123", {
+  partition: "tenant-456"
 });
 ```
 
-### 3. Projection Worker
+See [Snapshot Projections documentation](./docs/snapshot-projections.md) for details.
 
-Create a worker to continuously process projections:
+## Documentation
 
-```typescript
-#!/usr/bin/env node
-import { createReadStream } from "@wataruoguchi/emmett-event-store-kysely";
-import {
-  createProjectionRegistry,
-  createProjectionRunner,
-} from "@wataruoguchi/emmett-event-store-kysely/projections";
+📚 **[Complete Documentation](./docs/README.md)**
 
-async function main(partition: string) {
-  const db = getDb();
-  const readStream = createReadStream({ db, logger });
-  const registry = createProjectionRegistry(cartsProjection());
-  const runner = createProjectionRunner({
-    db,
-    readStream,
-    registry,
-  });
-
-  const subscriptionId = "carts-read-model";
-  const batchSize = 200;
-  const pollIntervalMs = 1000;
-
-  while (true) {
-    // Get streams for this partition
-    const streams = await db
-      .selectFrom("streams")
-      .select(["stream_id"])
-      .where("is_archived", "=", false)
-      .where("partition", "=", partition)
-      .where("stream_type", "=", "cart")
-      .execute();
-
-    // Process each stream
-    for (const stream of streams) {
-      await runner.projectEvents(subscriptionId, stream.stream_id, {
-        partition,
-        batchSize,
-      });
-    }
-
-    await new Promise((r) => setTimeout(r, pollIntervalMs));
-  }
-}
+### Core Guides
 
-// Run with: node projection-worker.js tenant-123
-main(process.argv[2]);
-```
+- [Database Setup](./docs/database-setup.md) - PostgreSQL schema and requirements
+- [Event Store](./docs/event-store.md) - Core event store API
+- [Snapshot Projections](./docs/snapshot-projections.md) - Build read models (recommended) ⭐
+- [Event Consumer](./docs/consumer.md) - Continuous background processing
+- [Projection Runner](./docs/projection-runner.md) - On-demand processing for tests
+
+### Examples
+
+- [Working Example](../example/) - Complete application with carts and generators
+- [Migration Example](./database/migrations/1758758113676_event_sourcing_migration_example.ts) - Database setup
 
 ## License
 
@@ -215,4 +127,4 @@ MIT
 
 ## Contributing
 
-Contributions are welcome! Please feel free to submit a Pull Request.
+Contributions are welcome! Please see our [GitHub repository](https://github.com/wataruoguchi/poc-emmett) for issues and PRs.

package/dist/event-store/consumers.d.ts

@@ -0,0 +1,23 @@
+import type { Event, ReadEvent, ReadEventMetadataWithGlobalPosition } from "@event-driven-io/emmett";
+import type { Dependencies } from "../types.js";
+export type KyselyEventStoreConsumerConfig = {
+    /** Consumer name for tracking subscription state */
+    consumerName?: string;
+    /** Batch size for processing events */
+    batchSize?: number;
+    /** Polling interval in milliseconds */
+    pollingInterval?: number;
+};
+export type KyselyEventStoreConsumer = {
+    /** Start consuming events */
+    start(): Promise<void>;
+    /** Stop consuming events */
+    stop(): Promise<void>;
+    /** Subscribe to specific event types */
+    subscribe<EventType extends Event>(handler: (event: ReadEvent<EventType, ReadEventMetadataWithGlobalPosition>) => Promise<void> | void, eventType: string): void;
+    /** Subscribe to all events */
+    subscribeToAll(handler: (event: ReadEvent<Event, ReadEventMetadataWithGlobalPosition>) => Promise<void> | void): void;
+};
+export declare function createKyselyEventStoreConsumer({ db, logger, consumerName, batchSize, pollingInterval, }: Dependencies & KyselyEventStoreConsumerConfig): KyselyEventStoreConsumer;
+export declare function createKyselyEventStoreConsumerWithDefaults(deps: Dependencies, config?: KyselyEventStoreConsumerConfig): KyselyEventStoreConsumer;
+//# sourceMappingURL=consumers.d.ts.map

package/dist/event-store/consumers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumers.d.ts","sourceRoot":"","sources":["../../src/event-store/consumers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,KAAK,EACL,SAAS,EACT,mCAAmC,EACpC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,MAAM,8BAA8B,GAAG;IAC3C,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,6BAA6B;IAC7B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,4BAA4B;IAC5B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,wCAAwC;IACxC,SAAS,CAAC,SAAS,SAAS,KAAK,EAC/B,OAAO,EAAE,CACP,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,mCAAmC,CAAC,KAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EACzB,SAAS,EAAE,MAAM,GAChB,IAAI,CAAC;IACR,8BAA8B;IAC9B,cAAc,CACZ,OAAO,EAAE,CACP,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,mCAAmC,CAAC,KACzD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACxB,IAAI,CAAC;CACT,CAAC;AAEF,wBAAgB,8BAA8B,CAAC,EAC7C,EAAE,EACF,MAAM,EACN,YAAiC,EACjC,SAAe,EACf,eAAsB,GACvB,EAAE,YAAY,GAAG,8BAA8B,GAAG,wBAAwB,CAkM1E;AAGD,wBAAgB,0CAA0C,CACxD,IAAI,EAAE,YAAY,EAClB,MAAM,GAAE,8BAAmC,GAC1C,wBAAwB,CAK1B"}

package/dist/event-store/consumers.js

@@ -0,0 +1,155 @@
+export function createKyselyEventStoreConsumer({ db, logger, consumerName = "default-consumer", batchSize = 100, pollingInterval = 1000, }) {
+    let isRunning = false;
+    let lastProcessedPosition = 0n;
+    const eventHandlers = new Map();
+    const allEventHandlers = [];
+    let pollingTimer = null;
+    const processEvents = async () => {
+        if (!isRunning)
+            return;
+        try {
+            // Get events from the last processed position
+            const events = await db
+                .selectFrom("messages")
+                .select([
+                "message_type",
+                "message_data",
+                "message_metadata",
+                "stream_position",
+                "global_position",
+                "message_id",
+                "stream_id",
+            ])
+                .where("global_position", ">", lastProcessedPosition)
+                .where("is_archived", "=", false)
+                .orderBy("global_position")
+                .limit(batchSize)
+                .execute();
+            if (events.length === 0) {
+                return;
+            }
+            // Process each event
+            for (const row of events) {
+                const event = {
+                    kind: "Event",
+                    type: row.message_type,
+                    data: row.message_data,
+                    metadata: {
+                        ...row.message_metadata,
+                        messageId: row.message_id,
+                        streamName: row.stream_id,
+                        streamPosition: BigInt(String(row.stream_position)),
+                        globalPosition: BigInt(String(row.global_position)),
+                    },
+                };
+                // Call type-specific handlers
+                const typeHandlers = eventHandlers.get(row.message_type) || [];
+                for (const handler of typeHandlers) {
+                    try {
+                        await handler(event);
+                    }
+                    catch (error) {
+                        logger.error({ error, event }, `Error processing event ${row.message_type}`);
+                    }
+                }
+                // Call all-event handlers
+                for (const handler of allEventHandlers) {
+                    try {
+                        await handler(event);
+                    }
+                    catch (error) {
+                        logger.error({ error, event }, "Error processing event in all-event handler");
+                    }
+                }
+                // Update last processed position
+                const globalPos = row.global_position;
+                if (globalPos !== null) {
+                    lastProcessedPosition = BigInt(String(globalPos));
+                }
+            }
+            // Update subscription tracking
+            await updateSubscriptionPosition();
+        }
+        catch (error) {
+            logger.error({ error }, "Error processing events");
+        }
+    };
+    const updateSubscriptionPosition = async () => {
+        try {
+            await db
+                .insertInto("subscriptions")
+                .values({
+                consumer_name: consumerName,
+                last_processed_position: lastProcessedPosition,
+                last_processed_transaction_id: lastProcessedPosition,
+                created_at: new Date(),
+                updated_at: new Date(),
+            })
+                .onConflict((oc) => oc.column("consumer_name").doUpdateSet({
+                last_processed_position: lastProcessedPosition,
+                last_processed_transaction_id: lastProcessedPosition,
+                updated_at: new Date(),
+            }))
+                .execute();
+        }
+        catch (error) {
+            logger.error({ error }, "Error updating subscription position");
+        }
+    };
+    const loadLastProcessedPosition = async () => {
+        try {
+            const subscription = await db
+                .selectFrom("subscriptions")
+                .select(["last_processed_position"])
+                .where("consumer_name", "=", consumerName)
+                .executeTakeFirst();
+            if (subscription) {
+                lastProcessedPosition = BigInt(String(subscription.last_processed_position));
+            }
+        }
+        catch (error) {
+            logger.error({ error }, "Error loading last processed position");
+        }
+    };
+    return {
+        async start() {
+            if (isRunning)
+                return;
+            isRunning = true;
+            await loadLastProcessedPosition();
+            logger.info({ consumerName }, "Starting event store consumer");
+            pollingTimer = setInterval(processEvents, pollingInterval);
+        },
+        async stop() {
+            if (!isRunning)
+                return;
+            isRunning = false;
+            if (pollingTimer) {
+                clearInterval(pollingTimer);
+                pollingTimer = null;
+            }
+            logger.info({ consumerName }, "Stopped event store consumer");
+        },
+        subscribe(handler, eventType) {
+            if (!eventHandlers.has(eventType)) {
+                eventHandlers.set(eventType, []);
+            }
+            const handlers = eventHandlers.get(eventType);
+            if (handlers) {
+                // Type assertion needed because we're storing handlers for specific event types
+                // in a generic Map that accepts Event handlers
+                handlers.push(handler);
+            }
+        },
+        subscribeToAll(handler) {
+            allEventHandlers.push(handler);
+        },
+    };
+}
+// Helper function to create consumer with default options
+export function createKyselyEventStoreConsumerWithDefaults(deps, config = {}) {
+    return createKyselyEventStoreConsumer({
+        ...deps,
+        ...config,
+    });
+}

package/dist/event-store/kysely-event-store.d.ts

@@ -0,0 +1,42 @@
+import { type AppendToStreamOptions, type AppendToStreamResultWithGlobalPosition, type Event, type EventStore, type EventStoreSessionFactory, type ReadEventMetadataWithGlobalPosition, type ReadStreamOptions, type ReadStreamResult } from "@event-driven-io/emmett";
+import type { Kysely } from "kysely";
+import { type Dependencies, type ExtendedOptions } from "../types.js";
+type KyselyReadEventMetadata = ReadEventMetadataWithGlobalPosition;
+type ExtendedAppendToStreamOptions = AppendToStreamOptions & ExtendedOptions;
+export type ProjectionReadStreamOptions = {
+    from?: bigint;
+    to?: bigint;
+    partition?: string;
+    maxCount?: bigint;
+};
+export interface KyselyEventStore extends EventStore<KyselyReadEventMetadata>, EventStoreSessionFactory<KyselyEventStore> {
+    readStream<EventType extends Event>(streamName: string, options?: ReadStreamOptions<bigint> | ProjectionReadStreamOptions): Promise<ReadStreamResult<EventType, KyselyReadEventMetadata>>;
+    appendToStream<EventType extends Event>(streamName: string, events: EventType[], options?: ExtendedAppendToStreamOptions): Promise<AppendToStreamResultWithGlobalPosition>;
+    close(): Promise<void>;
+    schema: {
+        sql(): string;
+        print(): void;
+        migrate(): Promise<void>;
+    };
+}
+export type KyselyEventStoreOptions = {
+    /** Database connection options */
+    connectionOptions?: {
+        /** Custom database executor (Kysely instance) */
+        db?: Kysely<unknown>;
+    };
+    /** Schema management options */
+    schema?: {
+        /** Auto-migration strategy */
+        autoMigration?: "CreateOrUpdate" | "None";
+    };
+    /** Hooks for lifecycle events */
+    hooks?: {
+        /** Called after schema is created */
+        onAfterSchemaCreated?: () => Promise<void> | void;
+    };
+};
+export declare const defaultKyselyOptions: KyselyEventStoreOptions;
+export declare const getKyselyEventStore: (deps: Dependencies) => KyselyEventStore;
+export {};
+//# sourceMappingURL=kysely-event-store.d.ts.map

package/dist/event-store/kysely-event-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kysely-event-store.d.ts","sourceRoot":"","sources":["../../src/event-store/kysely-event-store.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,qBAAqB,EAC1B,KAAK,sCAAsC,EAC3C,KAAK,KAAK,EACV,KAAK,UAAU,EAEf,KAAK,wBAAwB,EAE7B,KAAK,mCAAmC,EACxC,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACtB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,eAAe,EACrB,MAAM,aAAa,CAAC;AAErB,KAAK,uBAAuB,GAAG,mCAAmC,CAAC;AACnE,KAAK,6BAA6B,GAAG,qBAAqB,GAAG,eAAe,CAAC;AAI7E,MAAM,MAAM,2BAA2B,GAAG;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,gBACf,SAAQ,UAAU,CAAC,uBAAuB,CAAC,EACzC,wBAAwB,CAAC,gBAAgB,CAAC;IAE5C,UAAU,CAAC,SAAS,SAAS,KAAK,EAChC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,iBAAiB,CAAC,MAAM,CAAC,GAAG,2BAA2B,GAChE,OAAO,CAAC,gBAAgB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC;IACjE,cAAc,CAAC,SAAS,SAAS,KAAK,EACpC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,SAAS,EAAE,EACnB,OAAO,CAAC,EAAE,6BAA6B,GACtC,OAAO,CAAC,sCAAsC,CAAC,CAAC;IACnD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,MAAM,EAAE;QACN,GAAG,IAAI,MAAM,CAAC;QACd,KAAK,IAAI,IAAI,CAAC;QACd,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,MAAM,uBAAuB,GAAG;IACpC,kCAAkC;IAClC,iBAAiB,CAAC,EAAE;QAClB,iDAAiD;QACjD,EAAE,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;KACtB,CAAC;IACF,gCAAgC;IAChC,MAAM,CAAC,EAAE;QACP,8BAA8B;QAC9B,aAAa,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAAC;KAC3C,CAAC;IACF,iCAAiC;IACjC,KAAK,CAAC,EAAE;QACN,qCAAqC;QACrC,oBAAoB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;KACnD,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,uBAIlC,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAI,MAAM,YAAY,KAAG,gBAiKxD,CAAC"}