effect-cqrs 0.0.0-stage → 0.2.0

package/src/Saga.ts

@@ -0,0 +1,176 @@
+/**
+ * Sagas: reactions that span several aggregates and outlive one command, run as
+ * durable workflows from `effect/workflow`.
+ *
+ * A saga has two halves. Its trigger is a view model that follows the log from a
+ * checkpoint and starts a workflow for each event it cares about. The workflow is
+ * the process itself: each command it runs is an `Activity`, so when the engine
+ * replays the workflow after a restart it skips the commands that already
+ * finished, and `withCompensation` undoes earlier steps when a later one fails.
+ *
+ * Triggers see an event at least once, and the workflow's idempotency key turns
+ * a second start into a join of the first execution. That only holds while the
+ * engine remembers the execution, so pair `WorkflowEngine.layerMemory` with
+ * in-memory checkpoints, as for view models, and a durable engine such as
+ * `ClusterWorkflowEngine` from `effect/cluster` with durable ones.
+ *
+ * An activity that is cut off before its result is saved runs again, so the
+ * commands a saga runs should be idempotent, for instance by recording the id of
+ * the transfer they belong to.
+ */
+import { Effect, Layer, Schedule, Schema } from "effect"
+import { Activity, Workflow } from "effect/workflow"
+import type { WorkflowEngine } from "effect/workflow"
+import type * as Aggregate from "./Aggregate.ts"
+import type { CommandResult } from "./AggregateService.ts"
+import * as Causation from "./Causation.ts"
+import type { Checkpoints } from "./Checkpoints.ts"
+import type { EventStore, EventStoreError } from "./EventStore.ts"
+import * as ViewModel from "./ViewModel.ts"
+
+/**
+ * The causation a workflow carries in its payload. A workflow body runs in the
+ * engine's fibers, not in the trigger's, so this is how commands it runs learn
+ * which event caused them.
+ */
+export const CausationSchema = Schema.Struct({
+  correlationId: Schema.String,
+  causationId: Schema.String
+})
+
+/** A workflow whose payload carries the causation of the event that started it. */
+export type SagaWorkflow<
+  Tag extends string = string,
+  Fields extends Schema.Struct.Fields = Schema.Struct.Fields,
+  Success extends Schema.Top = Schema.Top,
+  Error extends Schema.Top = Schema.Top
+> = Workflow.Workflow<Tag, Schema.Struct<Fields & { readonly causation: typeof CausationSchema }>, Success, Error>
+
+/**
+ * Defines the workflow of a saga. It is `Workflow.make` with a `causation` field
+ * added to the payload, which {@link start} fills in and {@link toLayer} applies.
+ */
+export const workflow = <
+  const Tag extends string,
+  const Fields extends Schema.Struct.Fields,
+  Success extends Schema.Top = Schema.Void,
+  Error extends Schema.Top = Schema.Never
+>(tag: Tag, options: {
+  readonly payload: Fields
+  /** Identifies one run of the saga, such as a transfer id. Starting it again joins that run. */
+  readonly idempotencyKey: (payload: Schema.Struct.Type<Fields>) => string
+  readonly success?: Success
+  readonly error?: Error
+  readonly suspendedRetrySchedule?: Schedule.Schedule<any, unknown> | undefined
+}): SagaWorkflow<Tag, Fields, Success, Error> =>
+  Workflow.make(tag, {
+    ...options,
+    payload: { ...options.payload, causation: CausationSchema }
+  }) as any
+
+/**
+ * Registers the body of a saga's workflow with the engine. Commands the body
+ * runs record the event that started the saga as their cause.
+ */
+export const toLayer = <
+  Tag extends string,
+  Fields extends Schema.Struct.Fields,
+  Success extends Schema.Top,
+  Error extends Schema.Top,
+  R
+>(
+  self: SagaWorkflow<Tag, Fields, Success, Error>,
+  execute: (
+    payload: Schema.Struct.Type<Fields>,
+    executionId: string
+  ) => Effect.Effect<Success["Type"], Error["Type"], R>
+) =>
+  self.toLayer((payload, executionId) =>
+    Causation.withCausation(
+      execute(payload as Schema.Struct.Type<Fields>, executionId),
+      (payload as { readonly causation: Causation.Causation }).causation
+    )
+  )
+
+/**
+ * Starts a saga's workflow without waiting for it, and returns its execution id.
+ * Inside a trigger the event being handled becomes the saga's cause; elsewhere
+ * the current causation does, or a new correlation when there is none.
+ */
+export const start = <Tag extends string, Fields extends Schema.Struct.Fields, Success extends Schema.Top, Error extends Schema.Top>(
+  self: SagaWorkflow<Tag, Fields, Success, Error>,
+  payload: Schema.Struct.MakeIn<Fields>
+): Effect.Effect<string, never, WorkflowEngine.WorkflowEngine> =>
+  Causation.CurrentCausation.use((current) => {
+    const id = crypto.randomUUID()
+    const causation = current ?? { correlationId: id, causationId: id }
+    return self.execute({ ...payload, causation } as never, { discard: true })
+  }) as any
+
+/**
+ * The trigger of a saga: a view model whose handlers start workflows with
+ * {@link start}, so it keeps a checkpoint and retries like view models do. Run it
+ * with {@link layer}, or with `ViewModel.layer` next to the other view models,
+ * where replaying it starts every saga again and joins the executions the engine
+ * still remembers.
+ */
+export const make = <
+  const Name extends string,
+  const Aggregates extends ReadonlyArray<Aggregate.Any>,
+  const H extends ViewModel.Handlers<Aggregate.Event<Aggregates[number]>>
+>(options: {
+  readonly name: Name
+  readonly aggregates: Aggregates
+  readonly handlers: H
+}) => ViewModel.make({ ...options, dispose: Effect.void })
+
+/**
+ * Runs triggers in the background for as long as the layer lives, apart from
+ * the view models. Workflows usually run commands, so they sit on top of the
+ * command side, which view models are often provided to.
+ */
+export const layer = <const Sagas extends ReadonlyArray<ViewModel.Any>>(
+  sagas: Sagas
+): Layer.Layer<never, EventStoreError, EventStore | Checkpoints | ViewModel.Context<Sagas[number]>> =>
+  Layer.provide(Layer.empty, ViewModel.layer(sagas))
+
+/** What a saga keeps of a command it ran: enough to wait for view models to see it. */
+export const CommandOutcome = Schema.Struct({
+  aggregateId: Schema.String,
+  version: Schema.Int,
+  position: Schema.Int
+})
+
+// Losing a race or a store error usually passes, so try again before giving up.
+const transient = (error: unknown) =>
+  Schema.is(Schema.Struct({ _tag: Schema.Literals(["ConcurrencyConflict", "EventStoreError"]) }))(error)
+
+const transientRetry = Schedule.exponential("50 millis").pipe(Schedule.jittered)
+
+/**
+ * Runs a command as an activity of a saga's workflow, so a replay of the
+ * workflow returns the saved outcome instead of running it again.
+ *
+ * Failures matching `error`, such as a domain error the saga reacts to, are
+ * saved like a success. A lost race or a store error is retried a few times.
+ * Anything else ends the workflow as a defect, which runs its compensations.
+ */
+export const command = <Err extends Schema.Top = typeof Schema.Never, E = never, R = never>(
+  name: string,
+  options: {
+    readonly execute: Effect.Effect<CommandResult<unknown>, E, R>
+    readonly error?: Err | undefined
+  }
+): Activity.Activity<typeof CommandOutcome, Err, R> => {
+  const expected = options.error === undefined ? () => false : Schema.is(Schema.toType(options.error))
+  return Activity.make({
+    name,
+    success: CommandOutcome,
+    error: options.error,
+    execute: options.execute.pipe(
+      Effect.retry({ while: transient, schedule: transientRetry, times: 5 }),
+      Effect.map(({ aggregateId, position, version }) => ({ aggregateId, version, position })),
+      Effect.catch((error) => expected(error) ? Effect.fail(error as Err["Type"]) : Effect.die(error))
+    )
+  }) as any
+}

package/src/SqliteEventStore.ts

@@ -0,0 +1,196 @@
+/**
+ * Durable storage on SQLite, through any `effect/sql` SQLite driver.
+ *
+ * The tables are created on startup. The schema relies on SQLite's single writer
+ * for positions to follow commit order, so it is not portable to other databases.
+ */
+import { Clock, DateTime, Effect, Layer, Schema } from "effect"
+import { SqlClient, type SqlError } from "effect/sql"
+import { Checkpoints } from "./Checkpoints.ts"
+import * as EventStore from "./EventStore.ts"
+
+export interface Options {
+  readonly eventsTable?: string
+  readonly checkpointsTable?: string
+}
+
+const EventRow = Schema.Struct({
+  ...EventStore.StoredEvent.fields,
+  payload: Schema.fromJsonString(Schema.Json),
+  metadata: Schema.fromJsonString(Schema.JsonObject)
+})
+
+const decodeRows = Schema.decodeUnknownEffect(Schema.Array(EventRow))
+
+const storeError = (operation: string) => (cause: unknown) => new EventStore.EventStoreError({ operation, cause })
+
+const isUniqueViolation = (error: SqlError.SqlError) => error.reason._tag === "UniqueViolation"
+
+const addedColumns: ReadonlyArray<readonly [string, string]> = [
+  ["schema_version", "INTEGER NOT NULL DEFAULT 1"],
+  ["correlation_id", "TEXT"],
+  ["causation_id", "TEXT"],
+  ["metadata", "TEXT NOT NULL DEFAULT '{}'"]
+]
+
+export const make = Effect.fnUntraced(function*(options?: Options) {
+  const sql = yield* SqlClient.SqlClient
+  const events = sql(options?.eventsTable ?? "cqrs_events")
+  const indexPrefix = options?.eventsTable ?? "cqrs_events"
+
+  yield* sql`
+    CREATE TABLE IF NOT EXISTS ${events} (
+      position INTEGER PRIMARY KEY AUTOINCREMENT,
+      aggregate_name TEXT NOT NULL,
+      aggregate_id TEXT NOT NULL,
+      version INTEGER NOT NULL,
+      type TEXT NOT NULL,
+      schema_version INTEGER NOT NULL DEFAULT 1,
+      payload TEXT NOT NULL,
+      timestamp INTEGER NOT NULL,
+      correlation_id TEXT,
+      causation_id TEXT,
+      metadata TEXT NOT NULL DEFAULT '{}',
+      UNIQUE (aggregate_name, aggregate_id, version)
+    )
+  `.pipe(Effect.mapError(storeError("initialize")))
+  // Tables created before these columns existed get them here. SQLite has no
+  // `ADD COLUMN IF NOT EXISTS`, so the existing columns are listed first.
+  const existing = yield* sql<{ readonly name: string }>`PRAGMA table_info(${events})`.pipe(
+    Effect.map((rows) => new Set(rows.map(({ name }) => name))),
+    Effect.mapError(storeError("initialize"))
+  )
+  yield* Effect.forEach(
+    addedColumns.filter(([name]) => !existing.has(name)),
+    ([name, definition]) =>
+      sql`ALTER TABLE ${events} ADD COLUMN ${sql(name)} ${sql.literal(definition)}`.pipe(
+        Effect.mapError(storeError("initialize"))
+      ),
+    { discard: true }
+  )
+  yield* sql`
+    CREATE INDEX IF NOT EXISTS ${sql(`${indexPrefix}_type_idx`)} ON ${events} (type, position)
+  `.pipe(Effect.mapError(storeError("initialize")))
+
+  const columns = sql.literal(`
+    position,
+    aggregate_name AS "aggregateName",
+    aggregate_id AS "aggregateId",
+    version,
+    type,
+    schema_version AS "schemaVersion",
+    payload,
+    timestamp,
+    correlation_id AS "correlationId",
+    causation_id AS "causationId",
+    metadata
+  `)
+
+  const append = (input: EventStore.AppendInput) => {
+    const conflict = new EventStore.ConcurrencyConflict({
+      aggregateName: input.aggregateName,
+      aggregateId: input.aggregateId,
+      expectedVersion: input.expectedVersion
+    })
+    return Effect.gen(function*() {
+      const [current] = yield* sql<{ readonly version: number }>`
+        SELECT COALESCE(MAX(version), 0) AS version FROM ${events}
+        WHERE aggregate_name = ${input.aggregateName} AND aggregate_id = ${input.aggregateId}
+      `
+      if (current?.version !== input.expectedVersion) {
+        return yield* conflict
+      }
+      const timestamp = yield* Clock.currentTimeMillis
+      const metadata = JSON.stringify(input.metadata ?? {})
+      return yield* Effect.forEach(input.events, (event, index) =>
+        sql<{ readonly position: number }>`
+          INSERT INTO ${events} (
+            aggregate_name, aggregate_id, version, type, schema_version, payload, timestamp, correlation_id, causation_id,
+            metadata
+          )
+          VALUES (
+            ${input.aggregateName},
+            ${input.aggregateId},
+            ${input.expectedVersion + index + 1},
+            ${event.type},
+            ${event.schemaVersion ?? 1},
+            ${JSON.stringify(event.payload)},
+            ${timestamp},
+            ${input.correlationId ?? null},
+            ${input.causationId ?? null},
+            ${metadata}
+          )
+          RETURNING position
+        `.pipe(
+          Effect.map(([row]) => EventStore.toStored(input, event, index, row!.position, DateTime.makeUnsafe(timestamp)))
+        ))
+    }).pipe(
+      sql.withTransaction,
+      // The version check can't see a writer racing it on a driver that opens
+      // deferred transactions, but the UNIQUE constraint can.
+      Effect.catchTag("SqlError", (error) => Effect.fail(isUniqueViolation(error) ? conflict : storeError("append")(error)))
+    )
+  }
+
+  const readBatch = (filter: EventStore.EventFilter, afterPosition: number, limit: number) => {
+    const oneStream = EventStore.isOneStream(filter)
+    const clauses = [
+      sql`position > ${afterPosition}`,
+      ...(oneStream ? [sql`version > ${filter.afterVersion ?? 0}`] : []),
+      ...(filter.aggregateNames === undefined ? [] : [sql.in("aggregate_name", filter.aggregateNames)]),
+      ...(filter.aggregateIds === undefined ? [] : [sql.in("aggregate_id", filter.aggregateIds)]),
+      ...(filter.eventTypes === undefined ? [] : [sql.in("type", filter.eventTypes)]),
+      ...(filter.startTime === undefined ? [] : [sql`timestamp >= ${DateTime.toEpochMillis(filter.startTime)}`]),
+      ...(filter.finishTime === undefined ? [] : [sql`timestamp < ${DateTime.toEpochMillis(filter.finishTime)}`])
+    ]
+    // Inside a stream versions follow positions, and only versions are indexed.
+    const order = oneStream ? sql`version` : sql`position`
+    return sql`SELECT ${columns} FROM ${events} WHERE ${sql.and(clauses)} ORDER BY ${order} LIMIT ${limit}`.pipe(
+      Effect.flatMap(decodeRows),
+      Effect.mapError(storeError("read"))
+    )
+  }
+
+  const head = sql<{ readonly position: number }>`SELECT COALESCE(MAX(position), 0) AS position FROM ${events}`.pipe(
+    Effect.map(([row]) => row?.position ?? 0),
+    Effect.mapError(storeError("head"))
+  )
+
+  return yield* EventStore.make({ append, readBatch, head })
+})
+
+export const layer = (options?: Options): Layer.Layer<EventStore.EventStore, EventStore.EventStoreError, SqlClient.SqlClient> =>
+  Layer.effect(EventStore.EventStore, make(options))
+
+/** Keeps checkpoints in the same database, for view models that are durable too. */
+export const layerCheckpoints = (options?: Options): Layer.Layer<Checkpoints, EventStore.EventStoreError, SqlClient.SqlClient> =>
+  Layer.effect(
+    Checkpoints,
+    Effect.gen(function*() {
+      const sql = yield* SqlClient.SqlClient
+      const table = sql(options?.checkpointsTable ?? "cqrs_checkpoints")
+
+      yield* sql`
+        CREATE TABLE IF NOT EXISTS ${table} (
+          view_model TEXT PRIMARY KEY,
+          position INTEGER NOT NULL
+        )
+      `.pipe(Effect.mapError(storeError("initialize")))
+
+      return Checkpoints.of({
+        get: (viewModel) =>
+          sql<{ readonly position: number }>`SELECT position FROM ${table} WHERE view_model = ${viewModel}`.pipe(
+            Effect.map(([row]) => row?.position ?? 0),
+            Effect.mapError(storeError("checkpoint"))
+          ),
+        set: (viewModel, position) =>
+          sql`
+            INSERT INTO ${table} (view_model, position) VALUES (${viewModel}, ${position})
+            ON CONFLICT (view_model) DO UPDATE SET position = excluded.position
+          `.pipe(
+            Effect.asVoid,
+            Effect.mapError(storeError("checkpoint"))
+          )
+      })
+    })
+  )