effect-cqrs 0.0.0-stage → 0.1.0

package/src/AggregateService.ts

@@ -0,0 +1,342 @@
+/**
+ * Runs an aggregate against the {@link EventStore}: the `command`, `read-model`
+ * and `history` actions of moleculer-cqrs, as a typed service.
+ */
+import { Array as Arr, type DateTime, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
+import * as Aggregate from "./Aggregate.ts"
+import {
+  ConcurrencyConflict,
+  type EventFilter,
+  EventStore,
+  type EventStoreError,
+  type StoredEvent
+} from "./EventStore.ts"
+
+export class UnknownCommand extends Schema.TaggedError<UnknownCommand>()("UnknownCommand", {
+  aggregateName: Schema.String,
+  type: Schema.String
+}) {}
+
+export class InvalidCommand extends Schema.TaggedError<InvalidCommand>()("InvalidCommand", {
+  aggregateName: Schema.String,
+  type: Schema.String,
+  message: Schema.String
+}) {}
+
+/** A stored event no longer matches the schema, which means the data needs a migration. */
+export class EventDecodeError extends Schema.TaggedError<EventDecodeError>()("EventDecodeError", {
+  aggregateName: Schema.String,
+  aggregateId: Schema.String,
+  version: Schema.Int,
+  type: Schema.String,
+  message: Schema.String
+}) {}
+
+export type ReadError = EventStoreError | EventDecodeError
+
+/** How a command can fail, on top of the errors its own handler declares. */
+export type CommandFailure = ConcurrencyConflict | InvalidCommand | ReadError
+
+export interface CommandResult<Event> {
+  readonly aggregateName: string
+  readonly aggregateId: string
+  /** The stream version after the command. */
+  readonly version: number
+  /** The store position of the last event, to pass to `ViewModels.caughtUp`. */
+  readonly position: number
+  readonly events: ReadonlyArray<Event>
+}
+
+export interface HistoryEntry<Event> {
+  readonly version: number
+  readonly timestamp: DateTime.Utc
+  readonly type: string
+  readonly payload?: Event
+}
+
+export interface AggregateService<A extends Aggregate.Any> {
+  readonly aggregate: A
+  /** One typed function per command. */
+  readonly commands: {
+    readonly [K in Aggregate.CommandName<A>]: (
+      aggregateId: string,
+      payload: Aggregate.Payload<A, K>
+    ) => Effect.Effect<CommandResult<Aggregate.Event<A>>, Aggregate.CommandError<A, K> | CommandFailure>
+  }
+  /** Runs a `{ aggregateId, type, payload }` envelope received as JSON from outside the process. */
+  readonly dispatch: (
+    input: unknown
+  ) => Effect.Effect<CommandResult<Aggregate.Event<A>>, Aggregate.CommandError<A> | CommandFailure | UnknownCommand>
+  /** Materializes the state on the fly, optionally as it was before `finishTime`. */
+  readonly readModel: (
+    aggregateId: string,
+    options?: { readonly finishTime?: DateTime.Utc | undefined }
+  ) => Effect.Effect<Aggregate.State<A>, ReadError>
+  /** Lists the events of one aggregate, with their decoded payload when `payload` is set. */
+  readonly history: (
+    aggregateId: string,
+    options?: {
+      readonly payload?: boolean | undefined
+      readonly startTime?: DateTime.Utc | undefined
+      readonly finishTime?: DateTime.Utc | undefined
+    }
+  ) => Effect.Effect<ReadonlyArray<HistoryEntry<Aggregate.Event<A>>>, ReadError>
+}
+
+const CommandEnvelope = Schema.Struct({
+  aggregateId: Schema.NonEmptyString,
+  type: Schema.String,
+  payload: Schema.Unknown
+})
+
+/** Decodes a stored event with the schema of the aggregate it belongs to. */
+export const makeEventDecoder = <A extends Aggregate.Any>(aggregate: A) => {
+  const decode = Schema.decodeUnknownEffect(Schema.toCodecJson(aggregate.events))
+  return (stored: StoredEvent): Effect.Effect<Aggregate.Event<A>, EventDecodeError> =>
+    decode(stored.payload).pipe(
+      Effect.mapError((error) =>
+        new EventDecodeError({
+          aggregateName: stored.aggregateName,
+          aggregateId: stored.aggregateId,
+          version: stored.version,
+          type: stored.type,
+          message: error.message
+        })
+      )
+    ) as Effect.Effect<Aggregate.Event<A>, EventDecodeError>
+}
+
+/** Encodes events into the JSON form the store keeps. */
+export const makeEventEncoder = <A extends Aggregate.Any>(aggregate: A) => {
+  const encode = Schema.encodeUnknownEffect(Schema.toCodecJson(aggregate.events))
+  return (event: Aggregate.Event<A>) =>
+    encode(event).pipe(
+      // Events come from the aggregate's own handlers, so this is a programming error.
+      Effect.orDie,
+      Effect.map((payload) => ({ type: event._tag, payload: payload as Schema.Json }))
+    )
+}
+
+interface Loaded<State> {
+  readonly state: State
+  readonly version: number
+  readonly position: number
+}
+
+/** Remembers recent states, dropping the least recently used. A `Map` iterates in insertion order. */
+const makeStateCache = <State>(capacity: number) => {
+  const entries = new Map<string, Loaded<State>>()
+  return {
+    get: (aggregateId: string) => {
+      const entry = entries.get(aggregateId)
+      if (entry !== undefined) {
+        entries.delete(aggregateId)
+        entries.set(aggregateId, entry)
+      }
+      return entry
+    },
+    set: (aggregateId: string, loaded: Loaded<State>) => {
+      if (capacity === 0 || (entries.get(aggregateId)?.position ?? 0) > loaded.position) return
+      entries.delete(aggregateId)
+      entries.set(aggregateId, loaded)
+      if (entries.size > capacity) entries.delete(entries.keys().next().value!)
+    }
+  }
+}
+
+/** Runs effects with the same key one at a time. A key only exists while it is in use. */
+const makeKeyedLock = () => {
+  const locks = new Map<string, { readonly semaphore: Semaphore.Semaphore; users: number }>()
+  return (key: string) => <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    Effect.acquireUseRelease(
+      Effect.sync(() => {
+        const lock = locks.get(key) ?? { semaphore: Semaphore.makeUnsafe(1), users: 0 }
+        lock.users++
+        locks.set(key, lock)
+        return lock
+      }),
+      (lock) => lock.semaphore.withPermit(effect),
+      (lock) =>
+        Effect.sync(() => {
+          if (--lock.users === 0) locks.delete(key)
+        })
+    )
+}
+
+/**
+ * Gives other fibers a turn once in a while. `Effect.forEach` with `concurrency`
+ * carries on as long as items don't suspend, and a store on `node:sqlite` never does,
+ * so a bulk run would finish before any view model ran. Yielding per command is
+ * expensive, so it only happens when a millisecond has passed, using real time
+ * because this is about the CPU. The clock restarts after the yield rather than
+ * before it, so commands started meanwhile suspend too and the loop lets go.
+ */
+const makeTimeSlice = () => {
+  let started = performance.now()
+  return Effect.suspend(() =>
+    performance.now() - started < 1
+      ? Effect.void
+      : Effect.andThen(Effect.yieldNow, Effect.sync(() => void (started = performance.now())))
+  )
+}
+
+// Only a rival outside this service can cause a conflict, so give it room to finish.
+const conflictBackoff = Schedule.exponential("10 millis").pipe(Schedule.jittered)
+
+export const make = Effect.fnUntraced(function*<A extends Aggregate.Any>(
+  aggregate: A,
+  options?: {
+    /** How many times a command is re-decided after losing a race. Defaults to 5. */
+    readonly maxRetries?: number
+    /** How many aggregates keep their state in memory between commands. Defaults to 10,000. */
+    readonly stateCacheSize?: number
+  }
+) {
+  const store = yield* EventStore
+  const context = yield* Effect.context<Aggregate.Context<A>>()
+  const maxRetries = options?.maxRetries ?? 5
+  const aggregateName = aggregate.name
+  const decodeEvent = makeEventDecoder(aggregate)
+  const encodeEvent = makeEventEncoder(aggregate)
+  const states = makeStateCache<Aggregate.State<A>>(options?.stateCacheSize ?? 10_000)
+  const lockStream = makeKeyedLock()
+  const timeSlice = makeTimeSlice()
+
+  const replay = <Acc>(
+    stream: Stream.Stream<StoredEvent, EventStoreError>,
+    initial: Acc,
+    step: (acc: Acc, event: Aggregate.Event<A>, stored: StoredEvent) => Acc
+  ) =>
+    stream.pipe(
+      Stream.runFoldEffect(
+        () => initial,
+        (acc, stored) => Effect.map(decodeEvent(stored), (event) => step(acc, event, stored))
+      )
+    )
+
+  const streamOf = (
+    aggregateId: string,
+    range?: Pick<EventFilter, "startTime" | "finishTime" | "afterPosition" | "afterVersion">
+  ) => store.read({ aggregateNames: [aggregateName], aggregateIds: [aggregateId], ...range })
+
+  const replayFrom = (
+    start: Loaded<Aggregate.State<A>>,
+    aggregateId: string,
+    finishTime?: DateTime.Utc
+  ) =>
+    replay(
+      streamOf(aggregateId, { finishTime, afterPosition: start.position, afterVersion: start.version }),
+      start,
+      (acc, event, stored) => ({
+        state: Aggregate.evolve(aggregate, acc.state, event, stored),
+        version: stored.version,
+        position: stored.position
+      })
+    )
+
+  const initial: Loaded<Aggregate.State<A>> = { state: aggregate.initial as Aggregate.State<A>, version: 0, position: 0 }
+
+  // Streams only grow, so a remembered state plus what came after it is the current
+  // state, even if another process wrote those events.
+  const load = (aggregateId: string, finishTime?: DateTime.Utc) =>
+    finishTime === undefined
+      ? replayFrom(states.get(aggregateId) ?? initial, aggregateId).pipe(
+        Effect.tap((loaded) => Effect.sync(() => states.set(aggregateId, loaded)))
+      )
+      : replayFrom(initial, aggregateId, finishTime)
+
+  const decide = Effect.fnUntraced(function*(type: string, aggregateId: string, payload: unknown) {
+    const { position, state, version } = yield* load(aggregateId)
+    const { handler } = aggregate.commands[type]!
+    const events: ReadonlyArray<Aggregate.Event<A>> = Arr.ensure(yield* handler(state, payload, { aggregateId, version }))
+    if (events.length === 0) {
+      return { aggregateName, aggregateId, version, position, events }
+    }
+    const stored = yield* store.append({
+      aggregateName,
+      aggregateId,
+      expectedVersion: version,
+      events: yield* Effect.forEach(events, encodeEvent)
+    })
+    const last = stored[stored.length - 1]!
+    return { aggregateName, aggregateId, version: last.version, position: last.position, events }
+  })
+
+  // Commands on one stream take turns, so they can't race each other and only a
+  // rival in another process causes a conflict. A handler must not run a command on
+  // its own stream, which would wait for itself.
+  const execute = (type: string, aggregateId: string, payload: unknown) =>
+    decide(type, aggregateId, payload).pipe(
+      Effect.retry({
+        times: maxRetries,
+        while: (error) => error instanceof ConcurrencyConflict,
+        schedule: conflictBackoff
+      }),
+      Effect.provideContext(context),
+      lockStream(aggregateId),
+      (decided) => Effect.andThen(timeSlice, decided),
+      Effect.withSpan(`${aggregateName}.${type}`, { attributes: { "aggregate.id": aggregateId } })
+    )
+
+  const decodePayload = (type: string, schema: Aggregate.PayloadSchema, input: unknown) =>
+    Schema.decodeUnknownEffect(schema)(input).pipe(
+      Effect.mapError((error) => new InvalidCommand({ aggregateName, type, message: error.message }))
+    )
+
+  const commands = Object.fromEntries(
+    Object.entries(aggregate.commands).map(([type, command]) => [
+      type,
+      (aggregateId: string, payload: unknown) =>
+        decodePayload(type, Schema.toType(command.payload), payload).pipe(
+          Effect.flatMap((decoded) => execute(type, aggregateId, decoded))
+        )
+    ])
+  )
+
+  const decodeEnvelope = Schema.decodeUnknownEffect(CommandEnvelope)
+
+  const dispatch = (input: unknown) =>
+    Effect.gen(function*() {
+      const envelope = yield* decodeEnvelope(input).pipe(
+        Effect.mapError((error) => new InvalidCommand({ aggregateName, type: "unknown", message: error.message }))
+      )
+      if (!Object.hasOwn(aggregate.commands, envelope.type)) {
+        return yield* new UnknownCommand({ aggregateName, type: envelope.type })
+      }
+      const payload = yield* decodePayload(
+        envelope.type,
+        Schema.toCodecJson(aggregate.commands[envelope.type]!.payload),
+        envelope.payload
+      )
+      return yield* execute(envelope.type, envelope.aggregateId, payload)
+    })
+
+  const readModel: AggregateService<A>["readModel"] = (aggregateId, options) =>
+    load(aggregateId, options?.finishTime).pipe(
+      Effect.map(({ state }) => state),
+      Effect.withSpan(`${aggregateName}.readModel`, { attributes: { "aggregate.id": aggregateId } })
+    )
+
+  const history: AggregateService<A>["history"] = (aggregateId, options) =>
+    replay(
+      streamOf(aggregateId, { startTime: options?.startTime, finishTime: options?.finishTime }),
+      [] as Array<HistoryEntry<Aggregate.Event<A>>>,
+      (entries, event, stored) => {
+        entries.push({
+          version: stored.version,
+          timestamp: stored.timestamp,
+          type: stored.type,
+          ...(options?.payload === true ? { payload: event } : {})
+        })
+        return entries
+      }
+    ).pipe(Effect.withSpan(`${aggregateName}.history`, { attributes: { "aggregate.id": aggregateId } }))
+
+  return {
+    aggregate,
+    commands,
+    dispatch,
+    readModel,
+    history
+  } as unknown as AggregateService<A>
+})

package/src/Checkpoints.ts

@@ -0,0 +1,23 @@
+/**
+ * Remembers how far each view model has read.
+ *
+ * A checkpoint has to live as long as the view model it tracks: pair an
+ * in-memory view model with {@link layerMemory} so it rebuilds from the event
+ * store on restart, and a durable one with a durable checkpoint store.
+ */
+import { Context, Effect, Layer } from "effect"
+import type { EventStoreError } from "./EventStore.ts"
+
+export class Checkpoints extends Context.Service<Checkpoints, {
+  /** The last processed position, `0` when the view model never ran. */
+  get(viewModel: string): Effect.Effect<number, EventStoreError>
+  set(viewModel: string, position: number): Effect.Effect<void, EventStoreError>
+}>()("effect-cqrs/Checkpoints") {}
+
+export const layerMemory: Layer.Layer<Checkpoints> = Layer.sync(Checkpoints, () => {
+  const positions = new Map<string, number>()
+  return Checkpoints.of({
+    get: (viewModel) => Effect.sync(() => positions.get(viewModel) ?? 0),
+    set: (viewModel, position) => Effect.sync(() => void positions.set(viewModel, position))
+  })
+})

package/src/EventStore.ts

@@ -0,0 +1,233 @@
+/**
+ * The append-only log every aggregate and view model is built on.
+ *
+ * Each event belongs to a stream, identified by aggregate name and id, and has
+ * two coordinates: a `version` that orders it inside its stream and a `position`
+ * that orders it across the whole store. Writers use the version for optimistic
+ * concurrency, readers use the position as a durable cursor.
+ */
+import { Context, DateTime, Effect, Layer, Option, Ref, Schema, Stream, SubscriptionRef } from "effect"
+
+export class EventStoreError extends Schema.TaggedError<EventStoreError>()("EventStoreError", {
+  operation: Schema.String,
+  cause: Schema.Defect()
+}) {}
+
+export class ConcurrencyConflict extends Schema.TaggedError<ConcurrencyConflict>()("ConcurrencyConflict", {
+  aggregateName: Schema.String,
+  aggregateId: Schema.String,
+  expectedVersion: Schema.Int
+}) {}
+
+export const StoredEvent = Schema.Struct({
+  position: Schema.Int,
+  aggregateName: Schema.String,
+  aggregateId: Schema.String,
+  version: Schema.Int,
+  type: Schema.String,
+  payload: Schema.Json,
+  timestamp: Schema.DateTimeUtcFromMillis
+})
+export type StoredEvent = typeof StoredEvent.Type
+
+export interface NewEvent {
+  readonly type: string
+  readonly payload: Schema.Json
+}
+
+export interface AppendInput {
+  readonly aggregateName: string
+  readonly aggregateId: string
+  /** The stream version the writer based its decision on, `0` for a new stream. */
+  readonly expectedVersion: number
+  readonly events: ReadonlyArray<NewEvent>
+}
+
+/**
+ * Selects events by stream, type and time. Omitted fields match everything.
+ * `startTime` is inclusive and `finishTime` is exclusive, so adding one
+ * millisecond to a history timestamp selects up to and including that event.
+ */
+export interface EventFilter {
+  readonly aggregateNames?: ReadonlyArray<string> | undefined
+  readonly aggregateIds?: ReadonlyArray<string> | undefined
+  readonly eventTypes?: ReadonlyArray<string> | undefined
+  readonly startTime?: DateTime.Utc | undefined
+  readonly finishTime?: DateTime.Utc | undefined
+  readonly afterPosition?: number | undefined
+  /**
+   * For a filter on one stream: where to continue in it. Versions follow the stream's
+   * own index, so resuming there costs the same however far the log has moved on.
+   */
+  readonly afterVersion?: number | undefined
+}
+
+/** A filter on exactly one stream, which a store can read through that stream's index. */
+export const isOneStream = ({ aggregateIds, aggregateNames }: EventFilter): boolean =>
+  aggregateNames?.length === 1 && aggregateIds?.length === 1
+
+export class EventStore extends Context.Service<EventStore, {
+  /** Appends atomically, failing if the stream moved past `expectedVersion`. */
+  append(input: AppendInput): Effect.Effect<ReadonlyArray<StoredEvent>, ConcurrencyConflict | EventStoreError>
+  /** Reads the events stored so far, ordered by position, then completes. */
+  read(filter?: EventFilter): Stream.Stream<StoredEvent, EventStoreError>
+  /** Reads the events stored so far, then keeps following new ones. Never completes. */
+  subscribe(filter?: EventFilter): Stream.Stream<StoredEvent, EventStoreError>
+  /** The position of the latest event, `0` when the store is empty. */
+  readonly head: Effect.Effect<number, EventStoreError>
+  /**
+   * Emits the current latest position, then each newly appended one. Consumers
+   * can treat an element as "there may be something new": appends from other
+   * processes are not observed.
+   */
+  readonly changes: Stream.Stream<number>
+}>()("effect-cqrs/EventStore") {}
+
+/** What a storage adapter implements. {@link make} builds the rest on top. */
+export interface Backend {
+  readonly append: EventStore["Service"]["append"]
+  readonly readBatch: (
+    filter: EventFilter,
+    afterPosition: number,
+    limit: number
+  ) => Effect.Effect<ReadonlyArray<StoredEvent>, EventStoreError>
+  readonly head: Effect.Effect<number, EventStoreError>
+}
+
+export const make = Effect.fnUntraced(function*(backend: Backend, options?: { readonly batchSize?: number }) {
+  const batchSize = options?.batchSize ?? 500
+  const latest = yield* SubscriptionRef.make(yield* backend.head)
+
+  const append = Effect.fn("EventStore.append")((input: AppendInput) =>
+    backend.append(input).pipe(
+      Effect.tap((stored) => {
+        const last = stored.at(-1)
+        return last === undefined
+          ? Effect.void
+          : SubscriptionRef.update(latest, (position) => Math.max(position, last.position))
+      })
+    )
+  )
+
+  const read = (filter: EventFilter = {}) =>
+    Stream.paginate(
+      { afterPosition: filter.afterPosition ?? 0, afterVersion: filter.afterVersion ?? 0 },
+      ({ afterPosition, afterVersion }) =>
+        backend.readBatch({ ...filter, afterVersion }, afterPosition, batchSize).pipe(
+          Effect.map((events) => {
+            const last = events.at(-1)
+            return [
+              events,
+              last === undefined || events.length < batchSize
+                ? Option.none()
+                : Option.some({ afterPosition: last.position, afterVersion: last.version })
+            ] as const
+          })
+        )
+    )
+
+  const subscribe = (filter: EventFilter = {}) =>
+    Stream.unwrap(Effect.gen(function*() {
+      const cursor = yield* Ref.make({ position: filter.afterPosition ?? 0, version: filter.afterVersion ?? 0 })
+      return SubscriptionRef.changes(latest).pipe(
+        Stream.filterEffect((position) => Effect.map(Ref.get(cursor), (seen) => position > seen.position)),
+        Stream.flatMap((position) =>
+          Stream.unwrap(
+            Effect.map(Ref.get(cursor), (seen) =>
+              read({ ...filter, afterPosition: seen.position, afterVersion: seen.version }))
+          ).pipe(
+            Stream.tap((event) => Ref.set(cursor, { position: event.position, version: event.version })),
+            // Events the filter skipped must not trigger another read.
+            Stream.ensuring(
+              Ref.update(cursor, (seen) => seen.position >= position ? seen : { ...seen, position })
+            )
+          )
+        )
+      )
+    }))
+
+  return EventStore.of({
+    append,
+    read,
+    subscribe,
+    head: backend.head,
+    changes: SubscriptionRef.changes(latest)
+  })
+})
+
+const matches = (filter: EventFilter, event: StoredEvent): boolean =>
+  (filter.aggregateNames?.includes(event.aggregateName) ?? true) &&
+  (filter.aggregateIds?.includes(event.aggregateId) ?? true) &&
+  (filter.eventTypes?.includes(event.type) ?? true) &&
+  (filter.startTime === undefined || !DateTime.isLessThan(event.timestamp, filter.startTime)) &&
+  (filter.finishTime === undefined || DateTime.isLessThan(event.timestamp, filter.finishTime))
+
+/** The index of the first event past `position`, in events sorted by position. */
+const firstAfter = (events: ReadonlyArray<StoredEvent>, position: number): number => {
+  let low = 0
+  let high = events.length
+  while (low < high) {
+    const middle = (low + high) >>> 1
+    if (events[middle]!.position <= position) low = middle + 1
+    else high = middle
+  }
+  return low
+}
+
+const streamKey = (aggregateName: string, aggregateId: string) => JSON.stringify([aggregateName, aggregateId])
+
+export const makeMemory: Effect.Effect<EventStore["Service"], EventStoreError> = Effect.gen(function*() {
+  const events: Array<StoredEvent> = []
+  const streams = new Map<string, Array<StoredEvent>>()
+
+  // Every command reads one stream, which shouldn't cost a walk through the whole log.
+  const candidates = (filter: EventFilter): ReadonlyArray<StoredEvent> =>
+    isOneStream(filter)
+      ? streams.get(streamKey(filter.aggregateNames![0]!, filter.aggregateIds![0]!)) ?? []
+      : events
+
+  return yield* make({
+    append: Effect.fnUntraced(function*(input) {
+      const timestamp = yield* DateTime.now
+      const key = streamKey(input.aggregateName, input.aggregateId)
+      const stream = streams.get(key) ?? []
+      const version = stream.length
+      if (version !== input.expectedVersion) {
+        return yield* new ConcurrencyConflict({
+          aggregateName: input.aggregateName,
+          aggregateId: input.aggregateId,
+          expectedVersion: input.expectedVersion
+        })
+      }
+      const stored = input.events.map((event, index): StoredEvent => ({
+        position: events.length + index + 1,
+        aggregateName: input.aggregateName,
+        aggregateId: input.aggregateId,
+        version: version + index + 1,
+        type: event.type,
+        payload: event.payload,
+        timestamp
+      }))
+      for (const event of stored) {
+        events.push(event)
+        stream.push(event)
+      }
+      streams.set(key, stream)
+      return stored
+    }),
+    readBatch: (filter, afterPosition, limit) =>
+      Effect.sync(() => {
+        const source = candidates(filter)
+        const batch: Array<StoredEvent> = []
+        for (let index = firstAfter(source, afterPosition); index < source.length && batch.length < limit; index++) {
+          const event = source[index]!
+          if (matches(filter, event)) batch.push(event)
+        }
+        return batch
+      }),
+    head: Effect.sync(() => events.length)
+  })
+})
+
+/** Keeps events in process memory. They do not survive a restart. */
+export const layerMemory: Layer.Layer<EventStore, EventStoreError> = Layer.effect(EventStore, makeMemory)