@requence/event-sourcing 1.4.1 → 1.5.0

package/src/createCheckpointApi.ts

@@ -18,6 +18,14 @@ const checkpointSchema = z.object({
 
 export type Checkpoint = z.output<typeof checkpointSchema>
 
+/**
+ * The outcome of {@link CheckpointApi.claim}: either this instance owns the
+ * event and must run its work, or another instance already has it.
+ */
+export type ClaimResult =
+  | { claimed: false }
+  | { claimed: true; release: () => Promise<void> }
+
 export type CheckpointMethods = {
   get: (
     type: 'projection' | 'processManager',
@@ -57,12 +65,16 @@ export function createCheckpointApi(
   // every load/upsert so optimistic-concurrency writes can detect when another
   // instance has advanced the same checkpoint.
   let version: number | null = null
+  // The position that goes with `version`. Only {@link CheckpointApi.claim}
+  // needs it, to put the cursor back if the work it claimed then fails.
+  let position: number | null = null
 
   const api = {
     relatedStreamEvents: params.relatedStreamEvents,
     async clear() {
       await params.delete(params.type, params.name)
       version = null
+      position = null
     },
     /**
      * Load the persisted checkpoint and sync the local version. Returns the
@@ -72,8 +84,93 @@ export function createCheckpointApi(
     async load() {
       const checkpoint = await params.get(params.type, params.name)
       version = checkpoint?.version ?? null
+      position = checkpoint?.lastEventPosition ?? null
       return checkpoint ?? null
     },
+    /**
+     * Takes ownership of `event` *before* its work runs, so that exactly one
+     * instance does that work.
+     *
+     * Every instance runs its own copy of a projection against shared storage,
+     * and each decides what it has already applied from a cursor only it can
+     * see. Two instances catching up over the same backlog — or one catching up
+     * while the other applies an event it just appended — therefore both reach
+     * the same handler. Writing the cursor first, through the same
+     * compare-and-swap the ordinary write uses, is what breaks the tie: the
+     * loser is told so and skips the event instead of repeating its side
+     * effects.
+     *
+     * Returns `{ claimed: false }` when another writer already covers this
+     * position. On success the caller gets a `release` to call if the work then
+     * fails, which puts the cursor back so the event is picked up again rather
+     * than silently skipped — without it, claiming would turn every failed
+     * handler into a lost event.
+     */
+    async claim(
+      event: BaseOutputEvent,
+      metadata: Record<string, any> = {},
+    ): Promise<ClaimResult> {
+      while (true) {
+        const expectedVersion = version
+        const previousPosition = position
+        const nextVersion = (expectedVersion ?? -1) + 1
+
+        const committed = await params.upsert(
+          {
+            lastEventPosition: event.position,
+            type: params.type,
+            name: params.name,
+            metadata,
+            version: nextVersion,
+          },
+          expectedVersion,
+        )
+
+        if (committed) {
+          version = nextVersion
+          position = event.position
+
+          return {
+            claimed: true,
+            async release() {
+              // Only undo a claim that is still the newest write. Once anything
+              // else has moved the cursor, rewinding it would drop that work.
+              if (version !== nextVersion) {
+                return
+              }
+
+              if (previousPosition === null) {
+                await api.clear()
+                return
+              }
+
+              const restored = await params.upsert(
+                {
+                  lastEventPosition: previousPosition,
+                  type: params.type,
+                  name: params.name,
+                  metadata,
+                  version: nextVersion + 1,
+                },
+                nextVersion,
+              )
+
+              if (restored) {
+                version = nextVersion + 1
+                position = previousPosition
+              }
+            },
+          }
+        }
+
+        // Another writer advanced this checkpoint. Refresh our view of it.
+        const current = await api.load()
+
+        if (current && current.lastEventPosition >= event.position) {
+          return { claimed: false }
+        }
+      }
+    },
     /**
      * Persist the checkpoint position (and optional metadata) for the given
      * event.
@@ -108,6 +205,7 @@ export function createCheckpointApi(
 
         if (committed) {
           version = nextVersion
+          position = event.position
           return
         }
 

package/src/createProcessManager.ts

@@ -293,7 +293,9 @@ export function buildProcessManagerCreator(params: {
         await hydrated
       },
       async beginSession() {
-        if (!hasState) return
+        if (!hasState) {
+          return
+        }
         // Load through the checkpoint API so the local version is synced with
         // the store; the subsequent fold relies on it for compare-and-swap.
         const checkpoint = await checkpointApi.load()
@@ -310,7 +312,9 @@ export function buildProcessManagerCreator(params: {
         sessionActive = true
       },
       endSession() {
-        if (!hasState) return
+        if (!hasState) {
+          return
+        }
         state = null
         sessionActive = false
       },
@@ -379,10 +383,7 @@ export function buildProcessManagerCreator(params: {
       async state() {
         await hydrated
         if (!sessionActive && hasState) {
-          const checkpoint = await params.checkpoint.get(
-            'processManager',
-            name,
-          )
+          const checkpoint = await params.checkpoint.get('processManager', name)
           return checkpoint?.metadata?.state ?? clone(initialState)
         }
         return state

package/src/createProjection.ts

@@ -266,86 +266,112 @@ export function buildProjectionCreator(params: {
 
       const runner = (async () => {
         await unlocked
-        await withProjectionInfo({ name, event }, async () => {
-          try {
-            await handler?.(event, {
-              replaying,
-              async replayUntil(position, filterEvents) {
-                if (position >= event.position) {
-                  throw new ProjectionReplayLoopError(
-                    `Event handler on${event.type} of projection ${name} tried to replay to the future`,
-                    { name, eventHandler: `on${event.type}` },
-                    event,
-                  )
+
+        // Take ownership of the event before anything observable happens, so a
+        // peer applying the same event does not apply it twice. `claimed` is
+        // null while replaying: a replay deliberately re-applies everything and
+        // `beginReplay` leaves the stored cursor where it was, so a claim would
+        // read that not-yet-rewound position and skip the whole run.
+        const claimed = isReplaying() ? null : await checkpointApi.claim(event)
+
+        if (claimed && !claimed.claimed) {
+          return
+        }
+
+        const release = claimed?.claimed ? claimed.release : null
+
+        try {
+          await withProjectionInfo({ name, event }, async () => {
+            try {
+              await handler?.(event, {
+                replaying,
+                async replayUntil(position, filterEvents) {
+                  if (position >= event.position) {
+                    throw new ProjectionReplayLoopError(
+                      `Event handler on${event.type} of projection ${name} tried to replay to the future`,
+                      { name, eventHandler: `on${event.type}` },
+                      event,
+                    )
+                  }
+
+                  locks.delete(event.streamId)
+                  return api.replayUntil(event.streamId, position, filterEvents)
+                },
+              })
+            } catch (error) {
+              if (error instanceof ReplaySkipError) {
+                if (isReplaying()) {
+                  return
                 }
 
-                locks.delete(event.streamId)
-                return api.replayUntil(event.streamId, position, filterEvents)
-              },
-            })
-          } catch (error) {
-            if (error instanceof ReplaySkipError) {
-              if (isReplaying()) {
-                return
+                throw new Error(
+                  'ReplaySkipError should never be thrown outside replay',
+                )
               }
 
-              throw new Error(
-                'ReplaySkipError should never be thrown outside replay',
+              if (
+                !(error instanceof Error) ||
+                error instanceof ProjectionReplayLoopError
+              ) {
+                throw error
+              }
+              throw new ProjectionEventHandlerExecutionError(
+                name,
+                `on${event.type}`,
+                event,
+                error,
               )
             }
+            try {
+              await globalEventHandler?.(event)
+            } catch (error) {
+              if (!(error instanceof Error)) {
+                throw error
+              }
 
-            if (
-              !(error instanceof Error) ||
-              error instanceof ProjectionReplayLoopError
-            ) {
-              throw error
-            }
-            throw new ProjectionEventHandlerExecutionError(
-              name,
-              `on${event.type}`,
-              event,
-              error,
-            )
-          }
-          try {
-            await globalEventHandler?.(event)
-          } catch (error) {
-            if (!(error instanceof Error)) {
-              throw error
+              throw new ProjectionEventHandlerExecutionError(
+                name,
+                'GLOBAL',
+                event,
+                error,
+              )
             }
+          })
 
-            throw new ProjectionEventHandlerExecutionError(
-              name,
-              'GLOBAL',
+          if (params.snapshot && snapshotting) {
+            const appliedEvents = await params.snapshot.incrementAppliedCount({
+              projection: name,
+              id: event.streamId,
+            })
+            const snapshot = await snapshotting.internalCapture(
               event,
-              error,
+              appliedEvents,
             )
-          }
-        })
-
-        if (params.snapshot && snapshotting) {
-          const appliedEvents = await params.snapshot.incrementAppliedCount({
-            projection: name,
-            id: event.streamId,
-          })
-          const snapshot = await snapshotting.internalCapture(
-            event,
-            appliedEvents,
-          )
 
-          if (snapshot) {
-            await params.snapshot.put({
-              projectionType: name,
-              projectionId: event.streamId,
-              lastEventPosition: event.position,
-              state: snapshot.state,
-              schemaVersion: snapshot.version,
-              createdAt: new Date(),
-            })
+            if (snapshot) {
+              await params.snapshot.put({
+                projectionType: name,
+                projectionId: event.streamId,
+                lastEventPosition: event.position,
+                state: snapshot.state,
+                schemaVersion: snapshot.version,
+                createdAt: new Date(),
+              })
+            }
           }
+        } catch (error) {
+          // The cursor was moved before this work ran, so it has to go back or
+          // the event would never be retried — a claim must not turn a failing
+          // handler into a silently skipped one.
+          await release?.()
+          throw error
         }
 
-        await checkpointApi.upsert(event)
+        // A claim already wrote the cursor. A replay did not take one, so it
+        // still advances the cursor the ordinary way.
+        if (!release) {
+          await checkpointApi.upsert(event)
+        }
       })()
 
       if (exclusive) {

package/src/memory/index.spec.ts

@@ -0,0 +1,4 @@
+import { runStorageDriverContract } from '../tests/storageDriverContract.ts'
+import { createMemoryStorage } from './index.ts'
+
+runStorageDriverContract('memory', () => createMemoryStorage())

package/src/memory/index.ts

@@ -1,5 +1,9 @@
-import type { AnyAggregateRoot } from '../createAggregateRoot.ts'
-import type { Checkpoint } from '../createCheckpointApi.ts'
+import type {
+  AggregateRootSnapshotMethods,
+  AnyAggregateRoot,
+} from '../createAggregateRoot.ts'
+import type { Checkpoint, CheckpointMethods } from '../createCheckpointApi.ts'
+import type { ProjectionSnapshotMethods } from '../createProjection.ts'
 import {
   type AggregateRootSnapshot,
   ConcurrencyError,
@@ -13,16 +17,20 @@ import type {
   EventStore,
   EventStoreParamsWithAggregateRootSnapshots,
   EventsFromRoot,
+  LoadEvents,
   OnProgress,
 } from '../createEventStore.ts'
 import type { LockCreator } from '../lock.ts'
-import type { BaseOutputEvent, MaybePromise } from '../utilityTypes.js'
+import type {
+  BaseInputEvent,
+  BaseOutputEvent,
+  MaybePromise,
+  StreamId,
+} from '../utilityTypes.js'
 
 export { createAggregateRoot } from '../index.ts'
 
-export type OnEventsAppended = (
-  events: BaseOutputEvent[],
-) => MaybePromise<void>
+export type OnEventsAppended = (events: BaseOutputEvent[]) => MaybePromise<void>
 
 interface MemoryEventStoreParams<Root extends AnyAggregateRoot> {
   aggregateRoots: Root[]
@@ -34,33 +42,26 @@ interface MemoryEventStoreParams<Root extends AnyAggregateRoot> {
   onEventsAppended?: OnEventsAppended
 }
 
-export function createEventStore<const Root extends AnyAggregateRoot>({
-  aggregateRoots,
-  postProcessEvent,
-  onProjectionReplay,
-  onProcessManagerRefresh,
-  onEventsAppended,
-  autoInit,
-  lock,
-}: MemoryEventStoreParams<Root>): EventStore<
-  EventsFromRoot<Root>,
-  Root,
-  true
-> {
+export function createMemoryStorage(params?: {
+  onEventsAppended?: OnEventsAppended
+}): {
+  loadEvents: LoadEvents
+  appendEvents: (
+    streamId: StreamId,
+    events: BaseInputEvent[],
+    expectedVersion: number,
+  ) => BaseOutputEvent[]
+  checkpoint: CheckpointMethods
+  projectionSnapshot: ProjectionSnapshotMethods
+  aggregateRootSnapshot: AggregateRootSnapshotMethods
+} {
   const events: Event[] = []
   const checkpoints = new Set<Checkpoint>()
   const aggregateRootSnapshots = new Map<string, AggregateRootSnapshot>()
   const projectionSnapshots = new Set<ProjectionSnapshot>()
   const projectionAppliedCount = new Map<string, number>()
 
-  return createBaseEventStore({
-    aggregateRoots,
-    postProcessEvent,
-    onProjectionReplay,
-    onProcessManagerRefresh,
-    autoInit,
-    lock,
-
+  return {
     async *loadEvents(select, range) {
       let streamTypes: string[] | null = null
       let streamIds: string[] | null = null
@@ -131,7 +132,7 @@ export function createEventStore<const Root extends AnyAggregateRoot>({
       }))
 
       events.push(...extendedEvents)
-      onEventsAppended?.(extendedEvents)
+      params?.onEventsAppended?.(extendedEvents)
       return extendedEvents
     },
 
@@ -155,7 +156,10 @@ export function createEventStore<const Root extends AnyAggregateRoot>({
         if (persisted) {
           // Compare-and-swap: reject the write when another writer advanced the
           // checkpoint, or when the caller expected to create a fresh one.
-          if (expectedVersion === null || persisted.version !== expectedVersion) {
+          if (
+            expectedVersion === null ||
+            persisted.version !== expectedVersion
+          ) {
             return false
           }
           persisted.lastEventPosition = checkpoint.lastEventPosition
@@ -232,5 +236,27 @@ export function createEventStore<const Root extends AnyAggregateRoot>({
         }
       },
     },
+  }
+}
+
+export function createEventStore<const Root extends AnyAggregateRoot>({
+  aggregateRoots,
+  postProcessEvent,
+  onProjectionReplay,
+  onProcessManagerRefresh,
+  onEventsAppended,
+  autoInit,
+  lock,
+}: MemoryEventStoreParams<Root>): EventStore<EventsFromRoot<Root>, Root, true> {
+  const storage = createMemoryStorage({ onEventsAppended })
+
+  return createBaseEventStore({
+    aggregateRoots,
+    postProcessEvent,
+    onProjectionReplay,
+    onProcessManagerRefresh,
+    autoInit,
+    lock,
+    ...storage,
   }) as any
 }

package/src/surreal/index.spec.ts

@@ -0,0 +1,152 @@
+import { afterAll, describe, expect, it } from 'bun:test'
+import { createNodeEngines } from '@surrealdb/node'
+import { Surreal } from 'surrealdb'
+
+import { ConcurrencyError } from '../errors.ts'
+import { runStorageDriverContract } from '../tests/storageDriverContract.ts'
+import { createSurrealStorage, initSchema } from './index.ts'
+
+// One shared embedded engine for the whole file; isolation comes from a
+// fresh logical database per test (tests run sequentially).
+let connection: Surreal | undefined
+
+async function makeDatabase() {
+  if (!connection) {
+    connection = new Surreal({ engines: { ...createNodeEngines() } })
+    await connection.connect('mem://')
+  }
+  await connection.use({ namespace: 'test', database: crypto.randomUUID() })
+  return connection
+}
+
+afterAll(async () => {
+  await connection?.close()
+})
+
+runStorageDriverContract('surreal', async () => {
+  const database = await makeDatabase()
+  return createSurrealStorage({ database })
+})
+
+describe('surreal driver specifics', () => {
+  it('initializes the schema idempotently', async () => {
+    const database = await makeDatabase()
+    await initSchema(database)
+    await initSchema(database)
+
+    const storage = createSurrealStorage({ database })
+    const persisted = await storage.appendEvents(
+      { stream: 'order', id: 'o1' },
+      [{ type: 'created', payload: {}, metadata: {}, schemaVersion: 1 }],
+      0,
+    )
+    expect(persisted[0].position).toBe(1)
+  })
+
+  it('skips schema creation when initSchema is disabled', async () => {
+    const database = await makeDatabase()
+    const storage = createSurrealStorage({ database, initSchema: false })
+
+    let error: unknown
+    try {
+      await storage.appendEvents(
+        { stream: 'order', id: 'o1' },
+        [{ type: 'created', payload: {}, metadata: {}, schemaVersion: 1 }],
+        0,
+      )
+    } catch (err) {
+      error = err
+    }
+    // Without the SCHEMAFULL tables the counter arithmetic cannot resolve.
+    expect(error).toBeTruthy()
+  })
+
+  it('assigns gapless, monotonically increasing global positions', async () => {
+    const database = await makeDatabase()
+    const storage = createSurrealStorage({ database })
+
+    await storage.appendEvents(
+      { stream: 'order', id: 'o1' },
+      [
+        { type: 'created', payload: {}, metadata: {}, schemaVersion: 1 },
+        { type: 'updated', payload: {}, metadata: {}, schemaVersion: 1 },
+      ],
+      0,
+    )
+    const second = await storage.appendEvents(
+      { stream: 'order', id: 'o2' },
+      [{ type: 'created', payload: {}, metadata: {}, schemaVersion: 1 }],
+      0,
+    )
+
+    expect(second[0].position).toBe(3)
+  })
+
+  it('serializes concurrent appends to different streams without losing positions', async () => {
+    const database = await makeDatabase()
+    const storage = createSurrealStorage({ database })
+
+    const results = await Promise.all(
+      Array.from({ length: 8 }, (_, index) =>
+        storage.appendEvents(
+          { stream: 'order', id: `o${index}` },
+          [{ type: 'created', payload: {}, metadata: {}, schemaVersion: 1 }],
+          0,
+        ),
+      ),
+    )
+
+    const positions = results
+      .map(([event]) => event.position)
+      .toSorted((a, b) => a - b)
+    expect(positions).toEqual([1, 2, 3, 4, 5, 6, 7, 8])
+  })
+
+  it('reports the current version in ConcurrencyError', async () => {
+    const database = await makeDatabase()
+    const storage = createSurrealStorage({ database })
+
+    await storage.appendEvents(
+      { stream: 'order', id: 'o1' },
+      [
+        { type: 'created', payload: {}, metadata: {}, schemaVersion: 1 },
+        { type: 'updated', payload: {}, metadata: {}, schemaVersion: 1 },
+      ],
+      0,
+    )
+
+    let error: unknown
+    try {
+      await storage.appendEvents(
+        { stream: 'order', id: 'o1' },
+        [{ type: 'updated', payload: {}, metadata: {}, schemaVersion: 1 }],
+        1,
+      )
+    } catch (err) {
+      error = err
+    }
+
+    expect(error).toBeInstanceOf(ConcurrencyError)
+    expect((error as Error).message).toContain('expected v1, actual v2')
+  })
+
+  it('notifies onEventsAppended with the persisted events', async () => {
+    const database = await makeDatabase()
+    let appended: unknown[] = []
+    const storage = createSurrealStorage({
+      database,
+      onEventsAppended: (events) => {
+        appended = events
+      },
+    })
+
+    await storage.appendEvents(
+      { stream: 'order', id: 'o1' },
+      [{ type: 'created', payload: { n: 1 }, metadata: {}, schemaVersion: 1 }],
+      0,
+    )
+
+    expect(appended).toHaveLength(1)
+    expect((appended[0] as any).position).toBe(1)
+  })
+})