@photon-grove/evt 0.1.2 → 0.2.0

package/src/dynamodb/requests.ts

@@ -1,14 +1,15 @@
 // The `TransactWriteItems` actions of a commit and of a snapshot write, built from the storage
 // format alone (storage format, "Writes"). Every commit opens with the two control checks, the
-// feature marker's and the rebuild lock's, then its projector operations, then writes the stream:
-// its head update, its event puts, and its snapshot put when it has one.
+// feature marker's and the rebuild lock's, then its projector operations, then writes each stream:
+// its head update, its event puts, and its snapshot put when it has one. A transaction then checks
+// the head of every stream it fenced.
 
 import type {SerializedEvent, SerializedSnapshot} from '../event.js'
 import {int64Argument, safeIntegerArgument} from '../fields.js'
 import {EvtError} from '../errors.js'
-import type {Commit} from '../repository.js'
+import type {Commit, NormalizedTransaction, ProjectorGroup, ReadFence} from '../repository.js'
 import type {TransactItem} from './client.js'
-import {type HeadAction, headUpdate} from './head.js'
+import {type HeadAction, headFence, headUpdate} from './head.js'
 import {HEAD_SORT_KEY, type Item, SNAPSHOT_SORT_KEY, eventItem, snapshotItem} from './item.js'
 import {FEATURES_PK, REBUILD_LOCK_PK} from './marker.js'
 import {DYNAMODB_BACKEND} from './views.js'
@@ -102,8 +103,7 @@ export interface SnapshotTtl {
 /**
  * The actions of a commit with at least one event, in order, by a writer that validated the
  * feature manifest at `markerVersion`. `ttl` is given only for a stream with retention. A projector
- * group for any backend but DynamoDB is refused. Exported for the compatibility fixtures: its
- * signature changes when commits span streams.
+ * group for any backend but DynamoDB is refused. Exported for the compatibility fixtures.
  */
 export function commitTransaction(
   table: string,
@@ -111,16 +111,83 @@ export function commitTransaction(
   markerVersion: bigint,
   ttl?: CommitTtl,
 ): TransactItem[] {
-  const first = commit.events[0]
-  const last = commit.events.at(-1)
-  if (first === undefined || last === undefined) {
+  if (commit.events.length === 0) {
     throw new RangeError('a commit transaction needs at least one event')
   }
 
+  const transaction = {streams: [commit], fences: [], projections: commit.projections ?? []}
+
+  return transactionItems(table, transaction, markerVersion, () => ttl)
+}
+
+/** Where each stream's and fence's actions sit in a transaction, to classify a cancellation. */
+export interface TransactionLayout {
+  /** The index after the control checks and the projector operations. */
+  readonly projectorEnd: number
+  readonly streams: readonly StreamLayout[]
+  readonly fences: readonly {readonly index: number; readonly fence: ReadFence}[]
+}
+
+/** A written stream's actions in a transaction. */
+export interface StreamLayout {
+  readonly commit: Commit
+  /** The head update's index. Its event puts follow it. */
+  readonly head: number
+  /** The index after its last event put. */
+  readonly eventsEnd: number
+  /** The snapshot put's index, when it writes one. */
+  readonly snapshot: number | null
+}
+
+/**
+ * The actions of a normalized transaction that writes or fences at least one stream, in order:
+ * the control checks, the projector operations in declaration order, every written stream's head
+ * update, event puts and snapshot put, then a check on every fenced stream's head. `ttl` gives a
+ * written stream's `ttl`s when it has retention. Exported for the compatibility fixtures.
+ */
+export function transactionItems(
+  table: string,
+  transaction: NormalizedTransaction,
+  markerVersion: bigint,
+  ttl: (commit: Commit) => CommitTtl | undefined,
+): TransactItem[] {
+  return buildTransaction(table, transaction, markerVersion, ttl).items
+}
+
+/** {@link transactionItems}, with where each stream's and fence's actions sit. */
+export function buildTransaction(
+  table: string,
+  transaction: NormalizedTransaction,
+  markerVersion: bigint,
+  ttl: (commit: Commit) => CommitTtl | undefined,
+): {items: TransactItem[]; layout: TransactionLayout} {
   const items = [markerCheck(table, markerVersion), rebuildLockCheck(table)]
-  items.push(...projectorItems(commit))
+  items.push(...projectorItems(transaction.projections))
+  const projectorEnd = items.length
+
+  const streams: StreamLayout[] = []
+  for (const commit of transaction.streams) {
+    const head = items.length
+    items.push(...streamItems(table, commit, ttl(commit)))
+    const eventsEnd = head + 1 + commit.events.length
+    streams.push({commit, head, eventsEnd, snapshot: items.length > eventsEnd ? eventsEnd : null})
+  }
+
+  const fences = transaction.fences.map((fence) => {
+    items.push(fenceCheck(table, fence))
+
+    return {index: items.length - 1, fence}
+  })
+
+  return {items, layout: {projectorEnd, streams, fences}}
+}
+
+// A written stream's actions: its head update, its event puts and its snapshot put.
+function streamItems(table: string, commit: Commit, ttl: CommitTtl | undefined): TransactItem[] {
+  const first = commit.events[0]!
+  const last = commit.events.at(-1)!
   const head = headUpdate(commit.entityId, first.sequence, last.sequence, ttl?.head)
-  items.push(headUpdateItem(table, head))
+  const items = [headUpdateItem(table, head)]
   for (const event of commit.events) {
     items.push(eventPut(table, event, ttl?.items))
   }
@@ -140,6 +207,23 @@ export function commitTransaction(
   return items
 }
 
+// The check on a fenced stream's head item: it still has the head the transaction read.
+function fenceCheck(table: string, fence: ReadFence): TransactItem {
+  const action = headFence(fence.entityId, fence.head)
+  const check: NonNullable<TransactItem['ConditionCheck']> = {
+    TableName: table,
+    Key: headKey(fence.entityId),
+    ConditionExpression: action.conditionExpression,
+  }
+  if (action.values.length > 0) {
+    check.ExpressionAttributeValues = Object.fromEntries(
+      action.values.map(([placeholder, value]) => [placeholder, number(value)]),
+    )
+  }
+
+  return {ConditionCheck: check}
+}
+
 /**
  * The actions of a snapshot written outside a commit: the marker check, the head guard, and the
  * put, which never lowers the stored snapshot. It carries no rebuild-lock check. `ttl` is given
@@ -204,10 +288,10 @@ function snapshotGuard(
   }
 }
 
-/** The DynamoDB actions of a commit's projector groups, in order. */
-export function projectorItems(commit: Commit): TransactItem[] {
+/** The DynamoDB actions of projector groups, in order. */
+export function projectorItems(groups: readonly ProjectorGroup[]): TransactItem[] {
   const items: TransactItem[] = []
-  for (const group of commit.projections ?? []) {
+  for (const group of groups) {
     if (group.backend !== DYNAMODB_BACKEND) {
       throw new EvtError(
         'invalidCommit',

package/src/errors.ts

@@ -15,6 +15,10 @@
  * - `conflict`: optimistic concurrency failed. Another writer committed since the stream was
  *   read. Reload and run the command again.
  * - `sequenceGap`: a commit or a load skips a sequence. A conflict.
+ * - `fenceConflict`: another writer moved a stream the transaction read but didn't write. A
+ *   conflict, retryable when the engine observed the fenced head.
+ * - `retentionConflict`: a transaction fenced a stream whose entity type has retention, which a
+ *   fence can't protect. Nothing was written.
  * - `streamHeadDrift`: a stream's head disagrees with its events or snapshot. Never retried.
  * - `commitTooLarge`: the commit is over the transaction budget. Nothing was written.
  * - `compactionUncovered`: no snapshot covers the events compaction was asked to delete.
@@ -42,6 +46,8 @@ export type EvtErrorCode =
   | 'invalidCommit'
   | 'conflict'
   | 'sequenceGap'
+  | 'fenceConflict'
+  | 'retentionConflict'
   | 'streamHeadDrift'
   | 'commitTooLarge'
   | 'compactionUncovered'
@@ -283,6 +289,48 @@ export class RebuildInProgressError extends EvtError {
   }
 }
 
+/** Where a read fence's head came from (ADR 0004): the engine's own read, or the caller. */
+export type FenceProvenance = 'engine' | 'caller'
+
+/**
+ * A read fence failed: another writer moved a stream the transaction read but didn't write past
+ * the head it observed. A fence the engine filled is retryable, since loading again observes the
+ * new head. A caller-stamped one isn't: the caller's input is stale. Nothing was written.
+ */
+export class FenceConflictError extends EvtError {
+  override readonly name = 'FenceConflictError'
+  override readonly code = 'fenceConflict'
+  readonly entityId: string
+  /** The head the transaction fenced the stream at. */
+  readonly head: bigint
+  readonly provenance: FenceProvenance
+
+  constructor(entityId: string, head: bigint, provenance: FenceProvenance) {
+    super('fenceConflict', `read fence failed for ${entityId} at head ${head} (${provenance})`)
+    this.entityId = entityId
+    this.head = head
+    this.provenance = provenance
+  }
+}
+
+/**
+ * A transaction fenced a stream whose entity type has retention: TTL deletes items in no order, so
+ * a fence on a retained stream protects nothing. Nothing was written.
+ */
+export class RetentionConflictError extends EvtError {
+  override readonly name = 'RetentionConflictError'
+  override readonly code = 'retentionConflict'
+  readonly entityType: string
+
+  constructor(entityType: string) {
+    super(
+      'retentionConflict',
+      `read fences refused for retention-enabled entity type ${entityType}`,
+    )
+    this.entityType = entityType
+  }
+}
+
 /**
  * Compaction was refused because the store declares the entity's type history-dependent: its
  * projectors' rebuild replays the events compaction would delete. Nothing was deleted.
@@ -360,16 +408,27 @@ export class SnapshotError extends EvtError {
   }
 }
 
-/** Whether an error is an optimistic-concurrency conflict: a `conflict` or a `sequenceGap`. */
+/**
+ * Whether an error is an optimistic-concurrency conflict: a `conflict`, a `sequenceGap` or a
+ * `fenceConflict`.
+ */
 export function isConflict(error: unknown): boolean {
-  return error instanceof EvtError && (error.code === 'conflict' || error.code === 'sequenceGap')
+  return (
+    error instanceof EvtError &&
+    (error.code === 'conflict' || error.code === 'sequenceGap' || error.code === 'fenceConflict')
+  )
 }
 
 /**
- * Whether reloading and running the command again may succeed: after a conflict, while a rebuild
- * holds the store, after the feature manifest changed under the writer, or after throttling.
+ * Whether reloading and running the command again may succeed: after a conflict (a fence the
+ * caller stamped excepted), while a rebuild holds the store, after the feature manifest changed
+ * under the writer, or after throttling.
  */
 export function isRetryable(error: unknown): boolean {
+  // A caller-stamped fence stays stale however often the command runs again.
+  if (error instanceof FenceConflictError) {
+    return error.provenance === 'engine'
+  }
   if (isConflict(error)) {
     return true
   }

package/src/index.ts

@@ -33,11 +33,14 @@ export {
   EvtError,
   type EvtErrorCode,
   FeatureManifestChangedError,
+  FenceConflictError,
+  type FenceProvenance,
   HistoryDependentProjectionError,
   ProjectorOnlyCommitError,
   RebuildInProgressError,
   ReplayError,
   ReservedEntityIdError,
+  RetentionConflictError,
   SequenceGapError,
   SnapshotError,
   StoreNotInitializedError,
@@ -76,13 +79,18 @@ export {
   emptyMetadata,
   encodeMetadata,
 } from './metadata.js'
-export type {
-  CallOptions,
-  Commit,
-  NewSnapshot,
-  ProjectorGroup,
-  Repository,
-  StoreInitializer,
+export {
+  type CallOptions,
+  type Commit,
+  type NewSnapshot,
+  type NormalizedTransaction,
+  type ProjectorGroup,
+  type ReadFence,
+  type Repository,
+  type StoreInitializer,
+  type TransactionCommit,
+  type TransactionalRepository,
+  normalizeTransaction,
 } from './repository.js'
 export {
   DEFAULT_RETRY_POLICY,
@@ -91,6 +99,7 @@ export {
 } from './retry.js'
 export {MAX_COMMIT_ITEM_BYTES, snapshotCommitSize, snapshotEventsDue} from './snapshots.js'
 export {compareUtf8, utf8Length} from './utf8.js'
+export {type CallerFence, StreamRead, Transaction, type TransactionResult} from './transaction.js'
 export {
   DEFAULT_SNAPSHOT_INTERVAL,
   type ExecuteOptions,

package/src/memory/repository.ts

@@ -7,6 +7,7 @@ import {
   ConflictError,
   CompactionUncoveredError,
   EvtError,
+  FenceConflictError,
   HistoryDependentProjectionError,
   RebuildInProgressError,
   SequenceGapError,
@@ -27,11 +28,17 @@ import type {Metadata} from '../metadata.js'
 import {
   type CallOptions,
   type Commit,
-  type Repository,
+  type NormalizedTransaction,
+  type ReadFence,
   type StoreInitializer,
+  type TransactionCommit,
+  type TransactionalRepository,
   checkBudget,
+  checkFenceRetention,
   checkNotReserved,
-  commitSizes,
+  normalizeTransaction,
+  rankFailures,
+  transactionSizes,
   validateCommit,
 } from '../repository.js'
 
@@ -75,7 +82,7 @@ let controlsOf: (repo: MemRepository) => Controls
  * on its feature manifest, so {@link initialize} is needed only where code treats every backend
  * alike.
  */
-export class MemRepository implements Repository, StoreInitializer {
+export class MemRepository implements TransactionalRepository, StoreInitializer {
   readonly #streams = new Map<string, Stream>()
   readonly #controls: Controls = {manifest: null, rebuild: {generation: 0n, lock: null}}
 
@@ -95,44 +102,80 @@ export class MemRepository implements Repository, StoreInitializer {
     options?.signal?.throwIfAborted()
     validateCommit(commit)
     checkNotReserved(commit.entityId)
-    if (commit.events.length === 0) {
-      return
+    if (commit.events.length > 0) {
+      this.#write(normalizeTransaction({streams: [commit]}))
     }
+  }
+
+  async commitTransaction(transaction: TransactionCommit, options?: CallOptions): Promise<void> {
+    options?.signal?.throwIfAborted()
+    const normalized = normalizeTransaction(transaction)
+    if (normalized.streams.length > 0 || normalized.fences.length > 0) {
+      this.#write(normalized)
+    }
+  }
 
+  // Writes a transaction, or refuses it with nothing written, in the order the live backends
+  // check: the budget, the marker and the fences' retention, the rebuild lock, then every stream
+  // and fence, ranked.
+  #write(transaction: NormalizedTransaction): void {
     // Every backend budgets a commit the same way, so an oversized one fails before any write.
-    checkBudget(commitSizes(commit, () => undefined))
+    checkBudget(transactionSizes(transaction, () => undefined))
 
     // The control rows come first, as the two checks that open every live commit: the marker,
     // then the rebuild lock, which an unsettled generation fails, a force-released one included.
     this.#checkManifest()
+    const retention = this.#controls.manifest?.retention ?? []
+    checkFenceRetention(transaction, (entityType) => retention.includes(entityType))
     const {rebuild} = this.#controls
     if (rebuild.lock !== null || rebuild.generation % 2n !== 0n) {
       throw new RebuildInProgressError()
     }
 
-    const held = this.#streams.get(commit.entityId) ?? emptyStream()
-    checkWrite(commit, held)
-
-    // Copy everything before the first write, so a copy that throws leaves the stream as it was.
-    const events = commit.events.map(cloneEvent)
-    const last = events.at(-1)!.sequence
-    const snapshot = commit.snapshot && {
-      entityType: commit.snapshot.entityType,
-      entityId: commit.entityId,
-      sequence: commit.snapshot.sequence,
-      eventSequence: last,
-      payload: copy(commit.snapshot.payload),
+    const failures: EvtError[] = []
+    for (const stream of transaction.streams) {
+      const failure = checkWrite(stream, this.#streams.get(stream.entityId) ?? emptyStream())
+      if (failure !== null) {
+        failures.push(failure)
+      }
     }
-
-    for (const event of events) {
-      held.events.set(event.sequence, event)
+    for (const fence of transaction.fences) {
+      const failure = checkFence(fence, this.#streams.get(fence.entityId) ?? emptyStream())
+      if (failure !== null) {
+        failures.push(failure)
+      }
     }
-    held.head = held.head > last ? held.head : last
-    if (snapshot) {
-      held.snapshot = snapshot
+    const failure = rankFailures(failures)
+    if (failure !== undefined) {
+      throw failure
     }
 
-    this.#streams.set(commit.entityId, held)
+    // Copy everything before the first write, so a copy that throws leaves every stream as it was.
+    const writes = transaction.streams.map((commit) => {
+      const events = commit.events.map(cloneEvent)
+      const last = events.at(-1)!.sequence
+      const snapshot = commit.snapshot && {
+        entityType: commit.snapshot.entityType,
+        entityId: commit.entityId,
+        sequence: commit.snapshot.sequence,
+        eventSequence: last,
+        payload: copy(commit.snapshot.payload),
+      }
+
+      return {entityId: commit.entityId, events, last, snapshot}
+    })
+
+    for (const {entityId, events, last, snapshot} of writes) {
+      const held = this.#streams.get(entityId) ?? emptyStream()
+      for (const event of events) {
+        held.events.set(event.sequence, event)
+      }
+      held.head = held.head > last ? held.head : last
+      if (snapshot) {
+        held.snapshot = snapshot
+      }
+      this.#streams.set(entityId, held)
+    }
   }
 
   async events(entityId: string, options?: CallOptions): Promise<SerializedEvent[]> {
@@ -272,29 +315,47 @@ export class MemRepository implements Repository, StoreInitializer {
   }
 }
 
-// Refuses a commit the stored stream doesn't accept, as the live backends' conditions do. The
-// head update needs no head for a create and the head to be the sequence before the first event
-// for an append. Then no event may exist already, and a snapshot must follow the stored one.
-function checkWrite(commit: Commit, held: Stream): void {
+// The failure of a commit the stored stream doesn't accept, as the live backends' conditions find
+// it, or null. The head update needs no head for a create and the head to be the sequence before
+// the first event for an append. Then no event may exist already, and a snapshot must follow the
+// stored one.
+function checkWrite(commit: Commit, held: Stream): EvtError | null {
   const first = commit.events[0]!.sequence
   const headPasses = first <= 1n ? held.head === 0n : held.head === first - 1n
   if (!headPasses) {
-    throw probe(commit.entityId, held, first)
+    return probe(commit.entityId, held, first)
   }
 
   // The head update passed and an event put would fail: an event sits above the head.
   if (commit.events.some((event) => held.events.has(event.sequence))) {
-    throw drift(commit.entityId, held.head, derivedHead(held))
+    return drift(commit.entityId, held.head, derivedHead(held))
   }
 
   const snapshot = commit.snapshot
   const stored = held.snapshot?.sequence ?? 0n
   if (snapshot && snapshot.sequence !== stored + 1n) {
-    throw new ConflictError(
+    return new ConflictError(
       commit.entityId,
       `snapshot sequence moved: stored ${stored}, commit writes ${snapshot.sequence}`,
     )
   }
+
+  return null
+}
+
+// The failure of a fence whose stream's head moved, or null: drift when the head disagrees with
+// the log, and otherwise a fence conflict.
+function checkFence(fence: ReadFence, held: Stream): EvtError | null {
+  if (held.head === fence.head) {
+    return null
+  }
+
+  const derived = derivedHead(held)
+  if (held.head !== derived) {
+    return drift(fence.entityId, held.head, derived)
+  }
+
+  return new FenceConflictError(fence.entityId, fence.head, fence.provenance)
 }
 
 // Classifies a failed head update the way the live backends' failure-path probe does: a stored