@photon-grove/evt 0.1.1 → 0.2.0

package/src/dynamodb/repository.ts

@@ -14,6 +14,7 @@ import {
   ConflictError,
   EvtError,
   FeatureManifestChangedError,
+  FenceConflictError,
   HistoryDependentProjectionError,
   RebuildInProgressError,
   SequenceGapError,
@@ -34,12 +35,17 @@ import {int64Argument} from '../fields.js'
 import {
   type CallOptions,
   type Commit,
-  type Repository,
+  type NormalizedTransaction,
   type StoreInitializer,
+  type TransactionCommit,
+  type TransactionalRepository,
   checkBudget,
+  checkFenceRetention,
   checkNotReserved,
-  commitSizes,
   isReserved,
+  normalizeTransaction,
+  rankFailures,
+  transactionSizes,
   validateCommit,
 } from '../repository.js'
 import type {DynamoClient, Page, RequestOptions, TransactItem} from './client.js'
@@ -54,11 +60,15 @@ import {
   ttlFromItem,
 } from './item.js'
 import {FEATURES_PK, manifestFromItem, markerItem} from './marker.js'
+import {DYNAMODB_BACKEND, transactItemSize} from './views.js'
 import {
   type CommitTtl,
-  commitTransaction,
+  type StreamLayout,
+  type TransactionLayout,
   controlKey,
   headKey,
+  projectorItems,
+  buildTransaction,
   snapshotTransaction,
 } from './requests.js'
 
@@ -119,7 +129,7 @@ const MAX_SNAPSHOT_GUARD_RETRIES = 4
  * declares a feature, the next write fails, validates the manifest again, and either refuses an
  * unsupported feature or fails once with a retryable {@link FeatureManifestChangedError}.
  */
-export class DynamoRepository implements Repository, StoreInitializer {
+export class DynamoRepository implements TransactionalRepository, StoreInitializer {
   readonly #client: DynamoClient
   readonly #table: string
   readonly #lazy: boolean
@@ -185,24 +195,64 @@ export class DynamoRepository implements Repository, StoreInitializer {
     options?.signal?.throwIfAborted()
     validateCommit(commit)
     checkNotReserved(commit.entityId)
-    if (commit.events.length === 0) {
-      return
+    if (commit.events.length > 0) {
+      await this.#write(normalizeTransaction({streams: [commit]}), options)
+    }
+  }
+
+  /**
+   * Commits every stream and checks every fence in one `TransactWriteItems` call: the control
+   * checks, the projector operations, each written stream's actions, then a check on each fenced
+   * stream's head. A fence on a type this repository retains is refused before any request, and
+   * one on a type the store's manifest retains after the marker's validation.
+   */
+  async commitTransaction(transaction: TransactionCommit, options?: CallOptions): Promise<void> {
+    options?.signal?.throwIfAborted()
+    const normalized = normalizeTransaction(transaction)
+    if (normalized.streams.length > 0 || normalized.fences.length > 0) {
+      await this.#write(normalized, options)
     }
+  }
+
+  async #write(transaction: NormalizedTransaction, options?: CallOptions): Promise<void> {
+    // One `ttl` per entity type for the whole transaction, so a stream's head's is never below one
+    // of its own items.
+    const ttls = new Map<string, number | undefined>()
+    const ttlFor = (entityType: string): number | undefined => {
+      if (!ttls.has(entityType)) {
+        ttls.set(entityType, this.#ttlFor(entityType))
+      }
 
-    // One `ttl` for the whole commit, so its head's is never below one of its own items.
-    const entityType = commit.events[0]!.entityType
-    const ttl = this.#ttlFor(entityType)
+      return ttls.get(entityType)
+    }
 
     // Every backend budgets a commit the same way, so an oversized one fails before any request.
-    checkBudget(commitSizes(commit, (type) => (type === entityType ? ttl : undefined)))
+    // The projector operations are sized from the actions this repository sends, which refuses a
+    // group for another backend here too.
+    const sizes = projectorItems(transaction.projections).map(transactItemSize)
+    const projections = [{backend: DYNAMODB_BACKEND, operations: sizes, sizes}]
+    checkBudget(transactionSizes({...transaction, projections}, ttlFor))
+
+    // A fence on a type this repository retains needs no request to refuse.
+    checkFenceRetention(transaction, (entityType) => this.#retention.has(entityType))
 
     const request = requestOptions(options)
     const version = await this.#markerVersion(request)
-    const items = commitTransaction(this.#table, commit, version, this.#commitTtl(commit, ttl))
+    if (transaction.fences.length > 0) {
+      // The store's manifest may retain a type this repository doesn't. The marker check holds it
+      // at the version read here, or fails the transaction.
+      const manifest = await this.#readManifest(request)
+      const retention = manifest?.retention ?? []
+      checkFenceRetention(transaction, (entityType) => retention.includes(entityType))
+    }
+
+    const {items, layout} = buildTransaction(this.#table, transaction, version, (commit) =>
+      this.#commitTtl(commit, ttlFor(commit.events[0]!.entityType)),
+    )
     try {
       await this.#client.transactWriteItems({TransactItems: items}, request)
     } catch (error) {
-      throw await this.#classifyCommit(commit, error, request)
+      throw await this.#classifyCommit(layout, error, request)
     }
   }
 
@@ -290,6 +340,12 @@ export class DynamoRepository implements Repository, StoreInitializer {
           if (codes?.some((code) => THROTTLED.has(code))) {
             throw new TransientError('the snapshot write was throttled', {cause: error})
           }
+          if (codes?.includes('TransactionConflict')) {
+            throw new TransientError(
+              `transaction conflict on ${snapshot.entityId}: another writer was committing at the same time`,
+              {cause: error},
+            )
+          }
 
           throw error
         }
@@ -513,7 +569,14 @@ export class DynamoRepository implements Repository, StoreInitializer {
     return new FeatureManifestChangedError({cause})
   }
 
-  async #classifyCommit(commit: Commit, error: unknown, request: RequestOptions): Promise<unknown> {
+  // Maps a cancelled transaction to the error it reports. The control checks outrank everything.
+  // Then every failed stream and fence is classified, and the ranking reports the first drift, else
+  // the first gap, else the first other failure, in transaction order.
+  async #classifyCommit(
+    layout: TransactionLayout,
+    error: unknown,
+    request: RequestOptions,
+  ): Promise<unknown> {
     const codes = cancellationCodes(error)
     if (codes === null) {
       return error
@@ -521,10 +584,14 @@ export class DynamoRepository implements Repository, StoreInitializer {
 
     const failed = codes.indexOf(CONDITION_FAILED)
     if (failed === -1) {
-      if (codes.includes('TransactionConflict')) {
-        return new ConflictError(
-          commit.entityId,
-          'transaction conflict: another writer was committing at the same time',
+      // Another transaction was writing one of the items, perhaps one this commit never changes
+      // (a control check): nothing was written and nothing is known to be stale, so it is
+      // transient, like throttling, and never a conflict, as in Go and Rust.
+      const colliding = codes.indexOf('TransactionConflict')
+      if (colliding !== -1) {
+        return new TransientError(
+          `transaction conflict on ${entityAt(layout, colliding)}: another writer was committing at the same time`,
+          {cause: error},
         )
       }
       if (codes.some((code) => THROTTLED.has(code))) {
@@ -542,15 +609,57 @@ export class DynamoRepository implements Repository, StoreInitializer {
       return new RebuildInProgressError()
     }
 
-    // The stream's head update, then its event puts and its snapshot put.
-    const head = 2
-    const eventsEnd = head + 1 + commit.events.length
-    const eventFailed = codes.slice(head + 1, eventsEnd).includes(CONDITION_FAILED)
+    const failures: EvtError[] = []
+    for (const stream of layout.streams) {
+      const failure = await this.#streamFailure(stream, codes, request)
+      if (failure !== null) {
+        failures.push(failure)
+      }
+    }
+    for (const {index, fence} of layout.fences) {
+      if (codes[index] !== CONDITION_FAILED) {
+        continue
+      }
+
+      const verdict = await this.#probe(fence.entityId, false, request)
+      failures.push(
+        verdict.drift ?? new FenceConflictError(fence.entityId, fence.head, fence.provenance),
+      )
+    }
+
+    const ranked = rankFailures(failures)
+    if (ranked !== undefined) {
+      return ranked
+    }
+
+    // Only a projector operation's condition failed: its own error, as Go reports it.
+    if (failed < layout.projectorEnd) {
+      return error
+    }
+
+    return new ConflictError(
+      entityAt(layout, failed),
+      'conditional check failed: the sequence or snapshot changed since it was read',
+    )
+  }
+
+  // What a written stream contributes to a failed transaction, or null when none of its actions
+  // failed. A failed head update is classified by the probe. A head update that passed while an
+  // event put failed is an event above the head, unless retention explains it. A failed snapshot
+  // put alone is an ordinary conflict.
+  async #streamFailure(
+    stream: StreamLayout,
+    codes: string[],
+    request: RequestOptions,
+  ): Promise<EvtError | null> {
+    const {commit} = stream
     const {entityId} = commit
     const first = commit.events[0]!.sequence
     const retained = this.#retention.has(commit.events[0]!.entityType)
+    const eventFailed = codes.slice(stream.head + 1, stream.eventsEnd).includes(CONDITION_FAILED)
+    const snapshotFailed = stream.snapshot !== null && codes[stream.snapshot] === CONDITION_FAILED
 
-    if (codes[head] === CONDITION_FAILED) {
+    if (codes[stream.head] === CONDITION_FAILED) {
       const verdict = await this.#probe(entityId, retained, request)
       if (verdict.drift) {
         return verdict.drift
@@ -558,20 +667,26 @@ export class DynamoRepository implements Repository, StoreInitializer {
       if (verdict.healthy && first > 1n && verdict.stored < first - 1n) {
         return new SequenceGapError(entityId, verdict.stored, first)
       }
-    } else if (codes[head] === 'None' && eventFailed) {
-      // The head update passed and an event put failed: an event sits above the head. With
-      // retention, TTL can delete the head before an expired event, so a create passes its head
-      // condition and fails on the leftover item: a conflict until TTL deletes the rest.
+
+      return conflict(entityId)
+    }
+
+    if (eventFailed && codes[stream.head] === 'None') {
+      // With retention, TTL can delete the head before an expired event, so a create passes its
+      // head condition and fails on the leftover item: a conflict until TTL deletes the rest. The
+      // probe already read the remaining items when it found the head item absent below the head
+      // its log derives; read them only if it didn't.
       const verdict = await this.#probe(entityId, retained, request)
-      if (!retained || !(await this.#remainingItemsExpired(entityId, request))) {
+      const expired =
+        retained && (verdict.expired ?? (await this.#remainingItemsExpired(entityId, request)))
+      if (!expired) {
         return verdict.drift ?? drift(entityId, verdict.head, verdict.derived)
       }
+
+      return conflict(entityId)
     }
 
-    return new ConflictError(
-      entityId,
-      'conditional check failed: the sequence or snapshot changed since it was read',
-    )
+    return eventFailed || snapshotFailed ? conflict(entityId) : null
   }
 
   // Classifies a stream whose head update failed, as the failure-path probe does: a stored head
@@ -584,8 +699,11 @@ export class DynamoRepository implements Repository, StoreInitializer {
     const head = (await this.#readHead(entityId, request))?.head ?? null
     const stored = head ?? 0n
     if (stored < derived) {
-      if (retained && head === null && (await this.#remainingItemsExpired(entityId, request))) {
-        return {healthy: false, drift: null, stored, head, derived}
+      if (retained && head === null) {
+        const expired = await this.#remainingItemsExpired(entityId, request)
+        const drifted = expired ? null : drift(entityId, head, derived)
+
+        return {healthy: false, drift: drifted, stored, head, derived, expired}
       }
 
       return {healthy: false, drift: drift(entityId, head, derived), stored, head, derived}
@@ -958,6 +1076,32 @@ interface Verdict {
   stored: bigint
   head: bigint | null
   derived: bigint
+  /** What `#remainingItemsExpired` found, when the probe read the remaining items. */
+  expired?: boolean
+}
+
+function conflict(entityId: string): ConflictError {
+  return new ConflictError(
+    entityId,
+    'conditional check failed: the sequence or snapshot changed since it was read',
+  )
+}
+
+// The stream an action of a transaction belongs to, for a failure no stream check explains: the
+// written or fenced stream at that index, or else the transaction's first.
+function entityAt(layout: TransactionLayout, index: number): string {
+  for (const stream of layout.streams) {
+    if (index >= stream.head && (index < stream.eventsEnd || index === stream.snapshot)) {
+      return stream.commit.entityId
+    }
+  }
+
+  const fenced = layout.fences.find((each) => each.index === index)
+  if (fenced !== undefined) {
+    return fenced.fence.entityId
+  }
+
+  return layout.streams[0]?.commit.entityId ?? layout.fences[0]?.fence.entityId ?? ''
 }
 
 function drift(entityId: string, head: bigint | null, derived: bigint): StreamHeadDriftError {

package/src/dynamodb/requests.ts

@@ -1,15 +1,18 @@
 // The `TransactWriteItems` actions of a commit and of a snapshot write, built from the storage
 // format alone (storage format, "Writes"). Every commit opens with the two control checks, the
-// feature marker's and the rebuild lock's, then writes the stream: its head update, its event
-// puts, and its snapshot put when it has one.
+// feature marker's and the rebuild lock's, then its projector operations, then writes each stream:
+// its head update, its event puts, and its snapshot put when it has one. A transaction then checks
+// the head of every stream it fenced.
 
 import type {SerializedEvent, SerializedSnapshot} from '../event.js'
 import {int64Argument, safeIntegerArgument} from '../fields.js'
-import type {Commit} from '../repository.js'
+import {EvtError} from '../errors.js'
+import type {Commit, NormalizedTransaction, ProjectorGroup, ReadFence} from '../repository.js'
 import type {TransactItem} from './client.js'
-import {type HeadAction, headUpdate} from './head.js'
+import {type HeadAction, headFence, headUpdate} from './head.js'
 import {HEAD_SORT_KEY, type Item, SNAPSHOT_SORT_KEY, eventItem, snapshotItem} from './item.js'
 import {FEATURES_PK, REBUILD_LOCK_PK} from './marker.js'
+import {DYNAMODB_BACKEND} from './views.js'
 
 /** The marker check of a writer that validated the manifest at version `:v`. */
 export const MARKER_CHECK_CONDITION = 'attribute_not_exists(pk) OR version = :v'
@@ -99,8 +102,8 @@ export interface SnapshotTtl {
 
 /**
  * The actions of a commit with at least one event, in order, by a writer that validated the
- * feature manifest at `markerVersion`. `ttl` is given only for a stream with retention. Exported
- * for the compatibility fixtures: its signature changes when commits span streams.
+ * feature manifest at `markerVersion`. `ttl` is given only for a stream with retention. A projector
+ * group for any backend but DynamoDB is refused. Exported for the compatibility fixtures.
  */
 export function commitTransaction(
   table: string,
@@ -108,15 +111,83 @@ export function commitTransaction(
   markerVersion: bigint,
   ttl?: CommitTtl,
 ): TransactItem[] {
-  const first = commit.events[0]
-  const last = commit.events.at(-1)
-  if (first === undefined || last === undefined) {
+  if (commit.events.length === 0) {
     throw new RangeError('a commit transaction needs at least one event')
   }
 
+  const transaction = {streams: [commit], fences: [], projections: commit.projections ?? []}
+
+  return transactionItems(table, transaction, markerVersion, () => ttl)
+}
+
+/** Where each stream's and fence's actions sit in a transaction, to classify a cancellation. */
+export interface TransactionLayout {
+  /** The index after the control checks and the projector operations. */
+  readonly projectorEnd: number
+  readonly streams: readonly StreamLayout[]
+  readonly fences: readonly {readonly index: number; readonly fence: ReadFence}[]
+}
+
+/** A written stream's actions in a transaction. */
+export interface StreamLayout {
+  readonly commit: Commit
+  /** The head update's index. Its event puts follow it. */
+  readonly head: number
+  /** The index after its last event put. */
+  readonly eventsEnd: number
+  /** The snapshot put's index, when it writes one. */
+  readonly snapshot: number | null
+}
+
+/**
+ * The actions of a normalized transaction that writes or fences at least one stream, in order:
+ * the control checks, the projector operations in declaration order, every written stream's head
+ * update, event puts and snapshot put, then a check on every fenced stream's head. `ttl` gives a
+ * written stream's `ttl`s when it has retention. Exported for the compatibility fixtures.
+ */
+export function transactionItems(
+  table: string,
+  transaction: NormalizedTransaction,
+  markerVersion: bigint,
+  ttl: (commit: Commit) => CommitTtl | undefined,
+): TransactItem[] {
+  return buildTransaction(table, transaction, markerVersion, ttl).items
+}
+
+/** {@link transactionItems}, with where each stream's and fence's actions sit. */
+export function buildTransaction(
+  table: string,
+  transaction: NormalizedTransaction,
+  markerVersion: bigint,
+  ttl: (commit: Commit) => CommitTtl | undefined,
+): {items: TransactItem[]; layout: TransactionLayout} {
   const items = [markerCheck(table, markerVersion), rebuildLockCheck(table)]
+  items.push(...projectorItems(transaction.projections))
+  const projectorEnd = items.length
+
+  const streams: StreamLayout[] = []
+  for (const commit of transaction.streams) {
+    const head = items.length
+    items.push(...streamItems(table, commit, ttl(commit)))
+    const eventsEnd = head + 1 + commit.events.length
+    streams.push({commit, head, eventsEnd, snapshot: items.length > eventsEnd ? eventsEnd : null})
+  }
+
+  const fences = transaction.fences.map((fence) => {
+    items.push(fenceCheck(table, fence))
+
+    return {index: items.length - 1, fence}
+  })
+
+  return {items, layout: {projectorEnd, streams, fences}}
+}
+
+// A written stream's actions: its head update, its event puts and its snapshot put.
+function streamItems(table: string, commit: Commit, ttl: CommitTtl | undefined): TransactItem[] {
+  const first = commit.events[0]!
+  const last = commit.events.at(-1)!
   const head = headUpdate(commit.entityId, first.sequence, last.sequence, ttl?.head)
-  items.push(headUpdateItem(table, head))
+  const items = [headUpdateItem(table, head)]
   for (const event of commit.events) {
     items.push(eventPut(table, event, ttl?.items))
   }
@@ -136,6 +207,23 @@ export function commitTransaction(
   return items
 }
 
+// The check on a fenced stream's head item: it still has the head the transaction read.
+function fenceCheck(table: string, fence: ReadFence): TransactItem {
+  const action = headFence(fence.entityId, fence.head)
+  const check: NonNullable<TransactItem['ConditionCheck']> = {
+    TableName: table,
+    Key: headKey(fence.entityId),
+    ConditionExpression: action.conditionExpression,
+  }
+  if (action.values.length > 0) {
+    check.ExpressionAttributeValues = Object.fromEntries(
+      action.values.map(([placeholder, value]) => [placeholder, number(value)]),
+    )
+  }
+
+  return {ConditionCheck: check}
+}
+
 /**
  * The actions of a snapshot written outside a commit: the marker check, the head guard, and the
  * put, which never lowers the stored snapshot. It carries no rebuild-lock check. `ttl` is given
@@ -200,6 +288,22 @@ function snapshotGuard(
   }
 }
 
+/** The DynamoDB actions of projector groups, in order. */
+export function projectorItems(groups: readonly ProjectorGroup[]): TransactItem[] {
+  const items: TransactItem[] = []
+  for (const group of groups) {
+    if (group.backend !== DYNAMODB_BACKEND) {
+      throw new EvtError(
+        'invalidCommit',
+        `invalid commit: a DynamoDB repository can't write a ${group.backend} projector group`,
+      )
+    }
+    items.push(...(group.operations as TransactItem[]))
+  }
+
+  return items
+}
+
 function headUpdateItem(table: string, action: HeadAction): TransactItem {
   const update: NonNullable<TransactItem['Update']> = {
     TableName: table,

package/src/dynamodb/views.ts

@@ -0,0 +1,67 @@
+// The per-entity view projector: one row per entity in a views table, its state as the aggregate's
+// codec encodes it, put in the same transaction as the commit's events (storage format, "Views").
+// Commit-wide projectors, which own their keys, supersede it in Go and Rust, but a TypeScript
+// writer of an entity type that a Go service projects this way must write the row too.
+
+import type {Projection, Projector} from '../aggregate.js'
+import {utf8Length} from '../utf8.js'
+import type {TransactItem} from './client.js'
+import {itemSize, viewItem} from './item.js'
+
+/** The backend name of the projector groups the DynamoDB repository writes. */
+export const DYNAMODB_BACKEND = 'dynamodb'
+
+/**
+ * A projector that puts the entity's view row in `viewsTable` with every commit:
+ * `pk = {entityType}#{entityId}`, `sk = "VIEW"`, and the encoded state as its payload, as Go's
+ * `dynamo.NewViewProjector` writes it. The put is unconditional, so the row is the state after the
+ * entity's latest commit.
+ */
+export function viewProjector<S>(viewsTable: string): Projector<S> {
+  if (viewsTable === '') {
+    throw new RangeError('a view projector needs a views table')
+  }
+
+  return {
+    project({entityType, entityId, encodedState}: Projection<S>) {
+      if (entityId === '') {
+        return null
+      }
+
+      const put: TransactItem = {
+        Put: {TableName: viewsTable, Item: viewItem(entityType, entityId, encodedState)},
+      }
+
+      return {backend: DYNAMODB_BACKEND, operations: [put], sizes: [transactItemSize(put)]}
+    },
+  }
+}
+
+/**
+ * An action's size in the commit budget: a put counts its item, and an update, delete or condition
+ * check counts its key plus its expression strings, name placeholders (the placeholder and the
+ * name) and attribute values.
+ */
+export function transactItemSize(action: TransactItem): number {
+  if (action.Put) {
+    return itemSize(action.Put.Item)
+  }
+
+  const body = action.Update ?? action.Delete ?? action.ConditionCheck
+  if (!body) {
+    return 0
+  }
+
+  let size = itemSize(body.Key) + utf8Length(body.ConditionExpression ?? '')
+  if (action.Update) {
+    size += utf8Length(action.Update.UpdateExpression)
+  }
+  for (const [placeholder, name] of Object.entries(body.ExpressionAttributeNames ?? {})) {
+    size += utf8Length(placeholder) + utf8Length(name)
+  }
+  for (const [placeholder, value] of Object.entries(body.ExpressionAttributeValues ?? {})) {
+    size += utf8Length(placeholder) + itemSize({'': value})
+  }
+
+  return size
+}