@photon-grove/evt 0.1.0 → 0.1.2

package/src/dynamodb/requests.ts

@@ -0,0 +1,276 @@
+// The `TransactWriteItems` actions of a commit and of a snapshot write, built from the storage
+// format alone (storage format, "Writes"). Every commit opens with the two control checks, the
+// feature marker's and the rebuild lock's, then its projector operations, then writes the stream:
+// its head update, its event puts, and its snapshot put when it has one.
+
+import type {SerializedEvent, SerializedSnapshot} from '../event.js'
+import {int64Argument, safeIntegerArgument} from '../fields.js'
+import {EvtError} from '../errors.js'
+import type {Commit} from '../repository.js'
+import type {TransactItem} from './client.js'
+import {type HeadAction, headUpdate} from './head.js'
+import {HEAD_SORT_KEY, type Item, SNAPSHOT_SORT_KEY, eventItem, snapshotItem} from './item.js'
+import {FEATURES_PK, REBUILD_LOCK_PK} from './marker.js'
+import {DYNAMODB_BACKEND} from './views.js'
+
+/** The marker check of a writer that validated the manifest at version `:v`. */
+export const MARKER_CHECK_CONDITION = 'attribute_not_exists(pk) OR version = :v'
+
+/** The rebuild-lock check: no lock row, so the rebuild generation is settled. */
+export const REBUILD_LOCK_CONDITION = 'attribute_not_exists(pk)'
+
+/** The condition of an event put: the sequence is new. */
+export const EVENT_PUT_CONDITION = 'attribute_not_exists(sk)'
+
+/** The condition of a commit's first snapshot of a stream. */
+export const FIRST_SNAPSHOT_CONDITION = 'attribute_not_exists(seq)'
+
+/** The condition of a commit's later snapshot: the stored one is the one before it. */
+export const NEXT_SNAPSHOT_CONDITION = 'seq = :seq'
+
+/** A snapshot write's head guard: the head covers the snapshot. */
+export const SNAPSHOT_GUARD_CONDITION = 'head >= :new'
+
+/** The guard of a snapshot of an empty stream (event sequence 0), which needs no head. */
+export const SNAPSHOT_GUARD_EMPTY_CONDITION = 'attribute_not_exists(head) OR head >= :new'
+
+/** The update expression of a retained stream's snapshot guard: it refreshes the head's `ttl`. */
+export const SNAPSHOT_GUARD_RETAINED_UPDATE = 'SET #ttl = :t'
+
+/** {@link SNAPSHOT_GUARD_CONDITION} with retention: the head's `ttl` is raised, never lowered. */
+export const SNAPSHOT_GUARD_RETAINED_CONDITION =
+  'head >= :new AND (attribute_not_exists(#ttl) OR #ttl <= :t)'
+
+/** A snapshot write's put: it never lowers the stored snapshot's event sequence. */
+export const SNAPSHOT_WRITE_CONDITION = 'attribute_not_exists(eventSeq) OR eventSeq <= :new'
+
+/** The key of a control row, or of a stream's snapshot. */
+export function controlKey(pk: string): Item {
+  return {pk: {S: pk}, sk: {N: SNAPSHOT_SORT_KEY.toString()}}
+}
+
+/** The key of a stream's head item. */
+export function headKey(entityId: string): Item {
+  return {pk: {S: entityId}, sk: {N: HEAD_SORT_KEY.toString()}}
+}
+
+/** The marker check of a writer that validated the manifest at `version`. */
+export function markerCheck(table: string, version: bigint): TransactItem {
+  return {
+    ConditionCheck: {
+      TableName: table,
+      Key: controlKey(FEATURES_PK),
+      ConditionExpression: MARKER_CHECK_CONDITION,
+      ExpressionAttributeValues: {':v': number(version)},
+    },
+  }
+}
+
+/** The rebuild-lock check every commit carries second. */
+export function rebuildLockCheck(table: string): TransactItem {
+  return {
+    ConditionCheck: {
+      TableName: table,
+      Key: controlKey(REBUILD_LOCK_PK),
+      ConditionExpression: REBUILD_LOCK_CONDITION,
+    },
+  }
+}
+
+/** The `ttl`s of a commit to a stream with retention, in Unix seconds. */
+export interface CommitTtl {
+  /** The commit's own `ttl`, which its event and snapshot items carry. */
+  items: number
+  /**
+   * The head item's: the commit's own `ttl` on a create, and on an append the larger of it and
+   * the largest `ttl` the writer loaded for the stream, because the head's is never lowered.
+   */
+  head: number
+}
+
+/** The `ttl` of a snapshot written outside a commit, to a stream with retention. */
+export interface SnapshotTtl {
+  /** The snapshot's `ttl`, in Unix seconds, stamped when it's written. */
+  ttl: number
+  /**
+   * Whether the head guard refreshes the head's `ttl` to at least the snapshot's. True unless the
+   * stored head's `ttl` is known to cover it already.
+   */
+  refresh: boolean
+}
+
+/**
+ * The actions of a commit with at least one event, in order, by a writer that validated the
+ * feature manifest at `markerVersion`. `ttl` is given only for a stream with retention. A projector
+ * group for any backend but DynamoDB is refused. Exported for the compatibility fixtures: its
+ * signature changes when commits span streams.
+ */
+export function commitTransaction(
+  table: string,
+  commit: Commit,
+  markerVersion: bigint,
+  ttl?: CommitTtl,
+): TransactItem[] {
+  const first = commit.events[0]
+  const last = commit.events.at(-1)
+  if (first === undefined || last === undefined) {
+    throw new RangeError('a commit transaction needs at least one event')
+  }
+
+  const items = [markerCheck(table, markerVersion), rebuildLockCheck(table)]
+  items.push(...projectorItems(commit))
+  const head = headUpdate(commit.entityId, first.sequence, last.sequence, ttl?.head)
+  items.push(headUpdateItem(table, head))
+  for (const event of commit.events) {
+    items.push(eventPut(table, event, ttl?.items))
+  }
+
+  const snapshot = commit.snapshot
+  if (snapshot) {
+    const stored: SerializedSnapshot = {
+      entityType: snapshot.entityType,
+      entityId: commit.entityId,
+      sequence: snapshot.sequence,
+      eventSequence: last.sequence,
+      payload: snapshot.payload,
+    }
+    items.push(commitSnapshotPut(table, stored, ttl?.items))
+  }
+
+  return items
+}
+
+/**
+ * The actions of a snapshot written outside a commit: the marker check, the head guard, and the
+ * put, which never lowers the stored snapshot. It carries no rebuild-lock check. `ttl` is given
+ * only for a stream with retention. Exported for the compatibility fixtures.
+ */
+export function snapshotTransaction(
+  table: string,
+  snapshot: SerializedSnapshot,
+  markerVersion: bigint,
+  ttl?: SnapshotTtl,
+): TransactItem[] {
+  const eventSequence = int64Argument(snapshot.eventSequence, 'snapshot.eventSequence')
+
+  return [
+    markerCheck(table, markerVersion),
+    snapshotGuard(table, snapshot.entityId, eventSequence, ttl),
+    {
+      Put: {
+        TableName: table,
+        Item: snapshotItem(snapshot, ttl?.ttl),
+        ConditionExpression: SNAPSHOT_WRITE_CONDITION,
+        ExpressionAttributeValues: {':new': number(eventSequence)},
+      },
+    },
+  ]
+}
+
+// The head item's action in a snapshot write: a check that the head covers the snapshot, or, for a
+// stream with retention, an update that also keeps the head's `ttl` at least the snapshot's. A
+// snapshot of an empty stream may have no head item, so it's always a check: an update would
+// create a head item without a head.
+function snapshotGuard(
+  table: string,
+  entityId: string,
+  eventSequence: bigint,
+  ttl: SnapshotTtl | undefined,
+): TransactItem {
+  const values: Record<string, {N: string}> = {':new': number(eventSequence)}
+  if (eventSequence === 0n || ttl === undefined || !ttl.refresh) {
+    return {
+      ConditionCheck: {
+        TableName: table,
+        Key: headKey(entityId),
+        ConditionExpression:
+          eventSequence === 0n ? SNAPSHOT_GUARD_EMPTY_CONDITION : SNAPSHOT_GUARD_CONDITION,
+        ExpressionAttributeValues: values,
+      },
+    }
+  }
+
+  values[':t'] = number(BigInt(safeIntegerArgument(ttl.ttl, 'ttl')))
+
+  return {
+    Update: {
+      TableName: table,
+      Key: headKey(entityId),
+      UpdateExpression: SNAPSHOT_GUARD_RETAINED_UPDATE,
+      ConditionExpression: SNAPSHOT_GUARD_RETAINED_CONDITION,
+      ExpressionAttributeNames: {'#ttl': 'ttl'},
+      ExpressionAttributeValues: values,
+    },
+  }
+}
+
+/** The DynamoDB actions of a commit's projector groups, in order. */
+export function projectorItems(commit: Commit): TransactItem[] {
+  const items: TransactItem[] = []
+  for (const group of commit.projections ?? []) {
+    if (group.backend !== DYNAMODB_BACKEND) {
+      throw new EvtError(
+        'invalidCommit',
+        `invalid commit: a DynamoDB repository can't write a ${group.backend} projector group`,
+      )
+    }
+    items.push(...(group.operations as TransactItem[]))
+  }
+
+  return items
+}
+
+function headUpdateItem(table: string, action: HeadAction): TransactItem {
+  const update: NonNullable<TransactItem['Update']> = {
+    TableName: table,
+    Key: headKey(action.entityId),
+    UpdateExpression: action.updateExpression ?? '',
+    ConditionExpression: action.conditionExpression,
+    ExpressionAttributeValues: Object.fromEntries(
+      action.values.map(([placeholder, value]) => [placeholder, number(value)]),
+    ),
+  }
+  if (action.names.length > 0) {
+    update.ExpressionAttributeNames = Object.fromEntries(action.names)
+  }
+
+  return {Update: update}
+}
+
+function eventPut(table: string, event: SerializedEvent, ttl: number | undefined): TransactItem {
+  return {
+    Put: {TableName: table, Item: eventItem(event, ttl), ConditionExpression: EVENT_PUT_CONDITION},
+  }
+}
+
+// A commit's snapshot follows the stored one: sequence 1 needs none, and a later one needs the
+// one before it.
+function commitSnapshotPut(
+  table: string,
+  snapshot: SerializedSnapshot,
+  ttl: number | undefined,
+): TransactItem {
+  const sequence = int64Argument(snapshot.sequence, 'snapshot.sequence')
+  if (sequence <= 1n) {
+    return {
+      Put: {
+        TableName: table,
+        Item: snapshotItem(snapshot, ttl),
+        ConditionExpression: FIRST_SNAPSHOT_CONDITION,
+      },
+    }
+  }
+
+  return {
+    Put: {
+      TableName: table,
+      Item: snapshotItem(snapshot, ttl),
+      ConditionExpression: NEXT_SNAPSHOT_CONDITION,
+      ExpressionAttributeValues: {':seq': number(sequence - 1n)},
+    },
+  }
+}
+
+function number(value: bigint): {N: string} {
+  return {N: value.toString()}
+}

package/src/dynamodb/views.ts

@@ -0,0 +1,67 @@
+// The per-entity view projector: one row per entity in a views table, its state as the aggregate's
+// codec encodes it, put in the same transaction as the commit's events (storage format, "Views").
+// Commit-wide projectors, which own their keys, supersede it in Go and Rust, but a TypeScript
+// writer of an entity type that a Go service projects this way must write the row too.
+
+import type {Projection, Projector} from '../aggregate.js'
+import {utf8Length} from '../utf8.js'
+import type {TransactItem} from './client.js'
+import {itemSize, viewItem} from './item.js'
+
+/** The backend name of the projector groups the DynamoDB repository writes. */
+export const DYNAMODB_BACKEND = 'dynamodb'
+
+/**
+ * A projector that puts the entity's view row in `viewsTable` with every commit:
+ * `pk = {entityType}#{entityId}`, `sk = "VIEW"`, and the encoded state as its payload, as Go's
+ * `dynamo.NewViewProjector` writes it. The put is unconditional, so the row is the state after the
+ * entity's latest commit.
+ */
+export function viewProjector<S>(viewsTable: string): Projector<S> {
+  if (viewsTable === '') {
+    throw new RangeError('a view projector needs a views table')
+  }
+
+  return {
+    project({entityType, entityId, encodedState}: Projection<S>) {
+      if (entityId === '') {
+        return null
+      }
+
+      const put: TransactItem = {
+        Put: {TableName: viewsTable, Item: viewItem(entityType, entityId, encodedState)},
+      }
+
+      return {backend: DYNAMODB_BACKEND, operations: [put], sizes: [transactItemSize(put)]}
+    },
+  }
+}
+
+/**
+ * An action's size in the commit budget: a put counts its item, and an update, delete or condition
+ * check counts its key plus its expression strings, name placeholders (the placeholder and the
+ * name) and attribute values.
+ */
+export function transactItemSize(action: TransactItem): number {
+  if (action.Put) {
+    return itemSize(action.Put.Item)
+  }
+
+  const body = action.Update ?? action.Delete ?? action.ConditionCheck
+  if (!body) {
+    return 0
+  }
+
+  let size = itemSize(body.Key) + utf8Length(body.ConditionExpression ?? '')
+  if (action.Update) {
+    size += utf8Length(action.Update.UpdateExpression)
+  }
+  for (const [placeholder, name] of Object.entries(body.ExpressionAttributeNames ?? {})) {
+    size += utf8Length(placeholder) + utf8Length(name)
+  }
+  for (const [placeholder, value] of Object.entries(body.ExpressionAttributeValues ?? {})) {
+    size += utf8Length(placeholder) + itemSize({'': value})
+  }
+
+  return size
+}

package/src/errors.ts

@@ -19,6 +19,19 @@
  * - `commitTooLarge`: the commit is over the transaction budget. Nothing was written.
  * - `compactionUncovered`: no snapshot covers the events compaction was asked to delete.
  * - `reservedEntityId`: a write to an entity ID in a namespace only the engine writes.
+ * - `unsupportedFeature`: the store declares a feature this package doesn't model. Nothing was
+ *   written.
+ * - `streamHeadsNotMigrated`: the store holds data but doesn't declare stream-heads. Nothing was
+ *   written.
+ * - `storeNotInitialized`: the store doesn't declare stream-heads, and the writer doesn't
+ *   initialize a store itself. Nothing was written.
+ * - `featureManifestChanged`: another writer changed the store's feature manifest since this one
+ *   validated it. Retryable: the writer validates it again.
+ * - `transient`: the backend throttled the write. Retryable. Nothing was written.
+ * - `rebuildInProgress`: a projection rebuild holds the store, so commits wait. Retryable.
+ * - `historyDependentProjection`: compaction was refused for a history-dependent entity type.
+ * - `projectorOnlyCommit`: a commit carries projector operations but no events. Nothing was
+ *   written.
  * - `replay`: a stored event failed to upcast or decode while an aggregate loaded.
  * - `snapshot`: a stored snapshot failed to decode while an aggregate loaded.
  */
@@ -33,6 +46,14 @@ export type EvtErrorCode =
   | 'commitTooLarge'
   | 'compactionUncovered'
   | 'reservedEntityId'
+  | 'unsupportedFeature'
+  | 'streamHeadsNotMigrated'
+  | 'storeNotInitialized'
+  | 'featureManifestChanged'
+  | 'transient'
+  | 'rebuildInProgress'
+  | 'historyDependentProjection'
+  | 'projectorOnlyCommit'
   | 'replay'
   | 'snapshot'
 
@@ -162,6 +183,147 @@ export class ReservedEntityIdError extends EvtError {
   }
 }
 
+/** A feature and one of its stored-format versions. */
+export interface FeaturePair {
+  readonly name: string
+  readonly version: number
+}
+
+/**
+ * The store's feature manifest declares pairs this package doesn't model, so it refuses to write:
+ * a commit, a snapshot write, compaction or initialization. Nothing was written.
+ */
+export class UnsupportedFeatureError extends EvtError {
+  override readonly name = 'UnsupportedFeatureError'
+  override readonly code = 'unsupportedFeature'
+  /** Every unsupported pair, sorted by name. */
+  readonly unsupported: readonly FeaturePair[]
+
+  constructor(unsupported: readonly FeaturePair[]) {
+    const pairs = unsupported.map((pair) => `${pair.name}@${pair.version}`).join(', ')
+    super('unsupportedFeature', `store declares unsupported features (${pairs}); refusing to write`)
+    this.unsupported = unsupported
+  }
+}
+
+/**
+ * The store holds data but its feature manifest doesn't declare stream-heads: its streams may have
+ * no head items, so a head-aware write could fail every stream. Migrate the store with every older
+ * writer stopped. Nothing was written.
+ */
+export class StreamHeadsNotMigratedError extends EvtError {
+  override readonly name = 'StreamHeadsNotMigratedError'
+  override readonly code = 'streamHeadsNotMigrated'
+
+  constructor() {
+    super(
+      'streamHeadsNotMigrated',
+      'store does not declare the stream-heads feature and holds data without per-stream head ' +
+        'items; migrate it before writing',
+    )
+  }
+}
+
+/**
+ * The store's feature manifest doesn't declare stream-heads, and this writer doesn't initialize a
+ * store itself (the DynamoDB default), so it refuses rather than scan the table to learn whether
+ * it's fresh. Initialize a fresh store once, at deploy time, or migrate one written before stream
+ * heads. Nothing was written.
+ */
+export class StoreNotInitializedError extends EvtError {
+  override readonly name = 'StoreNotInitializedError'
+  override readonly code = 'storeNotInitialized'
+
+  constructor() {
+    super(
+      'storeNotInitialized',
+      'store is not initialized: the feature manifest does not declare stream-heads; initialize ' +
+        'a fresh store, or migrate one written before stream heads, before writing',
+    )
+  }
+}
+
+/**
+ * Another writer changed the store's feature manifest after this one validated it, so the commit's
+ * marker check failed. The writer has validated the new manifest, which it supports. Retryable.
+ * Nothing was written.
+ */
+export class FeatureManifestChangedError extends EvtError {
+  override readonly name = 'FeatureManifestChangedError'
+  override readonly code = 'featureManifestChanged'
+
+  constructor(options?: {cause?: unknown}) {
+    super('featureManifestChanged', 'feature manifest changed under this writer', options)
+  }
+}
+
+/**
+ * The backend refused the write for now, such as a DynamoDB transaction cancelled for throttling.
+ * Retryable, after a backoff. Nothing was written. The backend's error is its cause.
+ */
+export class TransientError extends EvtError {
+  override readonly name = 'TransientError'
+  override readonly code = 'transient'
+
+  constructor(detail: string, options?: {cause?: unknown}) {
+    super('transient', `transient failure: ${detail}`, options)
+  }
+}
+
+/**
+ * A projection rebuild holds the store, or one was force-released and no rebuild has finished
+ * since, so every commit is refused. Retryable: the command can run again once the rebuild ends.
+ */
+export class RebuildInProgressError extends EvtError {
+  override readonly name = 'RebuildInProgressError'
+  override readonly code = 'rebuildInProgress'
+
+  constructor() {
+    super('rebuildInProgress', 'a projection rebuild is in progress')
+  }
+}
+
+/**
+ * Compaction was refused because the store declares the entity's type history-dependent: its
+ * projectors' rebuild replays the events compaction would delete. Nothing was deleted.
+ */
+export class HistoryDependentProjectionError extends EvtError {
+  override readonly name = 'HistoryDependentProjectionError'
+  override readonly code = 'historyDependentProjection'
+  readonly entityType: string
+  readonly entityId: string
+
+  constructor(entityType: string, entityId: string) {
+    super(
+      'historyDependentProjection',
+      `entity type ${entityType} is declared history-dependent; compaction would delete the ` +
+        `commits its rebuild replays (entity ${entityId})`,
+    )
+    this.entityType = entityType
+    this.entityId = entityId
+  }
+}
+
+/**
+ * A commit carried projector operations but no events. Views are projections of committed events,
+ * so operations with no event behind them are refused on every backend. Nothing was written.
+ */
+export class ProjectorOnlyCommitError extends EvtError {
+  override readonly name = 'ProjectorOnlyCommitError'
+  override readonly code = 'projectorOnlyCommit'
+  /** How many projector operations the commit carried. */
+  readonly operations: number
+
+  constructor(operations: number) {
+    super(
+      'projectorOnlyCommit',
+      `a commit carries ${operations} projector operations but no events; views need a ` +
+        'backing event in the same commit',
+    )
+    this.operations = operations
+  }
+}
+
 /** A load failed closed on a stored event: an upcaster threw, or the payload didn't decode. */
 export class ReplayError extends EvtError {
   override readonly name = 'ReplayError'
@@ -203,9 +365,21 @@ export function isConflict(error: unknown): boolean {
   return error instanceof EvtError && (error.code === 'conflict' || error.code === 'sequenceGap')
 }
 
-/** Whether reloading and running the command again may succeed. So far, only after a conflict. */
+/**
+ * Whether reloading and running the command again may succeed: after a conflict, while a rebuild
+ * holds the store, after the feature manifest changed under the writer, or after throttling.
+ */
 export function isRetryable(error: unknown): boolean {
-  return isConflict(error)
+  if (isConflict(error)) {
+    return true
+  }
+
+  return (
+    error instanceof EvtError &&
+    (error.code === 'rebuildInProgress' ||
+      error.code === 'featureManifestChanged' ||
+      error.code === 'transient')
+  )
 }
 
 /** An error's message, or the value itself as text. */