@photon-grove/evt 0.1.1 → 0.1.2

package/src/postgres/schema.ts

@@ -0,0 +1,286 @@
+// The PostgreSQL layout of an event log, independent of any driver (storage format, "PostgreSQL"):
+// the table names a store is configured with, the engine relations derived from them, and the
+// idempotent DDL that Go's EnsureSchema and Rust's ensure_schema run.
+
+import {EvtError} from '../errors.js'
+
+/** The default events table. */
+export const DEFAULT_EVENTS_TABLE = 'evt_events'
+
+/** The default snapshots table. */
+export const DEFAULT_SNAPSHOTS_TABLE = 'evt_snapshots'
+
+/**
+ * The infix the engine relations derive from the events table name with, such as
+ * `evt_events__evt_features`. A configured table name may not contain it, in any case, so one
+ * log's tables can never equal another log's derived relation.
+ */
+export const RESERVED_INFIX = '__evt_'
+
+/**
+ * The longest events table name, in bytes. Its longest derived relation,
+ * `{events}__evt_rebuild_generation_pkey`, then fills PostgreSQL's 63-byte identifier limit.
+ */
+export const MAX_EVENTS_TABLE_NAME_BYTES = 63 - '__evt_rebuild_generation_pkey'.length
+
+/** The longest snapshots table name, in bytes, which leaves room for its `_pkey` index. */
+export const MAX_SNAPSHOTS_TABLE_NAME_BYTES = 63 - '_pkey'.length
+
+// Only pgTables can make a PgTables, so the names the DDL interpolates are always validated.
+declare const validated: unique symbol
+
+/**
+ * An event log's two configured tables, validated and folded to lower case. Only
+ * {@link pgTables} makes one.
+ */
+export interface PgTables {
+  readonly events: string
+  readonly snapshots: string
+  readonly [validated]: true
+}
+
+const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/
+
+// PostgreSQL's reserved keywords (pg_get_keywords() categories R and T), which can't be unquoted
+// table names. The same list as Go's and Rust's.
+const RESERVED_KEYWORDS = new Set(
+  `all analyse analyze and any array as asc asymmetric authorization binary both case cast check
+  collate collation column concurrently constraint create cross current_catalog current_date
+  current_role current_schema current_time current_timestamp current_user default deferrable desc
+  distinct do else end except false fetch for foreign freeze from full grant group having ilike in
+  initially inner intersect into is isnull join lateral leading left like limit localtime
+  localtimestamp natural not notnull null offset on only or order outer overlaps placing primary
+  references returning right select session_user similar some symmetric system_user table
+  tablesample then to trailing true union unique user using variadic verbose when where window
+  with`.split(/\s+/),
+)
+
+/**
+ * Validates an event log's table names and folds them to lower case, as PostgreSQL folds unquoted
+ * identifiers. Each must be a plain identifier that isn't a reserved keyword, within its length
+ * limit, without the {@link RESERVED_INFIX}, and no two relations the schema creates may share a
+ * name. A name that breaks a rule fails with `invalidArgument`, as Go and Rust refuse it at
+ * construction.
+ */
+export function pgTables(
+  events: string = DEFAULT_EVENTS_TABLE,
+  snapshots: string = DEFAULT_SNAPSHOTS_TABLE,
+): PgTables {
+  for (const [name, limit] of [
+    [events, MAX_EVENTS_TABLE_NAME_BYTES],
+    [snapshots, MAX_SNAPSHOTS_TABLE_NAME_BYTES],
+  ] as const) {
+    if (!validIdentifier(name, limit)) {
+      throw invalidConfig(`invalid table name ${JSON.stringify(name)}`)
+    }
+    if (name.toLowerCase().includes(RESERVED_INFIX)) {
+      throw invalidConfig(
+        `table name ${JSON.stringify(name)} must not contain the reserved infix ` +
+          JSON.stringify(RESERVED_INFIX),
+      )
+    }
+  }
+
+  const tables = {events: events.toLowerCase(), snapshots: snapshots.toLowerCase()} as PgTables
+  const shared = sharedRelation(tables)
+  if (shared !== null) {
+    throw invalidConfig(
+      `tables ${JSON.stringify(events)} and ${JSON.stringify(snapshots)} would both use the ` +
+        `name ${JSON.stringify(shared)}`,
+    )
+  }
+
+  return tables
+}
+
+/** The command-outcomes table: `evt_events` yields `evt_events__evt_command_outcomes`. */
+export function outcomesTable(tables: PgTables): string {
+  return derived(tables.events, 'command_outcomes')
+}
+
+/** The feature-marker table: `evt_events` yields `evt_events__evt_features`. */
+export function featuresTable(tables: PgTables): string {
+  return derived(tables.events, 'features')
+}
+
+/** The rebuild-lock table: `evt_events` yields `evt_events__evt_rebuild_lock`. */
+export function rebuildLockTable(tables: PgTables): string {
+  return derived(tables.events, 'rebuild_lock')
+}
+
+/** The rebuild-generation table: `evt_events` yields `evt_events__evt_rebuild_generation`. */
+export function rebuildGenerationTable(tables: PgTables): string {
+  return derived(tables.events, 'rebuild_generation')
+}
+
+/** The stream-heads table: `evt_events` yields `evt_events__evt_stream_heads`. */
+export function streamHeadsTable(tables: PgTables): string {
+  return derived(tables.events, 'stream_heads')
+}
+
+/** The entity-type index of the events table: `evt_events__evt_entity_type_idx`. */
+export function entityTypeIndex(tables: PgTables): string {
+  return derived(tables.events, 'entity_type_idx')
+}
+
+/**
+ * The idempotent DDL of an event log, byte for byte the DDL Go's `EnsureSchema` and Rust's
+ * `ensure_schema` run: the events and snapshots tables, the entity-type index (dropping its
+ * pre-v0.9 name), and the command-outcome, feature-marker, rebuild-fence and stream-heads tables.
+ * Run it inside one transaction, after {@link SCHEMA_LOCK} with the events table name, as Go and
+ * Rust do.
+ */
+export function schemaDdl(tables: PgTables): string {
+  const {events, snapshots} = tables
+
+  return `
+CREATE TABLE IF NOT EXISTS ${events} (
+    entity_id   text   NOT NULL,
+    sequence    bigint NOT NULL,
+    event_id    text   NOT NULL,
+    event_type  text   NOT NULL,
+    version     bigint NOT NULL,
+    entity_type text   NOT NULL,
+    payload     bytea  NOT NULL,
+    metadata    jsonb  NOT NULL DEFAULT '{}'::jsonb,
+    created_at  timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (entity_id, sequence)
+);
+CREATE INDEX IF NOT EXISTS ${entityTypeIndex(tables)} ON ${events} (entity_type);
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM pg_index
+        WHERE indexrelid = to_regclass('${events}_entity_type_idx')
+          AND indrelid = '${events}'::regclass
+    ) THEN
+        DROP INDEX ${events}_entity_type_idx;
+    END IF;
+END $$;
+
+CREATE TABLE IF NOT EXISTS ${snapshots} (
+    entity_id      text   PRIMARY KEY,
+    entity_type    text   NOT NULL,
+    sequence       bigint NOT NULL,
+    event_sequence bigint NOT NULL,
+    payload        bytea  NOT NULL,
+    updated_at     timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS ${outcomesTable(tables)} (
+    command_id text        PRIMARY KEY,
+    status     text        NOT NULL CHECK (status IN ('accepted', 'rejected')),
+    digest     text        NOT NULL,
+    response   bytea,
+    events     jsonb       NOT NULL DEFAULT '[]'::jsonb,
+    rejection  jsonb,
+    code       text,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    CHECK ((status = 'rejected' AND rejection IS NOT NULL AND code IS NOT NULL)
+           OR (status = 'accepted' AND rejection IS NULL AND code IS NULL))
+);
+
+CREATE TABLE IF NOT EXISTS ${featuresTable(tables)} (
+    id                integer PRIMARY KEY,
+    version           bigint NOT NULL,
+    features          text   NOT NULL,
+    history_dependent text   NOT NULL DEFAULT '[]',
+    retention         text   NOT NULL DEFAULT '[]'
+);
+
+CREATE TABLE IF NOT EXISTS ${rebuildLockTable(tables)} (
+    id               integer PRIMARY KEY,
+    owner            text        NOT NULL,
+    generation       bigint      NOT NULL,
+    lease_expires_at timestamptz NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS ${rebuildGenerationTable(tables)} (
+    id             integer PRIMARY KEY,
+    value          bigint   NOT NULL,
+    force_released boolean  NOT NULL DEFAULT false
+);
+
+CREATE TABLE IF NOT EXISTS ${streamHeadsTable(tables)} (
+    entity_id   text   PRIMARY KEY,
+    entity_type text   NOT NULL,
+    head        bigint NOT NULL CHECK (head >= 1)
+);
+`
+}
+
+/**
+ * Serializes schema changes across processes, in every language: `IF NOT EXISTS` alone can still
+ * race in PostgreSQL's catalogs. `$1` is the events table name.
+ */
+export const SCHEMA_LOCK = 'SELECT pg_advisory_xact_lock(1702261762, hashtext($1))'
+
+/**
+ * The per-entity transaction lock every commit and snapshot write takes, `$1` being the entity
+ * ID. Go and Rust run this exact statement, so writers in every language exclude each other.
+ */
+export const ENTITY_LOCK = 'SELECT pg_advisory_xact_lock(1702261760, hashtext($1))'
+
+/**
+ * The shared engine-row lock a commit takes and holds to `COMMIT`, `$1` being a key from
+ * {@link featuresLockKey} or {@link rebuildLockKey}. Commits don't serialize against each other.
+ */
+export const ENGINE_LOCK_SHARED = 'SELECT pg_advisory_xact_lock_shared(1702261761, hashtext($1))'
+
+/**
+ * The exclusive engine-row lock a declaration or a rebuild lifecycle step takes for its short
+ * transaction, so it never slips between a commit's check and its `COMMIT`.
+ */
+export const ENGINE_LOCK_EXCLUSIVE = 'SELECT pg_advisory_xact_lock(1702261761, hashtext($1))'
+
+/** The feature marker's engine-row lock key: `{events}:features`. */
+export function featuresLockKey(tables: PgTables): string {
+  return `${tables.events}:features`
+}
+
+/** The rebuild fence's engine-row lock key: `{events}:rebuild`. */
+export function rebuildLockKey(tables: PgTables): string {
+  return `${tables.events}:rebuild`
+}
+
+function derived(events: string, suffix: string): string {
+  return `${events}${RESERVED_INFIX}${suffix}`
+}
+
+// A valid identifier is ASCII, so its length is its byte count and folding it changes only A-Z.
+// The type check refuses a non-string passed from JavaScript, which the pattern would coerce.
+function validIdentifier(name: unknown, limit: number): name is string {
+  return (
+    typeof name === 'string' &&
+    name.length <= limit &&
+    IDENTIFIER.test(name) &&
+    !RESERVED_KEYWORDS.has(name.toLowerCase())
+  )
+}
+
+// A name two relations of the schema would share, or null, with the names already folded to lower
+// case. CREATE TABLE IF NOT EXISTS would otherwise skip a table whose name an index already took.
+function sharedRelation(tables: PgTables): string | null {
+  const {events, snapshots} = tables
+  const derivedTables = [
+    outcomesTable(tables),
+    featuresTable(tables),
+    rebuildLockTable(tables),
+    rebuildGenerationTable(tables),
+    streamHeadsTable(tables),
+  ]
+  const relations = [
+    events,
+    `${events}_pkey`,
+    entityTypeIndex(tables),
+    ...derivedTables.flatMap((table) => [table, `${table}_pkey`]),
+    snapshots,
+    `${snapshots}_pkey`,
+  ]
+
+  return relations.find((relation, index) => relations.indexOf(relation) !== index) ?? null
+}
+
+function invalidConfig(message: string): EvtError {
+  return new EvtError('invalidArgument', `postgres: ${message}`)
+}

package/src/repository.ts

@@ -4,7 +4,12 @@
 import {headActionSize, headUpdate} from './dynamodb/head.js'
 import {eventItem, itemSize, snapshotItem} from './dynamodb/item.js'
 import {FEATURES_PK, REBUILD_LOCK_PK} from './dynamodb/marker.js'
-import {CommitTooLargeError, EvtError, ReservedEntityIdError} from './errors.js'
+import {
+  CommitTooLargeError,
+  EvtError,
+  ProjectorOnlyCommitError,
+  ReservedEntityIdError,
+} from './errors.js'
 import {type SerializedEvent, type SerializedSnapshot, eventId} from './event.js'
 import {MAX_COMMIT_ITEM_BYTES} from './snapshots.js'
 
@@ -17,9 +22,24 @@ export interface NewSnapshot {
   payload: Uint8Array
 }
 
+/**
+ * Operations a projector adds to a commit, for the backend it names to write in the commit's own
+ * transaction (Go's transaction groups). The DynamoDB repository writes `dynamodb` groups, whose
+ * operations are `TransactWriteItems` actions. The in-memory repository budgets every group and
+ * writes none, as Go's does, so code with projectors runs on it unchanged.
+ */
+export interface ProjectorGroup {
+  /** The backend whose transaction carries the operations, such as `dynamodb`. */
+  readonly backend: string
+  /** The operations, in the order they're written. */
+  readonly operations: readonly unknown[]
+  /** Each operation's size in the commit budget, as DynamoDB counts it. */
+  readonly sizes: readonly number[]
+}
+
 /**
  * Everything a repository writes atomically for one command. A commit with no events is a no-op,
- * and its snapshot is ignored.
+ * and its snapshot is ignored, unless it carries projector operations: then it's refused.
  */
 export interface Commit {
   /** The stream the events are appended to. */
@@ -28,6 +48,8 @@ export interface Commit {
   events: SerializedEvent[]
   /** The snapshot to write, when the commit reaches a snapshot boundary. */
   snapshot?: NewSnapshot | null | undefined
+  /** The operations the aggregate's projectors add, written first, in order. */
+  projections?: readonly ProjectorGroup[] | null | undefined
 }
 
 /** Options every repository call takes. */
@@ -53,7 +75,9 @@ export interface CallOptions {
  *   {@link StreamHeadDriftError}, ranked above a gap and any other conflict;
  * - refuse a commit's snapshot whose sequence isn't the stored snapshot's plus one (1 with none
  *   stored) with a {@link ConflictError};
- * - write a commit's events, head and snapshot atomically, or nothing;
+ * - write a commit's events, head and snapshot atomically, or nothing, with the projector
+ *   operations it writes (a backend writes only its own groups), and refuse projector operations
+ *   without events with a {@link ProjectorOnlyCommitError};
  * - refuse an entity ID in a reserved namespace on every write path, before any request;
  * - refuse every write (a commit, a snapshot write, compaction) to a store whose feature manifest
  *   declares a pair this package doesn't model, with an {@link UnsupportedFeatureError}, and to
@@ -147,11 +171,27 @@ export function checkNotReserved(entityId: string): void {
  * Checks that a commit's events can be stored as written: each belongs to the commit's entity and
  * has the ID `{entityId}:{sequence}`, they share one entity type, the first sequence is at least
  * 1, and the sequences are contiguous. A snapshot must have the events' entity type and a
- * sequence of at least 1. A commit with no events is valid, and its snapshot isn't checked.
+ * sequence of at least 1, and each projector group a size per operation. A commit with no events
+ * is valid, and its snapshot isn't checked, but its projector operations are refused.
  */
 export function validateCommit(commit: Commit): void {
+  let operations = 0
+  for (const group of commit.projections ?? []) {
+    if (group.sizes.length !== group.operations.length) {
+      throw invalidCommit(
+        `a ${group.backend} projector group has ${group.operations.length} operations but ` +
+          `${group.sizes.length} sizes`,
+      )
+    }
+    operations += group.operations.length
+  }
+
   const [first] = commit.events
   if (first === undefined) {
+    if (operations > 0) {
+      throw new ProjectorOnlyCommitError(operations)
+    }
+
     return
   }
 
@@ -207,9 +247,10 @@ export function validateCommit(commit: Commit): void {
 
 /**
  * The size of each action a commit's transaction carries, in order: the feature-marker and
- * rebuild-lock checks, then the stream's head update, its event puts, and its snapshot put when
- * it writes one. `ttl` gives the expiry an entity type's items carry, if it's retained. Every
- * backend budgets a commit by these sizes, so an oversized commit fails the same way on each.
+ * rebuild-lock checks, the projector operations, then the stream's head update, its event puts,
+ * and its snapshot put when it writes one. `ttl` gives the expiry an entity type's items carry, if
+ * it's retained. Every backend budgets a commit by these sizes, so an oversized commit fails the
+ * same way on each.
  */
 export function commitSizes(
   commit: Commit,
@@ -217,6 +258,10 @@ export function commitSizes(
 ): number[] {
   // The control rows every commit checks: the feature marker and the rebuild lock.
   const sizes = [controlCheckSize(FEATURES_PK), controlCheckSize(REBUILD_LOCK_PK)]
+  for (const group of commit.projections ?? []) {
+    sizes.push(...group.sizes)
+  }
+
   const first = commit.events[0]
   const last = commit.events.at(-1)
   if (first === undefined || last === undefined) {

package/src/store.ts

@@ -5,6 +5,7 @@ import {
   type Aggregate,
   type EventDefinitions,
   type EventOf,
+  type Projection,
   type Rejection,
   encodeEventData,
   replayEvent,
@@ -13,7 +14,7 @@ import {commitIds} from './commit-id.js'
 import {EvtError, SequenceGapError, SnapshotError, isConflict, isRetryable} from './errors.js'
 import {type SerializedEvent, eventId} from './event.js'
 import {type Metadata, emptyMetadata} from './metadata.js'
-import type {CallOptions, NewSnapshot, Repository} from './repository.js'
+import type {CallOptions, NewSnapshot, ProjectorGroup, Repository} from './repository.js'
 import {
   DEFAULT_RETRY_POLICY,
   type RetryPolicy,
@@ -146,8 +147,10 @@ export class Store {
 
   /**
    * Loads the aggregate, runs the command, and commits its events, with a snapshot when the
-   * commit reaches the snapshot interval. A snapshot whose item is over the per-item commit budget
-   * is left out, so the commit still succeeds and later loads replay from the one before.
+   * commit reaches the snapshot interval, and the aggregate's projector operations. A snapshot
+   * whose item is over the per-item commit budget is left out, so the commit still succeeds and
+   * later loads replay from the one before, but a projector's own items are budgeted as they are:
+   * a view row of a state that large fails the commit.
    *
    * A command ID already carried by an event folded in since the latest snapshot makes the
    * command a `duplicate`, and the handler doesn't run. A conflict with another writer throws;
@@ -175,8 +178,13 @@ export class Store {
       return {kind: 'committed', state: loaded, events: []}
     }
 
-    const {state, events, snapshot} = this.#stage(aggregate, loaded, decision.events, metadata)
-    await this.repository.commit({entityId, events, snapshot}, options)
+    const {state, events, snapshot, projections} = this.#stage(
+      aggregate,
+      loaded,
+      decision.events,
+      metadata,
+    )
+    await this.repository.commit({entityId, events, snapshot, projections}, options)
 
     return {kind: 'committed', state, events}
   }
@@ -262,14 +270,19 @@ export class Store {
     return {entityId, state, sequence, snapshotSequence, seenCommandIds}
   }
 
-  // Applies the decided events, and builds the stored events and the snapshot the policy calls
-  // for. The state is only used once the commit succeeds.
+  // Applies the decided events, and builds the stored events, the snapshot the policy calls for,
+  // and the projectors' operations. The state is only used once the commit succeeds.
   #stage<S, C, D extends EventDefinitions>(
     aggregate: Aggregate<S, C, D>,
     loaded: Loaded<S>,
     decided: readonly EventOf<D>[],
     metadata: Metadata,
-  ): {state: Loaded<S>; events: SerializedEvent[]; snapshot: NewSnapshot | null} {
+  ): {
+    state: Loaded<S>
+    events: SerializedEvent[]
+    snapshot: NewSnapshot | null
+    projections: ProjectorGroup[]
+  } {
     const entityId = loaded.entityId
     let sequence = loaded.sequence
     let state = loaded.state
@@ -329,9 +342,46 @@ export class Store {
       },
       events,
       snapshot,
+      projections: this.#project(aggregate, entityId, state, events, snapshot),
     }
   }
 
+  // The operations the aggregate's projectors add to the commit, in declaration order.
+  #project<S, C, D extends EventDefinitions>(
+    aggregate: Aggregate<S, C, D>,
+    entityId: string,
+    state: S,
+    events: SerializedEvent[],
+    snapshot: NewSnapshot | null,
+  ): ProjectorGroup[] {
+    const projectors = aggregate.projectors ?? []
+    if (projectors.length === 0) {
+      return []
+    }
+
+    // Each projector gets its own frozen copy of the events, their metadata and every byte array,
+    // so none can change what another one sees, or what the commit writes.
+    const encodedState = snapshot?.payload ?? aggregate.state.encode(state)
+    const detached = (): Projection<S> =>
+      Object.freeze({
+        entityType: aggregate.type,
+        entityId,
+        state,
+        encodedState: encodedState.slice(),
+        events: Object.freeze(
+          events.map((event) =>
+            Object.freeze({
+              ...event,
+              payload: event.payload && event.payload.slice(),
+              metadata: Object.freeze(copyMetadata(event.metadata)),
+            }),
+          ),
+        ),
+      })
+
+    return projectors.flatMap((projector) => projector.project(detached()) ?? [])
+  }
+
   // The command's metadata with the store's region and the clock's timestamp where it has none,
   // and the commit ID the engine stamps: the command ID, else the caller's, else a generated one.
   // Field by field, so a field an untyped caller set to undefined is empty too, and every key is