@photon-grove/evt 0.1.0 → 0.1.2

package/src/features.ts

@@ -0,0 +1,175 @@
+// The feature manifest a store keeps on its SYSTEM#FEATURES marker row (storage format, "Feature
+// marker"): the extensions it declares, each at a stored-format version, and the entity types
+// declared history-dependent or retained. A writer refuses a store that declares a pair it doesn't
+// model, and every writer requires stream-heads.
+
+import {
+  EvtError,
+  type FeaturePair,
+  StreamHeadsNotMigratedError,
+  UnsupportedFeatureError,
+} from './errors.js'
+import {DOCUMENT, int64, invalid} from './fields.js'
+import {type JsonValue, parseJson, stringifyJson} from './json.js'
+import {compareUtf8} from './utf8.js'
+
+/** The feature every writer models and requires: per-stream head items (ADR 0007). */
+export const STREAM_HEADS_FEATURE = 'stream-heads'
+
+/** The stored-format version of {@link STREAM_HEADS_FEATURE} this package reads and writes. */
+export const STREAM_HEADS_VERSION = 1
+
+/**
+ * The `(feature, stored-format version)` pairs this package models. The other features the
+ * storage format defines (`journal`, `journal-compaction`, `postgres-views`) are Rust-first, so a
+ * store that declares any of them is refused.
+ */
+export const SUPPORTED_FEATURES: readonly FeaturePair[] = Object.freeze([
+  Object.freeze({name: STREAM_HEADS_FEATURE, version: STREAM_HEADS_VERSION}),
+])
+
+// What validation reads. Private, so no caller can add a pair to it.
+const supported = new Map(SUPPORTED_FEATURES.map((pair) => [pair.name, pair.version]))
+
+/** The parsed marker row. A store with no marker row has no manifest. */
+export interface FeatureManifest {
+  /** Bumped by exactly one on every declaration. */
+  readonly version: bigint
+  /** Each declared feature's stored-format version. */
+  readonly features: ReadonlyMap<string, number>
+  /** The entity types whose commit-wide projectors are history-dependent: never compacted. */
+  readonly historyDependent: readonly string[]
+  /** The entity types configured with retention: never fenced. */
+  readonly retention: readonly string[]
+}
+
+/**
+ * Refuses a manifest that declares a pair this package doesn't model, listing every such pair,
+ * with an {@link UnsupportedFeatureError}.
+ */
+export function checkSupported(manifest: FeatureManifest | null): void {
+  const unsupported: FeaturePair[] = []
+  for (const [name, version] of sortedFeatures(manifest?.features ?? new Map())) {
+    if (supported.get(name) !== version) {
+      unsupported.push({name, version})
+    }
+  }
+
+  if (unsupported.length > 0) {
+    throw new UnsupportedFeatureError(unsupported)
+  }
+}
+
+/** Whether the manifest declares stream-heads at the version this package models. */
+export function declaresStreamHeads(manifest: FeatureManifest | null): boolean {
+  return manifest?.features.get(STREAM_HEADS_FEATURE) === STREAM_HEADS_VERSION
+}
+
+/**
+ * Refuses a write to a store whose manifest declares an unsupported pair, or doesn't declare
+ * stream-heads, without declaring anything.
+ */
+export function checkWritable(manifest: FeatureManifest | null): void {
+  checkSupported(manifest)
+  if (!declaresStreamHeads(manifest)) {
+    throw new StreamHeadsNotMigratedError()
+  }
+}
+
+/** The manifest with stream-heads declared and its version bumped: a first writer's declaration. */
+export function withStreamHeads(manifest: FeatureManifest | null): FeatureManifest {
+  const features = new Map(manifest?.features)
+  features.set(STREAM_HEADS_FEATURE, STREAM_HEADS_VERSION)
+
+  return {
+    version: (manifest?.version ?? 0n) + 1n,
+    features,
+    historyDependent: manifest?.historyDependent ?? [],
+    retention: manifest?.retention ?? [],
+  }
+}
+
+/** A manifest's stored JSON texts, as the marker row holds them. */
+export interface ManifestTexts {
+  /** A JSON object, feature name to stored-format version, keys sorted. `{}` when empty. */
+  features: string
+  /** A JSON array of entity types, sorted by UTF-8 bytes and deduplicated. `[]` when empty. */
+  historyDependent: string
+  /** Like `historyDependent`. */
+  retention: string
+}
+
+/** Encodes a manifest's declarable parts as the marker row stores them. */
+export function encodeManifest(manifest: FeatureManifest): ManifestTexts {
+  return {
+    features: stringifyJson(new Map(manifest.features)),
+    historyDependent: stringifyJson(entityTypeSet(manifest.historyDependent)),
+    retention: stringifyJson(entityTypeSet(manifest.retention)),
+  }
+}
+
+/**
+ * Decodes a marker row's stored texts. An empty text is an empty map or set, as an absent
+ * attribute is.
+ */
+export function decodeManifest(version: bigint, texts: ManifestTexts): FeatureManifest {
+  return {
+    version,
+    features: texts.features === '' ? new Map() : parseFeatures(texts.features),
+    historyDependent: parseEntityTypes(texts.historyDependent, 'historyDependent'),
+    retention: parseEntityTypes(texts.retention, 'retention'),
+  }
+}
+
+/** An entity-type set in its stored order: sorted by UTF-8 bytes and deduplicated. */
+export function entityTypeSet(types: readonly string[]): string[] {
+  // A fresh array, sorted in place: ES2022 has no toSorted.
+  const set = Array.from(new Set(types))
+  set.sort(compareUtf8)
+
+  return set
+}
+
+function parseFeatures(text: string): Map<string, number> {
+  const parsed = parseStored(text, 'features')
+  if (!(parsed instanceof Map)) {
+    throw invalid('the feature manifest is not a JSON object')
+  }
+
+  // Go and Rust read a version as an int64. One past the safe integers is no version this
+  // package models, so rounding it changes only the refusal's text.
+  const features = new Map<string, number>()
+  for (const [name, version] of parsed) {
+    features.set(name, Number(int64(version, `the feature manifest's ${name} version`)))
+  }
+
+  return features
+}
+
+function parseEntityTypes(text: string, what: string): string[] {
+  if (text.trim() === '') {
+    return []
+  }
+
+  const parsed = parseStored(text, what)
+  if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string')) {
+    throw invalid(`the ${what} set is not a JSON array of strings`)
+  }
+
+  return entityTypeSet(parsed as string[])
+}
+
+function parseStored(text: string, what: string): JsonValue {
+  try {
+    return parseJson(text, DOCUMENT)
+  } catch (error) {
+    throw new EvtError('invalidRecord', `the stored ${what} is not JSON`, {cause: error})
+  }
+}
+
+function sortedFeatures(features: ReadonlyMap<string, number>): [string, number][] {
+  const sorted = Array.from(features)
+  sorted.sort(([a], [b]) => compareUtf8(a, b))
+
+  return sorted
+}

package/src/index.ts

@@ -9,6 +9,8 @@ export {
   type EventDefinition,
   type EventDefinitions,
   type EventOf,
+  type Projection,
+  type Projector,
   type Rejection,
   type Upcaster,
   defineAggregate,
@@ -30,11 +32,20 @@ export {
   ConflictError,
   EvtError,
   type EvtErrorCode,
+  FeatureManifestChangedError,
+  HistoryDependentProjectionError,
+  ProjectorOnlyCommitError,
+  RebuildInProgressError,
   ReplayError,
   ReservedEntityIdError,
   SequenceGapError,
   SnapshotError,
+  StoreNotInitializedError,
   StreamHeadDriftError,
+  StreamHeadsNotMigratedError,
+  TransientError,
+  UnsupportedFeatureError,
+  type FeaturePair,
   isConflict,
   isRetryable,
 } from './errors.js'
@@ -47,6 +58,15 @@ export {
   encodeSnapshot,
   eventId,
 } from './event.js'
+export {
+  type FeatureManifest,
+  type ManifestTexts,
+  STREAM_HEADS_FEATURE,
+  STREAM_HEADS_VERSION,
+  SUPPORTED_FEATURES,
+  decodeManifest,
+  encodeManifest,
+} from './features.js'
 export {JsonNumber, type JsonValue, parseJson, stringifyJson} from './json.js'
 export {
   type Metadata,
@@ -56,7 +76,14 @@ export {
   emptyMetadata,
   encodeMetadata,
 } from './metadata.js'
-export type {CallOptions, Commit, NewSnapshot, Repository} from './repository.js'
+export type {
+  CallOptions,
+  Commit,
+  NewSnapshot,
+  ProjectorGroup,
+  Repository,
+  StoreInitializer,
+} from './repository.js'
 export {
   DEFAULT_RETRY_POLICY,
   type RetryPolicy,

package/src/memory/index.ts

@@ -1,3 +1,11 @@
 // @photon-grove/evt/memory: the in-memory repository, for tests and local development.
 
-export {type HeadFixtures, MemRepository, headFixtures} from './repository.js'
+export {
+  type ControlFixtures,
+  type HeadFixtures,
+  type ManifestDraft,
+  MemRepository,
+  type RebuildState,
+  controlFixtures,
+  headFixtures,
+} from './repository.js'

package/src/memory/repository.ts

@@ -1,21 +1,34 @@
 // The in-memory repository. It enforces the storage contract the way the live backends do: a head
-// per stream that every commit raises, optimistic concurrency against it, and the same failure
-// classification and transaction budget. It's for tests and local development.
+// per stream that every commit raises, optimistic concurrency against it, the feature marker and
+// the rebuild-lock check, and the same failure classification and transaction budget. It's for
+// tests and local development.
 
 import {
   ConflictError,
   CompactionUncoveredError,
   EvtError,
+  HistoryDependentProjectionError,
+  RebuildInProgressError,
   SequenceGapError,
   StreamHeadDriftError,
+  StreamHeadsNotMigratedError,
 } from '../errors.js'
 import type {SerializedEvent, SerializedSnapshot} from '../event.js'
+import {
+  type FeatureManifest,
+  checkSupported,
+  checkWritable,
+  declaresStreamHeads,
+  entityTypeSet,
+  withStreamHeads,
+} from '../features.js'
 import {int64Argument} from '../fields.js'
 import type {Metadata} from '../metadata.js'
 import {
   type CallOptions,
   type Commit,
   type Repository,
+  type StoreInitializer,
   checkBudget,
   checkNotReserved,
   commitSizes,
@@ -30,8 +43,26 @@ interface Stream {
   head: bigint
 }
 
-// Reads a repository's streams, for headFixtures. The class's static block sets it.
+/** The state of the rebuild fence's two rows that a commit's lock check reads. */
+export interface RebuildState {
+  /** The generation counter: odd while a rebuild holds the store or after a force-release. */
+  generation: bigint
+  /**
+   * The lock row: a rebuild's lease, the item a force-release leaves in its place, or none. A
+   * commit is refused while either exists, and while the generation is odd.
+   */
+  lock: 'held' | 'forceReleased' | null
+}
+
+// The store-wide control rows: the feature marker and the rebuild fence.
+interface Controls {
+  manifest: FeatureManifest | null
+  rebuild: RebuildState
+}
+
+// Read a repository's private state, for the fixtures. The class's static block sets them.
 let streamsOf: (repo: MemRepository) => Map<string, Stream>
+let controlsOf: (repo: MemRepository) => Controls
 
 /**
  * An in-memory {@link Repository}. Every call copies what it stores and what it returns, so a
@@ -39,9 +70,26 @@ let streamsOf: (repo: MemRepository) => Map<string, Stream>
  *
  * Each call runs to completion before another starts, so concurrent commits to one stream behave
  * like racing writers on a live backend: one wins and the rest fail with a conflict.
+ *
+ * A fresh store is initialized by its first commit or snapshot write, which declares stream-heads
+ * on its feature manifest, so {@link initialize} is needed only where code treats every backend
+ * alike.
  */
-export class MemRepository implements Repository {
+export class MemRepository implements Repository, StoreInitializer {
   readonly #streams = new Map<string, Stream>()
+  readonly #controls: Controls = {manifest: null, rebuild: {generation: 0n, lock: null}}
+
+  async initialize(options?: CallOptions): Promise<boolean> {
+    options?.signal?.throwIfAborted()
+    checkSupported(this.#controls.manifest)
+    if (declaresStreamHeads(this.#controls.manifest)) {
+      return false
+    }
+
+    this.#declareStreamHeads()
+
+    return true
+  }
 
   async commit(commit: Commit, options?: CallOptions): Promise<void> {
     options?.signal?.throwIfAborted()
@@ -54,6 +102,14 @@ export class MemRepository implements Repository {
     // Every backend budgets a commit the same way, so an oversized one fails before any write.
     checkBudget(commitSizes(commit, () => undefined))
 
+    // The control rows come first, as the two checks that open every live commit: the marker,
+    // then the rebuild lock, which an unsettled generation fails, a force-released one included.
+    this.#checkManifest()
+    const {rebuild} = this.#controls
+    if (rebuild.lock !== null || rebuild.generation % 2n !== 0n) {
+      throw new RebuildInProgressError()
+    }
+
     const held = this.#streams.get(commit.entityId) ?? emptyStream()
     checkWrite(commit, held)
 
@@ -120,6 +176,7 @@ export class MemRepository implements Repository {
     checkNotReserved(snapshot.entityId)
     int64Argument(snapshot.sequence, 'snapshot.sequence')
     int64Argument(snapshot.eventSequence, 'snapshot.eventSequence')
+    this.#checkManifest()
 
     // The stream's head must cover the snapshot. One of an empty stream (event sequence 0) may
     // have no head at all.
@@ -153,12 +210,22 @@ export class MemRepository implements Repository {
       return 0
     }
 
+    // Compaction validates the manifest, but never declares stream-heads itself.
+    const {manifest} = this.#controls
+    checkWritable(manifest)
+
     const stream = this.#streams.get(entityId)
-    const covered = stream?.snapshot?.eventSequence ?? null
-    if (stream === undefined || covered === null || covered < through) {
+    const snapshot = stream?.snapshot
+    const covered = snapshot?.eventSequence ?? null
+    if (stream === undefined || !snapshot || covered === null || covered < through) {
       throw new CompactionUncoveredError(entityId, through, covered)
     }
 
+    // A history-dependent type's rebuild replays the events compaction would delete.
+    if (manifest?.historyDependent.includes(snapshot.entityType)) {
+      throw new HistoryDependentProjectionError(snapshot.entityType, entityId)
+    }
+
     let deleted = 0
     for (const sequence of stream.events.keys()) {
       if (sequence <= through) {
@@ -172,6 +239,27 @@ export class MemRepository implements Repository {
 
   static {
     streamsOf = (repo) => repo.#streams
+    controlsOf = (repo) => repo.#controls
+  }
+
+  // Refuses a write to a store whose manifest declares an unsupported pair, and declares
+  // stream-heads on a store without it, as its first writer, when the store holds no data.
+  #checkManifest(): void {
+    checkSupported(this.#controls.manifest)
+    if (!declaresStreamHeads(this.#controls.manifest)) {
+      this.#declareStreamHeads()
+    }
+  }
+
+  #declareStreamHeads(): void {
+    const holdsData = Array.from(this.#streams.values()).some(
+      (stream) => stream.events.size > 0 || stream.snapshot !== null,
+    )
+    if (holdsData) {
+      throw new StreamHeadsNotMigratedError()
+    }
+
+    this.#controls.manifest = withStreamHeads(this.#controls.manifest)
   }
 
   // The stream's events by sequence, ascending or descending. Sequences are unique.
@@ -282,6 +370,68 @@ export function headFixtures(repo: MemRepository): HeadFixtures {
   }
 }
 
+/** A feature manifest's declarable parts, which a declaration changes in place. */
+export interface ManifestDraft {
+  features: Map<string, number>
+  historyDependent: string[]
+  retention: string[]
+}
+
+/** What a conformance suite needs to declare features and hold the rebuild lock. */
+export interface ControlFixtures {
+  /** The stored manifest, or null when the store has no marker row. */
+  manifest(): FeatureManifest | null
+  /** Changes the manifest in one declaration, bumping its version by one, and returns it. */
+  declare(change: (draft: ManifestDraft) => void): FeatureManifest
+  /** Sets the rebuild fence's rows, as a rebuild's lifecycle would. */
+  setRebuild(state: RebuildState): void
+}
+
+/**
+ * Hooks that change a repository's control rows, for conformance suites only. A declaration may
+ * declare features this package doesn't model, which then refuse every write.
+ */
+export function controlFixtures(repo: MemRepository): ControlFixtures {
+  const controls = controlsOf(repo)
+
+  return {
+    manifest() {
+      return controls.manifest && cloneManifest(controls.manifest)
+    },
+    declare(change) {
+      const draft: ManifestDraft = {
+        features: new Map(controls.manifest?.features),
+        historyDependent: [...(controls.manifest?.historyDependent ?? [])],
+        retention: [...(controls.manifest?.retention ?? [])],
+      }
+      change(draft)
+
+      // Stored as the marker row stores them: each set sorted and deduplicated.
+      controls.manifest = {
+        version: (controls.manifest?.version ?? 0n) + 1n,
+        features: new Map(draft.features),
+        historyDependent: entityTypeSet(draft.historyDependent),
+        retention: entityTypeSet(draft.retention),
+      }
+
+      return cloneManifest(controls.manifest)
+    },
+    setRebuild(state) {
+      int64Argument(state.generation, 'generation')
+      controls.rebuild = {...state}
+    },
+  }
+}
+
+function cloneManifest(manifest: FeatureManifest): FeatureManifest {
+  return {
+    version: manifest.version,
+    features: new Map(manifest.features),
+    historyDependent: [...manifest.historyDependent],
+    retention: [...manifest.retention],
+  }
+}
+
 function emptyStream(): Stream {
   return {events: new Map(), snapshot: null, head: 0n}
 }

package/src/postgres/client.ts

@@ -0,0 +1,37 @@
+// The PostgreSQL client a PgRepository takes: a structural interface that `pg`'s Pool, Neon's
+// serverless Pool and PGlite satisfy without an adapter, so this package imports no driver.
+
+/** The result of one statement: its rows, as the driver returns them. */
+export interface PgResult {
+  readonly rows: readonly unknown[]
+}
+
+/** Something that runs one statement: `$1`, `$2` and so on in `text` take `values` in order. */
+export interface PgQueryable {
+  query(text: string, values?: unknown[]): Promise<PgResult>
+}
+
+/**
+ * A connection held for one transaction, as a pool lends it. `release` returns it to the pool, and
+ * gets the error when the connection may be unusable, so the pool can discard it.
+ */
+export interface PgConnection extends PgQueryable {
+  release(error?: Error): void
+}
+
+/**
+ * The client a {@link PgRepository} runs on.
+ *
+ * - A pool, such as `pg`'s `Pool` or Neon's serverless `Pool`, has `connect`, which lends one
+ *   connection for each transaction. Single statements outside a transaction go to `query`.
+ * - A single connection without `connect`, such as PGlite, runs everything itself, so every
+ *   repository on it runs one operation at a time on it. Nothing else may use the connection
+ *   meanwhile: a statement sent inside a repository's transaction would join it.
+ *
+ * Every value the repository sends is a string or null, with casts in the SQL, and every column it
+ * reads is text, so no driver's type parsing touches a stored value. A `pg` `Client` has a
+ * `connect` that opens the client rather than lending a connection, so pass a `Pool` instead.
+ */
+export interface PgClient extends PgQueryable {
+  connect?(): Promise<PgConnection>
+}

package/src/postgres/index.ts

@@ -0,0 +1,23 @@
+// @photon-grove/evt/postgres: the PostgreSQL backend. It imports no driver: the repository takes
+// `pg`'s Pool, or anything shaped like it, as a PgClient.
+
+export {type PgClient, type PgConnection, type PgQueryable, type PgResult} from './client.js'
+export {POSTGRES_BACKEND, PgRepository, type PgRepositoryOptions} from './repository.js'
+
+export {
+  DEFAULT_EVENTS_TABLE,
+  DEFAULT_SNAPSHOTS_TABLE,
+  MAX_EVENTS_TABLE_NAME_BYTES,
+  MAX_SNAPSHOTS_TABLE_NAME_BYTES,
+  type PgTables,
+  RESERVED_INFIX,
+  SCHEMA_LOCK,
+  entityTypeIndex,
+  featuresTable,
+  outcomesTable,
+  pgTables,
+  rebuildGenerationTable,
+  rebuildLockTable,
+  schemaDdl,
+  streamHeadsTable,
+} from './schema.js'