@kairos-es/store-postgres 0.0.0

package/src/store.ts

@@ -0,0 +1,413 @@
+/**
+ * The Postgres `DcbEventStore` engine: the runtime `read`/`append`/`subscribe`
+ * over the two-table B-tree + exclusive-lock design (ADR-0002). It satisfies the
+ * SAME `DcbEventStore` contract as the in-memory oracle, so the shared
+ * `store-contract-tests` suite runs unchanged against it with in-memory as the
+ * parity reference.
+ *
+ * Concurrency correctness does NOT live in TypeScript here — it lives in the
+ * plpgsql append function's `LOCK TABLE … IN EXCLUSIVE MODE` (see
+ * `internal/ddl.ts`). The TS side is transport: shape a `Query`/event batch into
+ * JSON, call the function, and map rows back to core types. In particular:
+ *
+ * - a CONFLICT is a ZERO-ROW function result, read from the row count — NEVER a
+ *   caught SQL error. `append` maps empty → `AppendConditionFailed`, one row →
+ *   the `MAX(id)` head position.
+ * - `position` is `bigserial`, so it is strictly-increasing but NOT gapless; we
+ *   order by `id` alone and never assume contiguity.
+ * - `subscribe` opens `listen` on `PgClientDirect` BEFORE the catch-up read, then
+ *   reconciles by paged reads and de-dups the live tail past the caught-up
+ *   boundary — the one place the two clients diverge (the Neon trap).
+ *
+ * That zero-row signal cannot be FORGED by a lost response, and what buys it is
+ * the absence of a retry rather than a check. `append` issues its guarded
+ * statement exactly ONCE: nothing in this engine re-issues one, and
+ * `retryOnConflict` resumes only on `AppendConditionFailed`, which an
+ * infrastructure fault is not — `makeContractStore` dies it, off the error
+ * channel a retry watches. So a write that reaches the server, commits, and
+ * loses its response surfaces as a DEFECT over an outcome this engine does not
+ * resolve, never as a conflict that never happened; there is no second issue of
+ * the guarded statement to meet its own row and read zero. Surfacing rather than
+ * RESOLVING is what makes that free: an engine willing to re-issue an undecided
+ * append needs a way to tell its own committed row from a rival's BEFORE it does,
+ * or the re-issue is the very thing that manufactures the phantom — see
+ * `@kairos-es/store-sqlite`, which resolves one by identity and re-issues only
+ * when identity answers absent. So the missing verify path here is a consequence
+ * of that choice rather than an omission from it.
+ *
+ * This module is WIRING + APPEND: the JSON/JSONB transport (the event/query-item
+ * (de)serialisers and row mapping) lives in `internal/transport.ts` and the
+ * read-plan compiler in `internal/readPlan.ts`. The wake/poll/de-dup subscribe
+ * machine is NOT here either — it is core's `subscribeStream`
+ * (`packages/core/src/SubscribeMachine.ts`), shared by every engine, and this
+ * module only hands it a `fetch` and a `listen` built from the two clients.
+ */
+import type { SqlClient, SqlError } from '@effect/sql'
+import type {
+  DcbEvent,
+  Position,
+  ReadOptions,
+  ReadResponse,
+  SequencedEvent,
+} from '@kairos-es/core'
+import {
+  type AppendCondition,
+  AppendConditionFailed,
+  DcbEventStore,
+  DEFAULT_CATCH_UP_PAGE_SIZE,
+  DEFAULT_POLL_INTERVAL_MILLIS,
+  makeContractStore,
+  ORIGIN,
+  type Query,
+  type RawDcbEventStore,
+  subscribeStream,
+} from '@kairos-es/core'
+import { type Array as Arr, Effect, Layer, type Scope, Stream } from 'effect'
+import { PgClientDirect, PgClientPooled } from './clients'
+import { decodePostgresStoreConfig, type PostgresStoreConfig } from './config'
+import {
+  quoteQualified,
+  type ResolvedNames,
+  resolveNames,
+} from './internal/ddl'
+import {
+  compileFetchPlan,
+  type ReadTables,
+  runFetch,
+} from './internal/readPlan'
+import {
+  type EventRow,
+  jsonbText,
+  rowToSequenced,
+  toEventJson,
+  toQueryItemsJson,
+} from './internal/transport'
+
+/**
+ * Set `lock_timeout` for the current transaction via a BOUND parameter.
+ *
+ * `SET LOCAL` cannot take a bound parameter, but `set_config(setting, value,
+ * is_local)` is its exact bound-parameter equivalent (`is_local = true` ≡ `SET
+ * LOCAL`) — so the timeout value is a parameter, never interpolated into SQL
+ * text. The value is `'<seconds>s'` (Postgres parses the `s` unit); `'0s'` = no
+ * timeout. Runs inside the append transaction so it scopes to that call only.
+ */
+const setLocalLockTimeout = (
+  sql: SqlClient.SqlClient,
+  lockTimeout: number,
+): Effect.Effect<unknown, SqlError.SqlError> =>
+  sql`SELECT set_config('lock_timeout', ${`${lockTimeout}s`}, true)`
+
+/**
+ * Build the engine MECHANICS (a `RawDcbEventStore`) over the two clients and a
+ * resolved name set. Contract policy (`assertServableQuery`, `ReadLimit` decode,
+ * error-vs-defect classification) is NOT here — `makeContractStore` adds it
+ * uniformly, so this engine surfaces raw `SqlError`s and the contract conflict.
+ */
+const make = (
+  config: PostgresStoreConfig,
+  names: ResolvedNames,
+): Effect.Effect<
+  RawDcbEventStore<SqlError.SqlError>,
+  never,
+  PgClientPooled | PgClientDirect
+> =>
+  Effect.gen(function* () {
+    const pooled = yield* PgClientPooled
+    const direct = yield* PgClientDirect
+    const lockTimeout = config.lockTimeout ?? 0
+    const pollInterval = config.pollInterval ?? DEFAULT_POLL_INTERVAL_MILLIS
+    const catchUpPageSize = config.catchUpPageSize ?? DEFAULT_CATCH_UP_PAGE_SIZE
+
+    // The pre-quoted table names every read plan embeds (resolved once).
+    const tables: ReadTables = {
+      main: quoteQualified(names.mainTable),
+      tag: quoteQualified(names.tagTable),
+    }
+
+    // --- read -------------------------------------------------------------
+
+    /**
+     * The global last position ignoring any filter — the head of a no-limit
+     * read. Empty store → ORIGIN.
+     *
+     * A VALUE over the resolved `pooled`, and deliberately NOT a function of a
+     * client — do not re-introduce the parameter. There is only one client it
+     * could take: `direct` exists solely for `subscribe`'s `listen` (see
+     * `clients.ts`), so a client parameter would advertise a selection seam the
+     * engine does not have.
+     *
+     * Nor would such a parameter be load-bearing for the REPEATABLE READ
+     * transaction this runs inside, which is the trap worth naming, since a
+     * closed-over client LOOKS like it must escape to its own snapshot.
+     * `@effect/sql` resolves a statement's connection when the statement RUNS,
+     * from the FIBRE CONTEXT rather than from the client value: `withTransaction`
+     * acquires one connection and `Effect.locally`s it into
+     * `FiberRef.currentContext` under its `TransactionConnection` tag, and every
+     * statement a client builds resolves through a `getConnection` that reads
+     * that tag first, falling back to the pool acquirer only when it is absent
+     * (`@effect/sql@0.52` `src/internal/client.ts`). A statement therefore joins
+     * a transaction by being RUN inside it, and passing the client along the call
+     * chain neither adds to that nor is required by it. `read-postgres`'s
+     * `pinnedConnection` test observes the same fact from the SERVER: two
+     * statements issued inside one transaction report one `pg_backend_pid()`.
+     *
+     * Built once at layer construction and re-run per read, which a statement
+     * supports: it carries no mutable state and recompiles on every execution.
+     */
+    const globalHead: Effect.Effect<Position, SqlError.SqlError> = pooled<{
+      readonly max: string | null
+    }>`
+      SELECT MAX(id) AS max FROM ${pooled(names.mainTable)}
+    `.pipe(
+      Effect.map((rows) => {
+        const max = rows[0]?.max
+        return max === null || max === undefined
+          ? ORIGIN
+          : (BigInt(max) as Position)
+      }),
+    )
+
+    /**
+     * The read MECHANICS: the rows a query selects, plus the head a decision
+     * model may then append under. `read` below only shapes the pair into a
+     * `ReadResponse`, so the two snapshot regimes argued for here stay in one
+     * place.
+     *
+     * Named for what it returns rather than for what it reads through: it closes
+     * over `pooled`, and takes no client, for the reason given on `globalHead`.
+     */
+    const readRowsAndHead = (
+      query: Query,
+      options?: ReadOptions,
+    ): Effect.Effect<
+      { readonly rows: ReadonlyArray<EventRow>; readonly head: Position },
+      SqlError.SqlError
+    > =>
+      Effect.gen(function* () {
+        // No validation here — the query and limit were validated once at the
+        // `makeContractStore` boundary, so the per-read re-validation is gone.
+        const plan = compileFetchPlan(tables, query)
+
+        // Limited read: `head` is the last RETURNED position, taken from the
+        // SAME single statement, so rows and head are already one consistent
+        // snapshot (or ORIGIN when nothing is returned). No transaction needed —
+        // a single statement sees a single snapshot.
+        if (options?.limit !== undefined) {
+          const rows = yield* runFetch(
+            pooled,
+            plan,
+            options.after,
+            options.limit,
+          )
+          const lastRow = rows.at(-1)
+          const head =
+            lastRow === undefined ? ORIGIN : (BigInt(lastRow.id) as Position)
+          return { rows, head }
+        }
+
+        // No-limit read: `head` is the GLOBAL last position (ignoring the query
+        // filter), which needs a SEPARATE `MAX(id)` statement. Rows and head
+        // MUST share one MVCC snapshot: under the default READ COMMITTED each
+        // statement snapshots independently, so a concurrent append committing
+        // between the row fetch and the `MAX(id)` would push `head` PAST an
+        // event absent from `events` — and a decision model appending under
+        // `{ after: head }` would then silently miss that event as a conflict,
+        // breaking optimistic concurrency. A REPEATABLE READ, READ ONLY
+        // transaction gives both statements a single snapshot, matching the
+        // in-memory oracle's atomic (mutex-guarded) read. Readers take only
+        // ACCESS SHARE, so this never contends with the exclusive-lock append.
+        return yield* pooled.withTransaction(
+          Effect.gen(function* () {
+            yield* pooled.unsafe(
+              'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY',
+            )
+            const rows = yield* runFetch(
+              pooled,
+              plan,
+              options?.after,
+              undefined,
+            )
+            const head = yield* globalHead
+            return { rows, head }
+          }),
+        )
+      })
+
+    const read = (
+      query: Query,
+      options?: ReadOptions,
+    ): Effect.Effect<ReadResponse, SqlError.SqlError> =>
+      readRowsAndHead(query, options).pipe(
+        Effect.map(
+          ({ rows, head }): ReadResponse => ({
+            events: Stream.fromIterable(rows.map(rowToSequenced)),
+            head,
+          }),
+        ),
+      )
+
+    // --- append -----------------------------------------------------------
+
+    /**
+     * One row of an append function's result: the new head (`MAX(id)`), or NULL /
+     * zero rows for the empty/conflict cases the callers interpret.
+     */
+    type AppendHeadRow = { readonly max: string | null }
+
+    /**
+     * The ONE transaction frame every append rides: open a transaction, apply the
+     * per-call `set_config('lock_timeout', …, true)`, then run the caller's
+     * `SELECT <fn>(…)`. Both the conditional and unconditional paths go through
+     * here, so the lock-timeout frame can never diverge between them — the
+     * function's LOCK/check/insert and the timeout are one atomic unit, and the
+     * conflict (zero rows) vs head (`MAX(id)`) is read from the returned rows by
+     * the caller, never from a caught error.
+     */
+    const callAppendFn = (
+      select: Effect.Effect<ReadonlyArray<AppendHeadRow>, SqlError.SqlError>,
+    ): Effect.Effect<ReadonlyArray<AppendHeadRow>, SqlError.SqlError> =>
+      pooled.withTransaction(
+        setLocalLockTimeout(pooled, lockTimeout).pipe(Effect.zipRight(select)),
+      )
+
+    const append = (
+      events: Arr.NonEmptyReadonlyArray<DcbEvent>,
+      condition?: AppendCondition,
+    ): Effect.Effect<Position, AppendConditionFailed | SqlError.SqlError> => {
+      const eventsJson = events.map(toEventJson)
+
+      // Raw mechanics: a CONFLICT is `AppendConditionFailed`; a `SqlError` (incl.
+      // a `55P03` lock-timeout abort) rides the channel too. `makeContractStore`
+      // classifies — conflict stays, everything else becomes a defect.
+      return Effect.gen(function* () {
+        if (condition === undefined) {
+          // No guard, but the unconditional twin STILL takes the exclusive lock
+          // (see `unconditionalAppendBody`). That lock on BOTH append paths is how
+          // this engine discharges the subscribe machine's ascending-commit-order
+          // precondition (`SubscribeMachine.ts` argues it): without it two
+          // concurrent unconditional writers can commit bigserial ids out of
+          // order, letting `subscribe` skip the later-committing lower id and
+          // `head` overtake an in-flight position. Sharing `callAppendFn` with the
+          // conditional path guarantees the lock is scoped to the call and the
+          // configured timeout applies identically to both append paths.
+          const rows = yield* callAppendFn(
+            pooled<AppendHeadRow>`
+              SELECT ${pooled(names.appendUnconditionalFn)}(
+                ${jsonbText(eventsJson)}::jsonb
+              ) AS max
+            `,
+          )
+          return headFromRows(rows)
+        }
+
+        const queryItems = toQueryItemsJson(condition.failIfEventsMatch)
+        const afterId = condition.after === undefined ? null : condition.after
+
+        // The function returns MAX(id) on success, ZERO rows on conflict — the
+        // conflict is the row count, never a caught error.
+        const rows = yield* callAppendFn(
+          pooled<AppendHeadRow>`
+            SELECT ${pooled(names.appendFn)}(
+              ${jsonbText(queryItems)}::jsonb,
+              ${afterId},
+              ${jsonbText(eventsJson)}::jsonb
+            ) AS max
+          `,
+        )
+        // Empty result set = conflict.
+        if (rows.length === 0) {
+          return yield* new AppendConditionFailed({ condition })
+        }
+        return headFromRows(rows)
+      })
+    }
+
+    /**
+     * Extract the new head `Position` from the function's single result row. The
+     * function returns exactly one row (`MAX(id)`) on a successful insert; that
+     * value is the last new position (read-your-writes).
+     */
+    const headFromRows = (rows: ReadonlyArray<AppendHeadRow>): Position => {
+      const max = rows[0]?.max
+      if (max === null || max === undefined) {
+        // Defensive: a successful unconditional/`NOT conflict` insert always
+        // yields a non-null MAX. A null here means nothing was inserted, which
+        // for an append (always ≥1 event) is a broken invariant → die.
+        throw new Error(
+          'kairos-es/store-postgres: append returned no head position',
+        )
+      }
+      return BigInt(max) as Position
+    }
+
+    // --- subscribe --------------------------------------------------------
+
+    /**
+     * Wire core's subscribe machine to the live clients: `fetch` is the
+     * per-subscription compiled read plan run against the pooled client and mapped
+     * to `SequencedEvent`s; `listen` is the direct client's ref-counted, non-pooled
+     * LISTEN on this store's channel (the Neon-safe path — `clients.ts` owns why
+     * there are two clients at all). The plan is compiled ONCE here so the poll
+     * loop re-binds only the cursor.
+     *
+     * The wake source is `NOTIFY`, and it is a LATENCY optimisation only: the
+     * machine's periodic position-based poll is what makes delivery correct, which
+     * matters more here than the general argument for it suggests. The
+     * `@effect/sql-pg` listen client SWALLOWS connection errors and does not
+     * auto-reconnect, so a dropped LISTEN typically goes quiet rather than
+     * erroring — the machine's listen-side retry then never fires, and the poll is
+     * the ONLY thing that recovers the subscription. The same silence is what a
+     * Neon pooled endpoint produces if `PgClientDirect` is misconfigured: the
+     * pooler drops `LISTEN`/`NOTIFY` without complaint and delivery degrades to
+     * poll-rate rather than failing, which is why that trap is a deployment-time
+     * wiring choice and not a runtime error.
+     *
+     * `E` is `SqlError.SqlError` — both seams fail with it, so the machine's single
+     * error parameter carries it straight through to `makeContractStore`.
+     */
+    const subscribe = (
+      query: Query,
+      after?: Position,
+    ): Stream.Stream<SequencedEvent, SqlError.SqlError, Scope.Scope> => {
+      const plan = compileFetchPlan(tables, query)
+      return subscribeStream<SqlError.SqlError>(
+        {
+          fetch: (from, limit) =>
+            runFetch(pooled, plan, from, limit).pipe(
+              Effect.map((rows) => rows.map(rowToSequenced)),
+            ),
+          listen: direct.listen(names.channel),
+          pollInterval,
+          catchUpPageSize,
+        },
+        after,
+      )
+    }
+
+    return { read, append, subscribe } as const
+  })
+
+/**
+ * The Postgres `DcbEventStore` layer factory. Builds the engine over
+ * `PgClientPooled` (reads/append/notify) and `PgClientDirect` (the subscribe
+ * listener). The caller wires those two client layers — pointing `PgClientDirect`
+ * at a direct (non-pooler) endpoint on Neon, or at the same URL as `PgClientPooled`
+ * everywhere else.
+ *
+ * `ensureSchema(config)` must have been run (by the caller's migrator or at
+ * boot) before the layer is used; the layer itself performs no DDL, so it stays
+ * a pure runtime dependency requiring only the two clients.
+ */
+export const layer = (
+  config: PostgresStoreConfig = {},
+): Layer.Layer<DcbEventStore, never, PgClientPooled | PgClientDirect> => {
+  // Decode through the config schema so a malformed config (empty/dotted atom,
+  // negative/non-finite timeout, over-63-byte derived name) fails LOUDLY at
+  // construction rather than late on the first append.
+  const decoded = decodePostgresStoreConfig(config)
+  return Layer.effect(
+    DcbEventStore,
+    Effect.map(make(decoded, resolveNames(decoded)), makeContractStore),
+  )
+}