@kairos-es/feed-postgres 0.0.0-stage → 0.0.0

package/src/syncStore.ts

@@ -0,0 +1,340 @@
+/**
+ * The Postgres `FeedSyncStore`: every sink's cursor, its parked list and its
+ * worker lease, kept in the source database beside the feed rows (ADR-0019 §10).
+ *
+ * `@kairos-es/feed`'s `FeedSyncStore.ts` owns the contract, and the in-memory feed
+ * is the oracle this store is held to; what is here is how each atomic step the
+ * contract names becomes one transaction or one statement.
+ *
+ * - `casCursor` is ONE transaction: the guarded, holder-fenced cursor update,
+ *   stamped with its `lastAdvance` from this process's `Clock`, and the parked
+ *   list moved with it. A guard that loses fails `CursorSuperseded` inside the
+ *   transaction, which rolls back the parked changes with it, so a park can never
+ *   outlive a refused advance.
+ * - `retryParked` is one transaction that takes the feed lock exclusively before it
+ *   re-stamps the row, so its new `seq` cannot commit out of order with the
+ *   materialisation's own writes (`internal/statements.ts` owns that argument).
+ * - `collectTombstones` is one statement that reads every registered sink's cursor
+ *   and deletes below the lowest, or nothing while any of them has no cursor, so
+ *   no cursor can move between the two.
+ * - `removeSink` is one transaction that first takes the sink's advisory lock for
+ *   its own length, which an advisory worker's session lock refuses, then deletes
+ *   the cursor row only when no live lease row holds it, in the same statement as
+ *   that test, and the parks after it. Both tests are therefore made whichever
+ *   lease kind this store was built with: a sink can be leased by either kind
+ *   (`lease.ts` owns how the two coexist), and the row alone cannot say whether
+ *   an advisory holder's session is alive.
+ * - `describe` is three reads, each its own snapshot, so a figure in it can be a
+ *   batch older or newer than the next. It is a report for an operator, and no
+ *   decision is taken on it.
+ *
+ * One store serves every feed in its prefix: unlike the in-memory feed, whose
+ * store belongs to one feed, a SQL store's state is keyed by feed name in shared
+ * tables, so a single store value is what every worker over the prefix takes.
+ *
+ * Infrastructure faults are `FeedUnreadable` on a read and `FeedWriteFailed` on a
+ * write, never defects, so the driver's supervision backs off and tries again.
+ */
+import {
+  CursorSuperseded,
+  type FeedCursor,
+  FeedKey,
+  FeedSeq,
+  type FeedSyncDescription,
+  type FeedSyncStore,
+  Generation,
+  LeaseHeld,
+  LeaseHolder,
+  type ParkedRecord,
+  SinkName,
+  type SinkSyncState,
+} from '@kairos-es/feed'
+import { Clock, DateTime, Effect, Option } from 'effect'
+import { SqlClient } from 'effect/sql'
+import type { FeedPostgresConfig } from './config'
+import {
+  bigintOf,
+  feedContext,
+  unreadable,
+  writeFailed,
+} from './internal/context'
+import {
+  advisoryLockName,
+  type FeedLeaseConfig,
+  leaseRow,
+  makeAcquireLease,
+} from './lease'
+
+/** Options for {@link makePostgresFeedSyncStore}. */
+export interface PostgresFeedSyncStoreOptions {
+  /** The `{ schema, tablePrefix }` pair the feeds were written under. */
+  readonly config?: FeedPostgresConfig
+  /** The lease kind; `leaseRow()` with its defaults when absent. */
+  readonly lease?: FeedLeaseConfig
+}
+
+type CursorRow = {
+  readonly generation: string | null
+  readonly seq: string | null
+}
+
+type ParkedRow = {
+  readonly key: string
+  readonly seq: string
+  readonly reason: string
+  readonly parked_at_ms: string
+}
+
+type SinkRow = CursorRow & {
+  readonly sink: string
+  readonly lease_holder: string | null
+  readonly lease_expires_at_ms: string | null
+  readonly last_advance_at_ms: string | null
+  readonly pending: string
+}
+
+/** A timestamp column read back as epoch milliseconds, or `None` when NULL. */
+const instantOf = (ms: string | null): Option.Option<DateTime.Utc> =>
+  ms === null ? Option.none() : Option.some(DateTime.makeUnsafe(Number(ms)))
+
+const cursorOf = (row: CursorRow | undefined): Option.Option<FeedCursor> =>
+  row === undefined || row.generation === null || row.seq === null
+    ? Option.none()
+    : Option.some({
+        generation: Generation.make(bigintOf(row.generation)),
+        seq: FeedSeq.make(bigintOf(row.seq)),
+      })
+
+const parkedOf = (row: ParkedRow): ParkedRecord => ({
+  key: FeedKey.make(row.key),
+  seq: FeedSeq.make(bigintOf(row.seq)),
+  reason: row.reason,
+  parkedAt: DateTime.makeUnsafe(Number(row.parked_at_ms)),
+})
+
+/** Park entries as the `jsonb` parameter the park statement takes; last per key wins. */
+const parkJson = (park: ReadonlyArray<ParkedRecord>): string => {
+  const byKey = new Map<string, ParkedRecord>()
+  for (const record of park) {
+    byKey.delete(record.key)
+    byKey.set(record.key, record)
+  }
+  return JSON.stringify(
+    Array.from(byKey.values(), (record) => ({
+      key: record.key,
+      seq: record.seq.toString(),
+      reason: record.reason,
+      parked_at_ms: String(DateTime.toEpochMillis(record.parkedAt)),
+    })),
+  )
+}
+
+/**
+ * Build the sync store over the ambient `SqlClient`. Dies at construction on a
+ * malformed config, a lease whose heartbeat is not shorter than its ttl or leaves
+ * no room for its margin, or an `advisoryLock` connection recognisably pooled.
+ * `ensureFeedSchema` must have run.
+ */
+export const makePostgresFeedSyncStore = (
+  options: PostgresFeedSyncStoreOptions = {},
+): Effect.Effect<FeedSyncStore, never, SqlClient.SqlClient> =>
+  Effect.gen(function* () {
+    const sql = (yield* SqlClient.SqlClient).withoutTransforms()
+    const context = feedContext(options.config ?? {})
+    const acquireLease = makeAcquireLease(
+      options.lease ?? leaseRow(),
+      sql,
+      context,
+    )
+    const { statements } = context
+
+    const readCursor: FeedSyncStore['readCursor'] = (id) =>
+      sql.unsafe<CursorRow>(statements.selectCursor, [id.feed, id.sink]).pipe(
+        Effect.mapError(unreadable(`reading the cursor of sink '${id.sink}'`)),
+        Effect.map((rows) => cursorOf(rows[0])),
+      )
+
+    const casCursor: FeedSyncStore['casCursor'] = (args) =>
+      sql
+        .withTransaction(
+          Effect.gen(function* () {
+            const expected = Option.getOrUndefined(args.expected)
+            const now = yield* DateTime.now
+            const landed = yield* sql.unsafe(statements.casCursor, [
+              args.feed,
+              args.sink,
+              args.holder,
+              args.next.generation.toString(),
+              args.next.seq.toString(),
+              expected === undefined ? null : expected.generation.toString(),
+              expected === undefined ? null : expected.seq.toString(),
+              String(DateTime.toEpochMillis(now)),
+            ])
+            if (landed.length === 0) {
+              return yield* new CursorSuperseded({
+                feed: args.feed,
+                sink: args.sink,
+              })
+            }
+            const change = args.parked
+            if (change._tag === 'Replace') {
+              yield* sql.unsafe(statements.clearParked, [args.feed, args.sink])
+            } else if (change.unpark.length > 0) {
+              yield* sql.unsafe(statements.unparkKeys, [
+                args.feed,
+                args.sink,
+                JSON.stringify(change.unpark),
+              ])
+            }
+            if (change.park.length > 0) {
+              yield* sql.unsafe(statements.parkKeys, [
+                args.feed,
+                args.sink,
+                parkJson(change.park),
+              ])
+            }
+          }),
+        )
+        .pipe(
+          Effect.catchTag('SqlError', (cause) =>
+            Effect.fail(
+              writeFailed(`moving the cursor of sink '${args.sink}'`)(cause),
+            ),
+          ),
+        )
+
+    const readParked: FeedSyncStore['readParked'] = (id) =>
+      sql.unsafe<ParkedRow>(statements.selectParked, [id.feed, id.sink]).pipe(
+        Effect.mapError(
+          unreadable(`reading the parked records of sink '${id.sink}'`),
+        ),
+        Effect.map((rows) => rows.map(parkedOf)),
+      )
+
+    const retryParked: FeedSyncStore['retryParked'] = (args) =>
+      sql
+        .withTransaction(
+          Effect.gen(function* () {
+            yield* sql.unsafe(statements.lockFeedExclusive, [args.feed])
+            const unparked = yield* sql.unsafe(statements.unparkOne, [
+              args.feed,
+              args.sink,
+              args.key,
+            ])
+            if (unparked.length === 0) {
+              return false
+            }
+            yield* sql.unsafe(statements.restampRow, [args.feed, args.key])
+            return true
+          }),
+        )
+        .pipe(
+          Effect.mapError(
+            writeFailed(
+              `retrying parked record '${args.key}' of sink '${args.sink}'`,
+            ),
+          ),
+        )
+
+    const collectTombstones: FeedSyncStore['collectTombstones'] = (feed) =>
+      sql.unsafe(statements.collectTombstones, [feed]).pipe(
+        Effect.mapError(
+          writeFailed(`collecting the tombstones of feed '${feed}'`),
+        ),
+        Effect.map((rows) => rows.length),
+      )
+
+    const removeSink: FeedSyncStore['removeSink'] = (id) =>
+      sql
+        .withTransaction(
+          Effect.gen(function* () {
+            const lock = yield* sql.unsafe<{ readonly taken: boolean }>(
+              statements.tryAdvisoryXactLock,
+              [advisoryLockName(context, id)],
+            )
+            if (lock[0]?.taken !== true) {
+              return yield* new LeaseHeld(id)
+            }
+            const now = yield* Clock.currentTimeMillis
+            const cursor = yield* sql.unsafe(
+              statements.deleteCursorUnlessLeased,
+              [id.feed, id.sink, String(now)],
+            )
+            if (cursor.length === 0) {
+              const left = yield* sql.unsafe(statements.cursorExists, [
+                id.feed,
+                id.sink,
+              ])
+              if (left.length > 0) {
+                return yield* new LeaseHeld(id)
+              }
+            }
+            const parked = yield* sql.unsafe(statements.deleteSinkParked, [
+              id.feed,
+              id.sink,
+            ])
+            return cursor.length > 0 || parked.length > 0
+          }),
+        )
+        .pipe(
+          Effect.catchTag('SqlError', (cause) =>
+            Effect.fail(writeFailed(`removing sink '${id.sink}'`)(cause)),
+          ),
+        )
+
+    const describe: FeedSyncStore['describe'] = (feed) =>
+      Effect.gen(function* () {
+        const head = yield* sql.unsafe<{ readonly head: string }>(
+          statements.selectHead,
+          [feed],
+        )
+        const sinks = yield* sql.unsafe<SinkRow>(statements.describeSinks, [
+          feed,
+        ])
+        const parked = yield* sql.unsafe<ParkedRow & { readonly sink: string }>(
+          statements.describeParked,
+          [feed],
+        )
+        const parkedBySink = new Map<string, Array<ParkedRecord>>()
+        for (const record of parked) {
+          const list = parkedBySink.get(record.sink)
+          if (list === undefined) {
+            parkedBySink.set(record.sink, [parkedOf(record)])
+          } else {
+            list.push(parkedOf(record))
+          }
+        }
+        const description: FeedSyncDescription = {
+          head: FeedSeq.make(bigintOf(head[0]?.head ?? '0')),
+          sinks: sinks.map(
+            (row): SinkSyncState => ({
+              sink: SinkName.make(row.sink),
+              cursor: cursorOf(row),
+              lastAdvance: instantOf(row.last_advance_at_ms),
+              lease:
+                row.lease_holder === null
+                  ? Option.none()
+                  : Option.some({
+                      holder: LeaseHolder.make(row.lease_holder),
+                      expiresAt: instantOf(row.lease_expires_at_ms),
+                    }),
+              parked: parkedBySink.get(row.sink) ?? [],
+              pending: Number(row.pending),
+            }),
+          ),
+        }
+        return description
+      }).pipe(Effect.mapError(unreadable(`describing feed '${feed}'`)))
+
+    const store: FeedSyncStore = {
+      acquireLease,
+      readCursor,
+      casCursor,
+      readParked,
+      retryParked,
+      collectTombstones,
+      removeSink,
+      describe,
+    }
+    return store
+  })

package/src/withFeed.ts

@@ -0,0 +1,148 @@
+/**
+ * `withFeed` — a `ProjectionStore` decorated so that resetting the checkpoint a
+ * feed is bound to also empties the feed and moves its generation on, in ONE
+ * transaction.
+ *
+ * ## Why a decorator
+ *
+ * Nothing in the library rebuilds a read model; a rebuild is an application
+ * resetting a checkpoint and letting the runner replay. A feed has to know when
+ * that happens, because a sink holding the old copy must be rebuilt from the
+ * replay rather than patched by it: a record the replay never writes again would
+ * otherwise survive in the sink. The reset is the one moment both facts are true,
+ * so the generation bump rides it.
+ *
+ * What it adds is the FEED's half of a rebuild and nothing more. A rebuild resets
+ * the checkpoint, drops the view, and starts again from `ORIGIN`; this resets the
+ * checkpoint and the feed, and the view is still the consumer's to drop. So
+ * rebuilding a DURABLE view means emptying it yourself before the replay, exactly
+ * as the port's own `resetCheckpoint` leaves it. Skip that and a row the replay
+ * never writes again survives in the view, and a read model whose `apply` reads
+ * its view back (a running count, say) builds on the old rows, which the feed
+ * then mirrors to every sink.
+ *
+ * The decorator keeps the port's three methods and replaces only
+ * `resetCheckpoint`, so it works through `forKey` exactly as the store does, and
+ * nothing about the read side has to know a feed exists.
+ *
+ * ## One transaction, and the client it needs
+ *
+ * The reset bumps the generation, deletes the checkpoint row through the store's
+ * own `resetCheckpoint`, and deletes the feed's rows, all inside one
+ * `withTransaction` on the client this decorator was built over. The store's own
+ * statement joins that transaction only when the store was built from the SAME
+ * client instance (ADR-0018 §3), which `@kairos-es/read-postgres`'s
+ * `makeSqlProjectionStore` is when both are built under one `SqlClient`. Build
+ * them apart and the checkpoint delete commits on its own: the reset still
+ * happens, but not atomically with the bump.
+ *
+ * The generation is bumped FIRST. That takes the feed lock exclusively, so a
+ * materialisation commit in flight finishes before the reset proceeds, and one
+ * that starts after it waits; and it takes the locks in the same order a commit
+ * does (the feed, then the checkpoint), so the two cannot deadlock. A commit that
+ * was mid-flight with the old checkpoint then loses its guard and rolls its feed
+ * rows back with its view.
+ *
+ * ## The binding
+ *
+ * A feed serves ONE materialisation, because one writer is what keeps its `seq`
+ * ascending in commit order. The binding is recorded on the feed's generations
+ * row the first time the feed is decorated, and checked at every construction
+ * after that, so a feed bound to another checkpoint key is refused across
+ * processes, not only within one. A refusal is a defect, as the read side's
+ * construction gate's are.
+ *
+ * The decorated store must be DURABLE. This feed is, and the feed's rows are
+ * written inside the store's commit transaction: an ephemeral store opens no
+ * transaction for them to join.
+ */
+import type { FeedWriteFailed } from '@kairos-es/feed'
+import {
+  type CheckpointKey,
+  type ProjectionStore,
+  ProjectionStoreError,
+} from '@kairos-es/read'
+import { Effect } from 'effect'
+import { SqlClient } from 'effect/sql'
+import { contextOf, type PostgresFeed } from './feed'
+import { writeFailed } from './internal/context'
+
+const sameKey = (left: CheckpointKey, right: CheckpointKey): boolean =>
+  left.projection === right.projection && left.partition === right.partition
+
+/**
+ * `store`, with `resetCheckpoint` for `key` also emptying `feed` and moving its
+ * generation on; see the module doc. Every other key passes through untouched.
+ */
+export const withFeed = <D>(
+  store: ProjectionStore,
+  feed: PostgresFeed<D>,
+  key: CheckpointKey,
+): Effect.Effect<ProjectionStore, FeedWriteFailed, SqlClient.SqlClient> =>
+  Effect.gen(function* () {
+    const context = contextOf(feed as PostgresFeed<unknown>)
+    if (context === undefined) {
+      return yield* Effect.die(
+        new Error(
+          `withFeed: feed '${feed.feed}' was not made by makePostgresFeed`,
+        ),
+      )
+    }
+    if (store.durability !== 'durable') {
+      return yield* Effect.die(
+        new Error(
+          `withFeed: feed '${feed.feed}' is written inside the store's commit transaction, and an ephemeral store opens none`,
+        ),
+      )
+    }
+    const { statements } = context
+    const sql = (yield* SqlClient.SqlClient).withoutTransforms()
+
+    const bound = yield* sql
+      .unsafe<{
+        readonly projection: string | null
+        readonly partition: string | null
+      }>(statements.bindFeed, [feed.feed, key.projection, key.partition])
+      .pipe(Effect.mapError(writeFailed(`binding feed '${feed.feed}'`)))
+    const binding = bound[0]
+    if (
+      binding !== undefined &&
+      (binding.projection !== key.projection ||
+        binding.partition !== key.partition)
+    ) {
+      return yield* Effect.die(
+        new Error(
+          `withFeed: feed '${feed.feed}' is already bound to projection '${binding.projection}' (partition '${binding.partition}'), and one feed serves one materialisation`,
+        ),
+      )
+    }
+
+    const resetBound = (reset: CheckpointKey) =>
+      sql
+        .withTransaction(
+          Effect.gen(function* () {
+            yield* sql.unsafe(statements.bumpGeneration, [feed.feed])
+            yield* store.resetCheckpoint(reset)
+            yield* sql.unsafe(statements.deleteFeedRows, [feed.feed])
+          }),
+        )
+        .pipe(
+          Effect.catchTag('SqlError', (cause) =>
+            Effect.fail(
+              new ProjectionStoreError({
+                reason: `resetting projection '${reset.projection}' partition '${reset.partition}' and its feed '${feed.feed}' failed: ${cause.message}`,
+                cause,
+              }),
+            ),
+          ),
+        )
+
+    const decorated: ProjectionStore = {
+      durability: store.durability,
+      readCheckpoint: store.readCheckpoint,
+      commit: store.commit,
+      resetCheckpoint: (reset) =>
+        sameKey(reset, key) ? resetBound(reset) : store.resetCheckpoint(reset),
+    }
+    return decorated
+  })