@kairos-es/feed-postgres 0.0.0-stage → 0.0.0

package/src/lease.ts

@@ -0,0 +1,503 @@
+/**
+ * The sync worker's lease on Postgres, in the two kinds `FeedSyncStore`'s doc
+ * describes, chosen in config to suit the infrastructure in front of the
+ * database.
+ *
+ * ## Why there are two
+ *
+ * A worker lease is what keeps a feed's sends to a sink in order: a sink has no
+ * transaction, so a rival worker's stale send lands before any compare-and-set can
+ * catch it (ADR-0019 §5). The two kinds trade the same two things against each
+ * other — what the connection must be, and how long a lapsed worker can go on
+ * sending — and neither wins on both.
+ *
+ * - **`leaseRow`, the default.** The holder and an expiry live on the sink's
+ *   cursor row, taken with a guarded upsert (only when free, expired, or held by
+ *   an advisory worker whose session is gone; see below) and renewed by a
+ *   heartbeat forked into the lease's scope. Every statement is an ordinary short
+ *   one, and the take one short transaction, so it works through a TRANSACTION
+ *   POOLER, which drops session state and so rules out a session lock, and it
+ *   needs no reserved connection. Its cost is a window. `ensureHeld` fails a
+ *   MARGIN before the stored expiry: once the last successful renewal is
+ *   `ttl - margin` old by the worker's own `Clock`, that renewal's expiry, which is
+ *   when a rival may take the row, is at most `margin` away. So a call to the sink
+ *   that `ensureHeld` let through, and that lands within `margin` of that check,
+ *   pauses included, lands before any rival can hold the lease, however its
+ *   renewals have been failing meanwhile, as long as the two workers' clocks
+ *   agree. A call that outlasts the margin, whether it is slow or its worker
+ *   pauses (a long GC pause, a network stall), can land after a rival took over:
+ *   ONE stale call, since the next `ensureHeld` fails; `FeedSyncStore.ts` owns what
+ *   that call costs. The margin is `ttl / 3` unless configured, and must leave the
+ *   heartbeat room under the lease (`heartbeat + margin < ttl`), or `ensureHeld`
+ *   would fail between two renewals that both landed at once. Each renewal is
+ *   bounded by the heartbeat and the next starts on the next tick, and the age
+ *   `ensureHeld` judges runs from when a renewal STARTED. So a renewal that lands
+ *   is at most `heartbeat` late, and the worst age `ensureHeld` meets is
+ *   `heartbeat` plus that latency with every renewal landing, and `2 * heartbeat`
+ *   plus it with one lost. A worker therefore goes on sending, without a single
+ *   failed `ensureHeld`, through renewals that are slow when
+ *   `2 * heartbeat + margin <= ttl`, and through one that fails or hangs when
+ *   `3 * heartbeat + margin <= ttl`. The defaults satisfy the second; the
+ *   construction check enforces only the first's zero-latency form, so a tighter
+ *   config is accepted and trades those guarantees for a shorter `ttl`.
+ * - **`advisoryLock`.** A session `pg_try_advisory_lock` on a connection RESERVED
+ *   from a client the consumer dedicates to it, keyed by a hash of the prefix, the
+ *   feed and the sink. The lock goes when the session goes, and `ensureHeld` asks
+ *   that same session, before every call that changes the sink, whether it still
+ *   holds the lock; any failure to answer is a lost lease. So a process pause
+ *   opens no stale window, since the lock survives it. What a check cannot see is
+ *   the session dying AFTER it: the call it let through is already under way and
+ *   runs to its end, so it can land as late as its whole duration after the session
+ *   died, and after a rival has taken the lock and sent. That is still ONE stale
+ *   call, since the next `ensureHeld` asks a session that is gone, but a sink call
+ *   can be long — a resync's `publish` most of all. The price is a connection per
+ *   running worker, and a connection that is genuinely a SESSION: a transaction
+ *   pooler would hand the lock's session to somebody else between statements. A
+ *   client this package can recognise as one is refused at construction; see
+ *   {@link pooledConnectionFault} for what it can and cannot recognise.
+ *
+ * ## Fencing is the same for both
+ *
+ * Both kinds record the holder on the cursor row, and the cursor compare-and-set
+ * requires it, so a worker whose lease has gone cannot move the cursor whichever
+ * kind it held. Mixing the two kinds on one feed and sink fails safe, because
+ * neither takes the row from a LIVE holder of the other. A lease-row worker takes
+ * over an advisory holder (one with no expiry to lapse) only while it holds that
+ * sink's advisory lock for its own transaction, which a live advisory session
+ * refuses; so a crashed advisory worker, whose scope never closed to clear the row,
+ * does not leave the sink unleasable by every lease-row worker after it. An
+ * advisory worker that wins the lock but finds a lease row unexpired on the row lets
+ * the lock go and fails `LeaseHeld`. Whichever took the row first keeps it while it
+ * lives. The advisory worker likewise takes over an ADVISORY holder recorded on the
+ * row, because holding the lock proves that session is gone; it records itself
+ * from the session that holds the lock, and only while it still does, so a session
+ * that died after its try-lock cannot overwrite the successor that took the lock
+ * from it.
+ */
+import {
+  type FeedLease,
+  type FeedSinkId,
+  type FeedUnreadable,
+  type FeedWriteFailed,
+  LeaseHeld,
+  LeaseHolder,
+  LeaseLost,
+} from '@kairos-es/feed'
+import {
+  Clock,
+  Duration,
+  Effect,
+  Option,
+  Random,
+  Redacted,
+  Ref,
+  type Scope,
+} from 'effect'
+import type { SqlClient } from 'effect/sql'
+import { type FeedContext, unreadable, writeFailed } from './internal/context'
+
+/** The lease kind, as config carries it. See the module doc. */
+export type FeedLeaseConfig =
+  | {
+      readonly _tag: 'LeaseRow'
+      /** How long a lease lasts without a renewal. */
+      readonly ttl: Duration.Input
+      /** How often the holder renews it; shorter than `ttl`. */
+      readonly heartbeat: Duration.Input
+      /**
+       * How long before the stored expiry `ensureHeld` starts failing: a sink call
+       * that takes less than this cannot land after a rival took the lease. See
+       * the module doc. `ttl / 3` when absent; `heartbeat + margin` must be under
+       * `ttl`.
+       */
+      readonly margin?: Duration.Input
+    }
+  | {
+      readonly _tag: 'AdvisoryLock'
+      /**
+       * A client used for nothing but leases, each lease reserving one connection
+       * from it for its lifetime. Its lifecycle is the consumer's, exactly as the
+       * main client's is.
+       */
+      readonly connection: SqlClient.SqlClient
+    }
+
+/** The default lease length. */
+export const DEFAULT_LEASE_TTL: Duration.Duration = Duration.seconds(30)
+
+/**
+ * The default renewal interval: a fifth of the lease. With the default margin, a
+ * third, `3 * heartbeat + margin` is under the `ttl`, so a worker goes on sending,
+ * with no failed `ensureHeld`, through one renewal that fails or hangs. However
+ * many fail, it stops sending a margin before its lease could lapse. The module
+ * doc has the arithmetic.
+ */
+export const DEFAULT_LEASE_HEARTBEAT: Duration.Duration = Duration.seconds(6)
+
+/** A lease row, the default; see the module doc. */
+export const leaseRow = (
+  options: {
+    readonly ttl?: Duration.Input
+    readonly heartbeat?: Duration.Input
+    readonly margin?: Duration.Input
+  } = {},
+): FeedLeaseConfig => ({
+  _tag: 'LeaseRow',
+  ttl: options.ttl ?? DEFAULT_LEASE_TTL,
+  heartbeat: options.heartbeat ?? DEFAULT_LEASE_HEARTBEAT,
+  ...(options.margin === undefined ? {} : { margin: options.margin }),
+})
+
+/** A session advisory lock on a dedicated connection; see the module doc. */
+export const advisoryLock = (options: {
+  readonly connection: SqlClient.SqlClient
+}): FeedLeaseConfig => ({
+  _tag: 'AdvisoryLock',
+  connection: options.connection,
+})
+
+/**
+ * Why `client` cannot carry a session lock, when this package can tell, or
+ * `undefined`.
+ *
+ * What it can tell is limited to what a client says about where it connects,
+ * which `@effect/sql-pg`'s client exposes as `config` and a generic `SqlClient`
+ * does not. It recognises the two pooler conventions that show in an address: a
+ * Neon pooled endpoint (a host containing `-pooler`), and PgBouncer's default
+ * port, 6432, or a `pgbouncer=true` URL parameter. A pooler on any other address,
+ * or behind a client that exposes nothing, passes; that is a wiring the consumer
+ * owns. The URL is read only to judge it and is never repeated in the message.
+ */
+export const pooledConnectionFault = (
+  client: SqlClient.SqlClient,
+): string | undefined => {
+  const config = (client as unknown as { readonly config?: unknown }).config
+  if (config === null || typeof config !== 'object') {
+    return undefined
+  }
+  const { host, port, url } = config as {
+    readonly host?: unknown
+    readonly port?: unknown
+    readonly url?: unknown
+  }
+  const hosts: Array<string> = typeof host === 'string' ? [host] : []
+  const ports: Array<number> = typeof port === 'number' ? [port] : []
+  let pgbouncer = false
+  const rawUrl = Redacted.isRedacted(url)
+    ? Redacted.value(url)
+    : typeof url === 'string'
+      ? url
+      : undefined
+  if (typeof rawUrl === 'string') {
+    try {
+      const parsed = new URL(rawUrl)
+      hosts.push(parsed.hostname)
+      if (parsed.port !== '') {
+        ports.push(Number(parsed.port))
+      }
+      pgbouncer = parsed.searchParams.get('pgbouncer') === 'true'
+    } catch {
+      // Not a URL this check can read; the driver will say so itself.
+    }
+  }
+  const signs = [
+    ...(hosts.some((h) => h.includes('-pooler'))
+      ? ['a "-pooler" host, which is a pooled endpoint']
+      : []),
+    ...(ports.includes(6432) ? ["port 6432, PgBouncer's default"] : []),
+    ...(pgbouncer ? ['a pgbouncer=true parameter'] : []),
+  ]
+  return signs.length === 0
+    ? undefined
+    : `the advisoryLock connection looks like a transaction pooler (${signs.join('; ')}), which hands a session to other clients between statements and so cannot hold a session lock; give it a direct connection, or use leaseRow`
+}
+
+/** How a sync store takes a lease. Package-internal. */
+export type AcquireLease = (
+  id: FeedSinkId,
+) => Effect.Effect<
+  FeedLease,
+  LeaseHeld | FeedUnreadable | FeedWriteFailed,
+  Scope.Scope
+>
+
+/** A positive, finite duration, or a construction-time throw naming it. */
+const positive = (label: string, input: Duration.Input): Duration.Duration => {
+  const decoded = Duration.fromInputUnsafe(input)
+  if (!Duration.isFinite(decoded) || Duration.toMillis(decoded) <= 0) {
+    throw new Error(
+      `feed-postgres: ${label} must be a positive, finite duration`,
+    )
+  }
+  return decoded
+}
+
+/** A fresh holder id, from `Random`, never the host. */
+const newHolder = (kind: string): Effect.Effect<LeaseHolder> =>
+  Effect.gen(function* () {
+    const high = yield* Random.nextIntBetween(0, 2 ** 31)
+    const low = yield* Random.nextIntBetween(0, 2 ** 31)
+    return LeaseHolder.make(
+      `pg-${kind}:${high.toString(16)}${low.toString(16).padStart(8, '0')}`,
+    )
+  })
+
+/**
+ * The advisory lock's name for one feed and sink, hashed server-side into its
+ * key. Package-internal, and the one home of the name: `removeSink` tries the
+ * same lock to tell whether an advisory worker is live, so the two must agree.
+ *
+ * @internal
+ */
+export const advisoryLockName = (
+  context: FeedContext,
+  id: FeedSinkId,
+): string =>
+  JSON.stringify([
+    'kairos-es/feed',
+    context.names.schema,
+    context.names.tablePrefix,
+    id.feed,
+    id.sink,
+  ])
+
+type LeaseRowConfig = Extract<FeedLeaseConfig, { readonly _tag: 'LeaseRow' }>
+type AdvisoryLockConfig = Extract<
+  FeedLeaseConfig,
+  { readonly _tag: 'AdvisoryLock' }
+>
+
+/**
+ * The lease-row kind; see the module doc. Throws on a heartbeat not under its ttl,
+ * or a margin that leaves the heartbeat no room under it.
+ */
+const leaseRowAcquirer = (
+  config: LeaseRowConfig,
+  sql: SqlClient.SqlClient,
+  context: FeedContext,
+): AcquireLease => {
+  const { statements } = context
+  const ttl = Duration.toMillis(positive('lease ttl', config.ttl))
+  const heartbeat = positive('lease heartbeat', config.heartbeat)
+  if (Duration.toMillis(heartbeat) >= ttl) {
+    throw new Error(
+      'feed-postgres: the lease heartbeat must be shorter than its ttl, or the lease lapses between renewals',
+    )
+  }
+  const margin =
+    config.margin === undefined
+      ? ttl / 3
+      : Duration.toMillis(positive('lease margin', config.margin))
+  if (Duration.toMillis(heartbeat) + margin >= ttl) {
+    throw new Error(
+      'feed-postgres: the lease heartbeat plus its margin must be shorter than its ttl, or ensureHeld fails between two renewals that both landed',
+    )
+  }
+  // `ensureHeld` fails once the last renewal is this old: its stored expiry is
+  // then at most `margin` away.
+  const usable = ttl - margin
+  return (id) =>
+    Effect.gen(function* () {
+      const holder = yield* newHolder('row')
+      const now = yield* Clock.currentTimeMillis
+      const taken = yield* sql
+        .withTransaction(
+          Effect.gen(function* () {
+            // Held to this transaction's end, so an ADVISORY holder on the row can
+            // be taken over only while no advisory session holds its lock, and no
+            // advisory worker can take the lock before this commits.
+            const lock = yield* sql.unsafe<{ readonly taken: boolean }>(
+              statements.tryAdvisoryXactLock,
+              [advisoryLockName(context, id)],
+            )
+            return yield* sql.unsafe(statements.takeLeaseRow, [
+              id.feed,
+              id.sink,
+              holder,
+              String(now + ttl),
+              String(now),
+              lock[0]?.taken === true,
+            ])
+          }),
+        )
+        .pipe(Effect.mapError(writeFailed('taking the feed lease')))
+      if (taken.length === 0) {
+        return yield* new LeaseHeld(id)
+      }
+      const state = yield* Ref.make({ renewedAt: now, lost: false })
+      yield* Effect.addFinalizer(() =>
+        Effect.ignore(
+          sql.unsafe(statements.releaseLease, [id.feed, id.sink, holder]),
+        ),
+      )
+      const renew = Effect.gen(function* () {
+        const at = yield* Clock.currentTimeMillis
+        const renewed = yield* sql.unsafe(statements.renewLeaseRow, [
+          id.feed,
+          id.sink,
+          holder,
+          String(at + ttl),
+        ])
+        if (renewed.length === 0) {
+          yield* Ref.update(state, (s) => ({ ...s, lost: true }))
+        } else {
+          yield* Ref.update(state, (s) => ({ ...s, renewedAt: at }))
+        }
+      }).pipe(
+        // Bounded by the heartbeat, so a renewal that hangs is abandoned in time
+        // for the next tick rather than holding every later one behind it. An
+        // abandoned renewal whose cancel comes too late can still land: it touches
+        // only this holder's own row, never moves its expiry back
+        // (`renewLeaseRow`), and leaves `renewedAt` at the last renewal this fibre
+        // saw land, so `ensureHeld` errs early, never late.
+        Effect.timeoutOption(heartbeat),
+        Effect.flatMap((done) =>
+          Option.isSome(done)
+            ? Effect.void
+            : Effect.logWarning('feed lease renewal timed out', {
+                feed: id.feed,
+                sink: id.sink,
+              }),
+        ),
+        // A renewal that could not be made is not yet a lost lease: the age
+        // check in `ensureHeld` decides that, a margin before the lease lapses.
+        Effect.catchTag('SqlError', (cause) =>
+          Effect.logWarning('feed lease renewal failed', {
+            feed: id.feed,
+            sink: id.sink,
+            reason: cause.message,
+          }),
+        ),
+      )
+      // On a fixed cadence from the take: a renewal never outlasts the interval,
+      // so each one starts on its own tick whatever became of the one before. A
+      // worker that falls behind (a pause) renews once at once, and does not
+      // replay the ticks it missed.
+      const interval = Duration.toMillis(heartbeat)
+      yield* Effect.forkScoped(
+        Effect.gen(function* () {
+          let next = now + interval
+          while (true) {
+            const at = yield* Clock.currentTimeMillis
+            if (next > at) {
+              yield* Effect.sleep(Duration.millis(next - at))
+            }
+            yield* renew
+            next = Math.max(next + interval, yield* Clock.currentTimeMillis)
+          }
+        }),
+      )
+      const lease: FeedLease = {
+        holder,
+        ensureHeld: Effect.gen(function* () {
+          const s = yield* Ref.get(state)
+          const at = yield* Clock.currentTimeMillis
+          if (s.lost || at - s.renewedAt >= usable) {
+            return yield* new LeaseLost(id)
+          }
+        }),
+      }
+      return lease
+    })
+}
+
+/** The advisory-lock kind; see the module doc. Throws on a recognisably pooled connection. */
+const advisoryLockAcquirer = (
+  config: AdvisoryLockConfig,
+  sql: SqlClient.SqlClient,
+  context: FeedContext,
+): AcquireLease => {
+  const fault = pooledConnectionFault(config.connection)
+  if (fault !== undefined) {
+    throw new Error(`feed-postgres: ${fault}`)
+  }
+  const { connection } = config
+  const { statements } = context
+  return (id) =>
+    Effect.gen(function* () {
+      const lockName = advisoryLockName(context, id)
+      const session = yield* connection.reserve.pipe(
+        Effect.mapError(unreadable('reserving the feed lease connection')),
+      )
+      const taken: ReadonlyArray<{ readonly taken?: unknown }> = yield* session
+        .execute(statements.tryAdvisoryLock, [lockName], undefined)
+        .pipe(Effect.mapError(unreadable('taking the feed advisory lock')))
+      if (taken[0]?.taken !== true) {
+        return yield* new LeaseHeld(id)
+      }
+      yield* Effect.addFinalizer(() =>
+        Effect.ignore(
+          session.execute(statements.advisoryUnlock, [lockName], undefined),
+        ),
+      )
+      const holder = yield* newHolder('advisory')
+      const now = yield* Clock.currentTimeMillis
+      // On the lock's own session, and conditional on it still holding the lock:
+      // a session that died since the try-lock cannot run it, and one that lost
+      // the lock lands nothing, so neither overwrites a successor's claim.
+      const claimed: ReadonlyArray<unknown> = yield* session
+        .execute(
+          statements.claimAdvisoryHolder,
+          [id.feed, id.sink, holder, String(now), lockName],
+          undefined,
+        )
+        .pipe(Effect.mapError(writeFailed('recording the feed lease holder')))
+      if (claimed.length === 0) {
+        // A live lease row holds the sink, or this session no longer holds the
+        // lock, so this worker must not send. The lock
+        // guards nothing now, so it is let go at once rather than when the
+        // caller's scope closes, and another advisory worker may try the moment
+        // that lease row lapses. The finalizer's second unlock then finds nothing
+        // held, which is harmless.
+        yield* Effect.ignore(
+          session.execute(statements.advisoryUnlock, [lockName], undefined),
+        )
+        return yield* new LeaseHeld(id)
+      }
+      yield* Effect.addFinalizer(() =>
+        Effect.ignore(
+          sql.unsafe(statements.releaseLease, [id.feed, id.sink, holder]),
+        ),
+      )
+      const held: Effect.Effect<boolean> = session
+        .execute(statements.holdsAdvisoryLock, [lockName], undefined)
+        .pipe(
+          Effect.map(
+            (rows: ReadonlyArray<{ readonly held?: unknown }>) =>
+              rows[0]?.held === true,
+          ),
+          Effect.catchTag('SqlError', () => Effect.succeed(false)),
+        )
+      const lease: FeedLease = {
+        holder,
+        // Asked of the session that holds the lock, so a dead session cannot
+        // answer, and an unanswered question is a lost lease.
+        ensureHeld: Effect.flatMap(held, (yes) =>
+          yes ? Effect.void : Effect.fail(new LeaseLost(id)),
+        ),
+      }
+      return lease
+    })
+}
+
+/**
+ * Build the acquirer for `config`, checking it first: a `heartbeat` that is not
+ * shorter than its `ttl`, a `margin` that leaves the heartbeat no room under the
+ * `ttl`, or an `advisoryLock` connection recognisably pooled,
+ * throws here, which the store's factory turns into a defect at construction.
+ * The one place the lease kind is dispatched on.
+ *
+ * @internal
+ */
+export const makeAcquireLease = (
+  config: FeedLeaseConfig,
+  sql: SqlClient.SqlClient,
+  context: FeedContext,
+): AcquireLease => {
+  switch (config._tag) {
+    case 'LeaseRow':
+      return leaseRowAcquirer(config, sql, context)
+    case 'AdvisoryLock':
+      return advisoryLockAcquirer(config, sql, context)
+  }
+}