@kairos-es/feed-postgres 0.0.0-stage → 0.0.0

package/package.json

@@ -1,6 +1,38 @@
 {
   "name": "@kairos-es/feed-postgres",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.0.0",
+  "description": "Postgres change-feed tables, feed writer, feed source, sync store and worker lease for kairos-es",
+  "license": "BSD-3-Clause",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/neverbland/kairos-es.git",
+    "directory": "packages/feed-postgres"
+  },
+  "sideEffects": [],
+  "homepage": "https://github.com/neverbland/kairos-es",
+  "peerDependencies": {
+    "effect": "^4.0.5",
+    "@kairos-es/core": "^1.0.0-alpha.5",
+    "@kairos-es/feed": "^0.0.0",
+    "@kairos-es/read": "^1.0.0-alpha.5"
+  },
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/dts-cjs/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/dts-esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/dts-cjs/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "typesVersions": {
+    "*": {}
+  }
 }

package/src/config.ts

@@ -0,0 +1,46 @@
+/**
+ * Public configuration for the Postgres change feed.
+ *
+ * A feed backend is defined by its `{ schema, tablePrefix }` naming pair alone:
+ * every object it owns is DERIVED from it (`internal/ddl.ts`), so one config
+ * value is one isolated set of feed tables, cohabiting per prefix beside the
+ * event store's objects and `@kairos-es/read-postgres`'s checkpoint table. Pass
+ * the same pair to all three and one prefix names one complete world: the log,
+ * the checkpoints, and the feeds the read models write.
+ *
+ * It is a `Schema` rather than a bare interface so that a malformed value fails
+ * LOUDLY at construction rather than obscurely at the first statement: an empty or
+ * dotted atom (a dot would break the qualified-name split), or a pair whose
+ * derived names would exceed Postgres's 63-byte identifier limit and be silently
+ * truncated. The shape and its checks are `@kairos-es/read-postgres`'s, written
+ * again here rather than imported, because each backend measures only the names it
+ * creates and a sibling backend's internals are not a public surface.
+ */
+import { Schema } from 'effect'
+import { oversizedIdentifier } from './internal/ddl'
+
+/** A single identifier atom: non-empty and dot-free. */
+const IdentifierAtom = Schema.String.check(
+  Schema.isNonEmpty(),
+  Schema.isPattern(/^[^.]+$/, {
+    identifier: 'PostgresIdentifierAtom',
+    description: 'a non-empty identifier atom containing no "."',
+  }),
+)
+
+/**
+ * Configuration for the feed's tables.
+ *
+ * - `schema` — the Postgres schema that owns them (default `public`).
+ * - `tablePrefix` — the stem every name derives from (default `dcb_events`,
+ *   giving `dcb_events_feed_rows` and its siblings).
+ */
+export const FeedPostgresConfig = Schema.Struct({
+  schema: Schema.optionalKey(IdentifierAtom),
+  tablePrefix: Schema.optionalKey(IdentifierAtom),
+}).check(Schema.makeFilter((config) => oversizedIdentifier(config)))
+export type FeedPostgresConfig = typeof FeedPostgresConfig.Type
+
+/** Decode a config, throwing a `SchemaError` on any violation. */
+export const decodeFeedPostgresConfig =
+  Schema.decodeUnknownSync(FeedPostgresConfig)

package/src/ensureFeedSchema.ts

@@ -0,0 +1,32 @@
+/**
+ * `ensureFeedSchema` — the feed's idempotent DDL, as a payload for the consumer's
+ * migrator rather than a migrator of its own, exactly as `ensureCheckpointSchema`
+ * is for the read side and `ensureSchema` for the event store.
+ *
+ * It requires the GENERIC `SqlClient`, so it drops into an existing migrator:
+ *
+ * ```ts
+ * Migrator.fromRecord({
+ *   '0003_kairos_feed': ensureFeedSchema(config),
+ * })
+ * ```
+ *
+ * or runs once at boot. Every statement is `IF NOT EXISTS`, so running it again
+ * against existing tables keeps every row. Nothing in this package performs DDL at
+ * runtime: a feed cannot conjure its own tables, so schema ownership stays with
+ * whoever owns the migrations.
+ */
+import { Effect } from 'effect'
+import { SqlClient, type SqlError } from 'effect/sql'
+import { decodeFeedPostgresConfig, type FeedPostgresConfig } from './config'
+import { resolveFeedNames, runFeedDdl } from './internal/ddl'
+
+/** The feed tables' migration payload for `config`. */
+export const ensureFeedSchema = (
+  config: FeedPostgresConfig = {},
+): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
+  Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient
+    const names = resolveFeedNames(decodeFeedPostgresConfig(config))
+    yield* Effect.all(runFeedDdl(sql, names), { discard: true })
+  })

package/src/feed.ts

@@ -0,0 +1,284 @@
+/**
+ * The Postgres feed's two faces: the `BatchedFeedWriter` a read model's `apply`
+ * writes through, and the `FeedSource` a sync worker tails.
+ *
+ * ## The writer rides the read model's own client
+ *
+ * A feed row must exist if and only if the view write and the checkpoint advance
+ * that produced it landed (ADR-0019 §2). `@kairos-es/read-postgres`'s `commit` is
+ * one transaction over the client the store was built from, and `effect/sql`
+ * carries that transaction in the Effect CONTEXT under a key that belongs to that
+ * one client instance (ADR-0018 §3). So the writer does not capture a client: it
+ * reads `SqlClient` from the context AT WRITE TIME, which inside `commit`'s
+ * `viewWrites` is the read model's own, and its statements join the open
+ * transaction and roll back with it on a lost guard. That is why `SqlClient` sits
+ * in the writer's `RW` and not in the factory's requirements alone. A write
+ * through any OTHER client instance is not bracketed, and the exactly-once
+ * property does not extend to it.
+ *
+ * ## What one `write` sends
+ *
+ * The body's upserts and removes are buffered, last write per key winning, and
+ * land as at most three statements whatever the batch size: the feed lock (see
+ * `internal/statements.ts` for why the writer takes it), one upsert of every
+ * changed document and one tombstone of every removed key. The upsert writes
+ * nothing for a document equal to the stored one, compared as `jsonb`, so key
+ * order is no change; the tombstone writes nothing for a key that is absent or
+ * already a tombstone.
+ *
+ * Each document is encoded through its `Schema` before anything is sent. A
+ * document that fails to encode is a DEFECT in the read model, not a fault of the
+ * feed and not something a sink should ever be asked to refuse.
+ *
+ * ## The source
+ *
+ * `changesAfter` pages the feed's rows by `seq`, tombstones included, and decodes
+ * each document through the same `Schema`. A stored document that no longer
+ * decodes is a defect too: it means the schema changed without the feed being
+ * rebuilt, which retrying cannot mend.
+ *
+ * `listen` is optional and is a WAKE only. This package takes no reserved
+ * connection for a `LISTEN` of its own, as `@kairos-es/read-postgres` takes none,
+ * so the poll is the guarantee. {@link eventStoreWake} offers the event store's
+ * `subscribe` as a wake: it fires when an event is appended, which is BEFORE the
+ * projection has applied it and written the feed, so a woken read may find nothing
+ * new and the poll picks the row up. It shortens the wait; it never decides
+ * correctness.
+ */
+import {
+  DcbEventStore,
+  Position,
+  type Query,
+  type ReadLimit,
+} from '@kairos-es/core'
+import {
+  type BatchedFeedWriter,
+  FeedKey,
+  type FeedName,
+  type FeedRow,
+  FeedSeq,
+  type FeedSource,
+  type FeedUnreadable,
+  type FeedWriteFailed,
+  type FeedWriter,
+  Generation,
+} from '@kairos-es/feed'
+import { Effect, Option, Schema, Stream } from 'effect'
+import { SqlClient } from 'effect/sql'
+import type { FeedPostgresConfig } from './config'
+import {
+  bigintOf,
+  type FeedContext,
+  feedContext,
+  unreadable,
+  writeFailed,
+} from './internal/context'
+
+/** A Postgres feed: its writer, and the source a sync worker reads. */
+export interface PostgresFeed<D>
+  extends BatchedFeedWriter<D, SqlClient.SqlClient> {
+  readonly source: FeedSource<D>
+}
+
+/** Options for {@link makePostgresFeed}. */
+export interface PostgresFeedOptions<D, I> {
+  /** The `{ schema, tablePrefix }` pair; the same one the event store and read side take. */
+  readonly config?: FeedPostgresConfig
+  readonly feed: FeedName
+  /** The document's schema: what the writer encodes through and the source decodes through. */
+  readonly doc: Schema.Codec<D, I>
+  /** A wake for the sync worker, such as {@link eventStoreWake}'s. The poll is the guarantee. */
+  readonly listen?: Stream.Stream<unknown, FeedUnreadable>
+}
+
+/** The context a feed was built over, for `withFeed`. */
+const contexts = new WeakMap<object, FeedContext>()
+
+/** @internal */
+export const contextOf = (
+  feed: PostgresFeed<unknown>,
+): FeedContext | undefined => contexts.get(feed)
+
+type ChangeRow = {
+  readonly key: string
+  readonly doc: string | null
+  readonly seq: string
+  readonly event_position: string
+}
+
+/**
+ * Build a feed over the ambient `SqlClient`. Creates the feed's generations row
+ * if it is absent, at generation 0, so the writer's feed lock always has a row to
+ * take; that is the only write it makes. `ensureFeedSchema` must have run first.
+ */
+export const makePostgresFeed = <D, I>(
+  options: PostgresFeedOptions<D, I>,
+): Effect.Effect<PostgresFeed<D>, FeedWriteFailed, SqlClient.SqlClient> =>
+  Effect.gen(function* () {
+    const context = feedContext(options.config ?? {})
+    const { statements } = context
+    const feed = options.feed
+    const sql = (yield* SqlClient.SqlClient).withoutTransforms()
+    const encode = Schema.encodeEffect(options.doc)
+    const decode = Schema.decodeUnknownEffect(options.doc)
+
+    yield* sql
+      .unsafe(statements.ensureGeneration, [feed])
+      .pipe(
+        Effect.mapError(
+          writeFailed(`creating the generation of feed '${feed}'`),
+        ),
+      )
+
+    const write = <A, E, R>(
+      at: Position,
+      body: (writer: FeedWriter<D>) => Effect.Effect<A, E, R>,
+    ): Effect.Effect<A, E | FeedWriteFailed, R | SqlClient.SqlClient> =>
+      Effect.gen(function* () {
+        const buffer = new Map<FeedKey, Option.Option<D>>()
+        const set = (key: FeedKey, doc: Option.Option<D>) =>
+          Effect.sync(() => {
+            buffer.delete(key)
+            buffer.set(key, doc)
+          })
+        const result = yield* body({
+          upsert: (key, doc) => set(key, Option.some(doc)),
+          remove: (key) => set(key, Option.none()),
+        })
+        if (buffer.size === 0) {
+          return result
+        }
+        const upserts: Array<{ readonly key: FeedKey; readonly doc: I }> = []
+        const removes: Array<FeedKey> = []
+        for (const [key, doc] of buffer) {
+          if (Option.isSome(doc)) {
+            upserts.push({ key, doc: yield* Effect.orDie(encode(doc.value)) })
+          } else {
+            removes.push(key)
+          }
+        }
+        const upsertJson =
+          upserts.length === 0
+            ? undefined
+            : yield* Effect.try({
+                try: () => JSON.stringify(upserts),
+                catch: (cause) =>
+                  new Error(
+                    `feed '${feed}': an encoded document is not JSON-serialisable`,
+                    { cause },
+                  ),
+              }).pipe(Effect.orDie)
+        // The read model's own client, from the context at write time; see the
+        // module doc. Transform-free, for the same reason the read side's store
+        // is: the copy keeps the original's transaction key.
+        const client = (yield* SqlClient.SqlClient).withoutTransforms()
+        const position = at.toString()
+        yield* client
+          .withTransaction(
+            Effect.gen(function* () {
+              yield* client.unsafe(statements.lockFeedShared, [feed])
+              if (upsertJson !== undefined) {
+                yield* client.unsafe(statements.upsertRows, [
+                  feed,
+                  position,
+                  upsertJson,
+                ])
+              }
+              if (removes.length > 0) {
+                yield* client.unsafe(statements.tombstoneRows, [
+                  feed,
+                  position,
+                  JSON.stringify(removes),
+                ])
+              }
+            }),
+          )
+          .pipe(Effect.mapError(writeFailed(`writing feed '${feed}'`)))
+        return result
+      })
+
+    const decodeRow = (row: ChangeRow): Effect.Effect<FeedRow<D>> =>
+      Effect.gen(function* () {
+        // Bound to a `const` so the `null` check narrows inside the `try` below.
+        const stored = row.doc
+        const doc =
+          stored === null
+            ? Option.none<D>()
+            : Option.some(
+                yield* Effect.orDie(
+                  Effect.flatMap(
+                    Effect.try({
+                      try: () => JSON.parse(stored) as unknown,
+                      catch: (cause) =>
+                        new Error(
+                          `feed '${feed}': a stored document is not JSON`,
+                          {
+                            cause,
+                          },
+                        ),
+                    }),
+                    (json) => decode(json),
+                  ),
+                ),
+              )
+        return {
+          key: FeedKey.make(row.key),
+          doc,
+          seq: FeedSeq.make(bigintOf(row.seq)),
+          eventPosition: Position.make(bigintOf(row.event_position)),
+        }
+      })
+
+    const source: FeedSource<D> = {
+      feed,
+      generation: sql
+        .unsafe<{ readonly generation: string }>(statements.selectGeneration, [
+          feed,
+        ])
+        .pipe(
+          Effect.mapError(
+            unreadable(`reading the generation of feed '${feed}'`),
+          ),
+          Effect.map((rows) => {
+            const stored = rows[0]?.generation
+            return Generation.make(stored === undefined ? 0n : bigintOf(stored))
+          }),
+        ),
+      changesAfter: (after: FeedSeq, limit: ReadLimit) =>
+        sql
+          .unsafe<ChangeRow>(statements.changesAfter, [
+            feed,
+            after.toString(),
+            limit,
+          ])
+          .pipe(
+            Effect.mapError(unreadable(`reading feed '${feed}'`)),
+            Effect.flatMap((rows) => Effect.forEach(rows, decodeRow)),
+          ),
+      ...(options.listen === undefined ? {} : { listen: options.listen }),
+    }
+
+    const result: PostgresFeed<D> = { feed, write, source }
+    contexts.set(result, context)
+    return result
+  })
+
+/**
+ * A wake from the event store: one element per event matching `query` appended
+ * after the moment the stream is subscribed, and nothing else. Pass it as
+ * `listen`. Early by the projection's own lag, as the module doc says, so the
+ * worker's poll still bounds the latency; `query` should be the read model's own,
+ * so events it never applies wake nothing.
+ */
+export const eventStoreWake = (
+  query: Query,
+): Effect.Effect<Stream.Stream<unknown>, never, DcbEventStore> =>
+  Effect.gen(function* () {
+    const store = yield* DcbEventStore
+    const wake: Stream.Stream<unknown> = Stream.unwrap(
+      Effect.map(store.read(query), (response) =>
+        store.subscribe(query, response.head),
+      ),
+    ).pipe(Stream.scoped, Stream.as(undefined))
+    return wake
+  })

package/src/index.ts

@@ -0,0 +1,67 @@
+/**
+ * @kairos-es/feed-postgres — the Postgres change feed: the feed tables and their
+ * idempotent DDL, the `FeedWriter` that rides a read model's own `SqlClient`
+ * inside its commit, the `FeedSource`, the `FeedSyncStore` with its worker lease,
+ * and the store decorator that moves the feed's generation on a reset
+ * (ADR-0019).
+ *
+ * A package of its own rather than part of `@kairos-es/read-postgres`, so that a
+ * read model with no feed gains no `@kairos-es/feed` peer (ADR-0003). It needs
+ * only the GENERIC `SqlClient`, and takes no reserved connection unless the
+ * consumer chooses the advisory-lock lease and dedicates one to it.
+ *
+ * Where each part's argument lives:
+ *
+ * - `config.ts` and `internal/ddl.ts` — the `{ schema, tablePrefix }` pair and the
+ *   five objects derived from it; `ensureFeedSchema.ts` — the migration payload.
+ * - `internal/statements.ts` — every statement the package sends, and the feed
+ *   lock that keeps `seq` ascending in commit order.
+ * - `feed.ts` — the writer, the source and the wake.
+ * - `syncStore.ts` — the cursor, the parked list and their atomic steps.
+ * - `lease.ts` — the two lease kinds, and what each costs.
+ * - `withFeed.ts` — the reset, the generation and the binding.
+ *
+ * Wiring, end to end:
+ *
+ * ```ts
+ * const config = { schema: 'public', tablePrefix: 'dcb_events' }
+ * const key = checkpointKey(ProjectionId.make('course-search'))
+ *
+ * // 1. In YOUR migrator, beside `ensureSchema` and `ensureCheckpointSchema`:
+ * //    ensureFeedSchema(config)
+ *
+ * // 2. Where the read model is wired, under ONE SqlClient:
+ * const feed = yield* makePostgresFeed({ config, feed: FeedName.make('course-search'), doc: CourseDoc })
+ * const store = yield* withFeed(yield* makeSqlProjectionStore(config), feed, key)
+ * // …and in the read model's `apply`, inside `commit`:
+ * //    feed.write(lastPosition, (writer) => writer.upsert(FeedKey.make(id), toDoc(course)))
+ *
+ * // 3. Beside the projections, one worker per feed and sink:
+ * const syncStore = yield* makePostgresFeedSyncStore({ config })
+ * yield* Effect.forkScoped(runFeedSync({ source: feed.source, sink, store: syncStore }))
+ * ```
+ */
+export {
+  decodeFeedPostgresConfig,
+  FeedPostgresConfig,
+} from './config'
+export { ensureFeedSchema } from './ensureFeedSchema'
+export {
+  eventStoreWake,
+  makePostgresFeed,
+  type PostgresFeed,
+  type PostgresFeedOptions,
+} from './feed'
+export {
+  advisoryLock,
+  DEFAULT_LEASE_HEARTBEAT,
+  DEFAULT_LEASE_TTL,
+  type FeedLeaseConfig,
+  leaseRow,
+  pooledConnectionFault,
+} from './lease'
+export {
+  makePostgresFeedSyncStore,
+  type PostgresFeedSyncStoreOptions,
+} from './syncStore'
+export { withFeed } from './withFeed'

package/src/internal/context.ts

@@ -0,0 +1,44 @@
+/**
+ * What every part of this package shares about one config: the resolved names,
+ * the compiled statements, and the error mapping from `SqlError` to the feed's own
+ * two infrastructure errors.
+ *
+ * A `PostgresFeed` carries its context privately, in a `WeakMap` keyed by the
+ * feed value, so `withFeed` can reach the feed's names and statements without the
+ * public type growing a field no consumer should touch.
+ */
+import { FeedUnreadable, FeedWriteFailed } from '@kairos-es/feed'
+import type { SqlError } from 'effect/sql'
+import { decodeFeedPostgresConfig, type FeedPostgresConfig } from '../config'
+import { type ResolvedFeedNames, resolveFeedNames } from './ddl'
+import { compileFeedStatements, type FeedStatements } from './statements'
+
+/** One config's names and statements. */
+export interface FeedContext {
+  readonly names: ResolvedFeedNames
+  readonly statements: FeedStatements
+}
+
+/** Decode `config` and compile its statements; throws on a malformed config. */
+export const feedContext = (config: FeedPostgresConfig): FeedContext => {
+  const names = resolveFeedNames(decodeFeedPostgresConfig(config))
+  return { names, statements: compileFeedStatements(names) }
+}
+
+/** A read that failed, named for what was being read. */
+export const unreadable =
+  (attempt: string) =>
+  (cause: SqlError.SqlError): FeedUnreadable =>
+    new FeedUnreadable({ reason: `${attempt} failed: ${cause.message}`, cause })
+
+/** A write that failed, named for what was being written. */
+export const writeFailed =
+  (attempt: string) =>
+  (cause: SqlError.SqlError): FeedWriteFailed =>
+    new FeedWriteFailed({
+      reason: `${attempt} failed: ${cause.message}`,
+      cause,
+    })
+
+/** A 64-bit column read back as text, as a `bigint`; see `statements.ts`. */
+export const bigintOf = (text: string | bigint | number): bigint => BigInt(text)