@kairos-es/core 1.0.0-alpha.5 → 1.0.0-alpha.7

package/src/SubscribeMachine.ts

@@ -5,6 +5,15 @@
  * the cursor's exclusive lower bound) and the poll loop that together turn a paged
  * read into a live subscription.
  *
+ * The algorithm is `subscribeItems`, generic over the item `A` it delivers and the
+ * cursor `C` it resumes from, because nothing in it is about an event: any source
+ * that can page-read strictly after a cursor and name the cursor of what it
+ * returned is subscribable by the same loop. `subscribeStream` is the event-store
+ * shape — `SequencedEvent` ordered by `Position` — over the same loop, and it is
+ * what every engine calls, unchanged by the generalisation. A second kind of
+ * source reuses the loop rather than growing a second one, for the reason the next
+ * paragraph gives.
+ *
  * It lives in core because delivery semantics are contract semantics. An engine
  * supplies TRANSPORT through two STUBBABLE seams — `fetch`, a paged read strictly
  * after a cursor, and `listen`, an optional wake source — and inherits the delivery
@@ -15,48 +24,52 @@
  * Both seams take plain in-memory values in a test, so the wake/poll/de-dup logic
  * is exercisable with no live backend at all.
  *
- * All THREE are the callers, and the count matters to a reader here: it means there
- * is exactly ONE delivery algorithm in this repo, and it is the code below. The
+ * All THREE engines are callers, and the count matters to a reader here: it means
+ * there is exactly ONE delivery algorithm in this repo, and it is the code below. The
  * in-memory store is a caller like the rest — it hands in a `fetch` over its log
  * `Ref` and its append `PubSub` as a wake source and keeps no delivery logic of its
  * own — which is what makes the shared contract suite's subscribe cases a comparison
  * of two BACKENDS rather than of two delivery algorithms.
  *
- * Correctness rests on the periodic position-based reconcile — a re-read from the
+ * Correctness rests on the periodic cursor-based reconcile — a re-read from the
  * cursor every poll interval — and NEVER on the wake source, which only lowers
  * delivery latency below that interval. A subscription with no wake source at all
  * is therefore complete and correct, merely slower, which is why `listen` is
  * optional in the TYPE rather than by the convention of passing `Stream.never`:
  * poll-only delivery is a supported shape, not a degraded one.
  *
- * ## The precondition every engine inherits: ASCENDING COMMIT ORDER
+ * ## The precondition every caller inherits: ASCENDING COMMIT ORDER
  *
- * A position that is committed and VISIBLE must never be below one already
- * delivered. Gaps are fine: the cursor is an ascending exclusive lower bound, so a
- * number that is allocated and never committed, or never allocated at all, is
- * simply a number the machine never sees. Out-of-order commits are not fine. If one
+ * It binds EVERY caller, over whatever cursor it chose — an engine's `Position`, or
+ * any other `C` handed to `subscribeItems` — and the example below is written in
+ * positions only because the engines are the callers it was first written for. A
+ * cursor that is committed and VISIBLE must never be below one already delivered.
+ * Gaps are fine: the cursor is an ascending exclusive lower bound, so a number
+ * that is allocated and never committed, or never allocated at all, is simply a
+ * number the machine never sees. Out-of-order commits are not fine. If one
  * writer allocates position 5 and another allocates 6, and 6 commits first, a poll
  * landing between the two commits delivers 6 and advances the cursor past it — and
  * 5 is then lost forever, because no later read looks below the cursor and nothing
- * anywhere notices the hole. An engine whose concurrent writers can commit out of
+ * anywhere notices the hole. A caller whose concurrent writers can commit out of
  * order MUST serialise them, or this machine will silently skip. An engine that
  * takes an exclusive lock on both of its append paths discharges the obligation
  * that way; an engine whose append is a single statement discharges it
  * structurally.
  *
  * The obligation is written down HERE, in the algorithm that depends on it, rather
- * than in whichever engine happens to satisfy it. It is a property of this delivery
- * loop and of nothing else, so an engine author reading only their own source has
+ * than in whichever caller happens to satisfy it. It is a property of this delivery
+ * loop and of nothing else, so an author reading only their own source has
  * no way to discover that anything downstream needs it — and the failure mode is
  * the worst kind there is: no error, no defect, no failing test, just an event that
  * a subscriber never receives. A single engine owning both its append path and its
  * own wake loop can leave the assumption implicit; a shared machine cannot.
  *
- * ## `catchUpPageSize` bounds the READ, not the emission
+ * ## `catchUpPageSize` bounds the READ, not the emission — unless `emit: 'page'`
  *
- * `drainFrom` below PAGES the catch-up read — `catchUpPageSize` bounds each
- * `fetch` — but it COLLECTS every page into one array, which is emitted as a SINGLE
- * chunk. So peak memory at subscription start is O(backlog) rather than O(page): a
+ * Under the default `emit: 'backlog'`, which is what `subscribeStream` and so every
+ * engine uses, `drainFrom` below PAGES the catch-up read — `catchUpPageSize` bounds
+ * each `fetch` — but it COLLECTS every page into one array, which is emitted as a
+ * SINGLE chunk. So peak memory at subscription start is O(backlog) rather than O(page): a
  * subscriber joining behind a large log materialises the whole backlog before its
  * consumer sees the first element.
  *
@@ -67,6 +80,19 @@
  * stay small even against a one-chunk catch-up. The cost therefore lands on backlog
  * memory alone, and it lands hardest on the durable-log-with-in-memory-view mode,
  * where an ephemeral view re-catches-up the entire log on every boot.
+ *
+ * `emit: 'page'` changes exactly that and nothing else. Each fetched page is
+ * emitted as its own chunk and the cursor advances past it before the next read, so
+ * peak memory is O(page) and the consumer sees the first item after ONE read rather
+ * than after the whole backlog. Delivery is unchanged: the same items in the same
+ * order, the same de-dup, the same wake and poll. What a caller gives up is the
+ * one-chunk catch-up, so it must not read meaning into chunk boundaries under
+ * either mode. It is for a consumer that does NOT re-chunk downstream and whose
+ * catch-up can be the whole of a large source — a full resync of a table, say —
+ * where holding the backlog would be the cost the paging was meant to avoid. The
+ * engines keep `'backlog'` because the read side's re-chunking already bounds what
+ * they commit, and changing their emission would move a behaviour nobody asked to
+ * move.
  */
 import {
   Effect,
@@ -107,6 +133,15 @@ export const DEFAULT_CATCH_UP_PAGE_SIZE = 512
  */
 export const DEFAULT_POLL_INTERVAL_MILLIS = 1000
 
+/** A usable `pollInterval`: see `subscriptionTuningFields` for the bounds' argument. */
+const PollIntervalMillis = Schema.Number.check(
+  Schema.isGreaterThan(0),
+  Schema.isFinite(),
+)
+
+/** A usable `catchUpPageSize`: see `subscriptionTuningFields` for the bounds' argument. */
+const CatchUpPageSize = Schema.Int.check(Schema.isGreaterThan(0))
+
 /**
  * The two subscribe-tuning config FIELDS, for an engine whose config is a `Schema`
  * to spread into its own struct.
@@ -137,14 +172,36 @@ export const DEFAULT_POLL_INTERVAL_MILLIS = 1000
  * absent key, never an explicit `undefined`.
  */
 export const subscriptionTuningFields = {
-  pollInterval: Schema.optionalKey(
-    Schema.Number.check(Schema.isGreaterThan(0), Schema.isFinite()),
-  ),
-  catchUpPageSize: Schema.optionalKey(
-    Schema.Int.check(Schema.isGreaterThan(0)),
-  ),
+  pollInterval: Schema.optionalKey(PollIntervalMillis),
+  catchUpPageSize: Schema.optionalKey(CatchUpPageSize),
 } as const
 
+/**
+ * Why a caller-supplied tuning knob is unusable, naming the field and the value, or
+ * `undefined` when both are usable or absent. It judges with the SAME two schemas
+ * the config fields are built from, so `subscribeItems`, whose caller may have no
+ * config to decode, accepts exactly what an engine's config accepts and the bounds
+ * have one home.
+ */
+const tuningFault = (deps: {
+  readonly pollInterval?: number
+  readonly catchUpPageSize?: number
+}): string | undefined => {
+  if (
+    deps.pollInterval !== undefined &&
+    !Schema.is(PollIntervalMillis)(deps.pollInterval)
+  ) {
+    return `subscribeItems: pollInterval must be a finite number of milliseconds greater than 0, got ${deps.pollInterval}`
+  }
+  if (
+    deps.catchUpPageSize !== undefined &&
+    !Schema.is(CatchUpPageSize)(deps.catchUpPageSize)
+  ) {
+    return `subscribeItems: catchUpPageSize must be an integer greater than 0, got ${deps.catchUpPageSize}`
+  }
+  return undefined
+}
+
 /**
  * Retry a TRANSIENT poll fault (an `E` from one reconcile read) on a capped
  * exponential backoff rather than letting `Stream.orDie` kill a subscription that
@@ -161,9 +218,79 @@ const POLL_RETRY_SCHEDULE = Schedule.max([
 ])
 
 /**
- * The two seams the subscribe machine runs over, plus the two tuning knobs. `E` is
- * the engine's own infrastructure error — whatever its reads fail with — and it
- * stays on the channel through to `makeContractStore`, which dies it.
+ * How a subscription hands its catch-up to the consumer: the whole backlog as ONE
+ * chunk (`'backlog'`), or each fetched page as a chunk of its own (`'page'`). The
+ * module doc's "`catchUpPageSize` bounds the READ" section owns what each costs.
+ *
+ * A closed union of two words rather than a boolean, so a call site says WHICH
+ * behaviour it chose instead of what it turned off, and a third shape, should one
+ * ever be argued for, is added beside these rather than encoded in a second flag.
+ */
+export type SubscriptionEmission = 'backlog' | 'page'
+
+/**
+ * The machine's inputs over ANY item `A` with an ascending cursor `C`: the two seams,
+ * where the cursor comes from and starts, and the tuning.
+ *
+ * Generic in the item and the cursor because nothing in the algorithm is about an
+ * EVENT. The wake queue, the poll, the paged reconcile and the exclusive-lower-bound
+ * de-dup need only three things of what they deliver: a page-read strictly after a
+ * cursor, a way to read the cursor OFF a delivered item, and a starting cursor. A
+ * log of `SequencedEvent`s ordered by `Position` is one such source; a feed of rows
+ * ordered by a feed-local sequence is another, and a second copy of this loop for it
+ * would be exactly the divergence the module doc argues against.
+ *
+ * The machine never COMPARES two cursors. `fetch` is the only thing that interprets
+ * one — its exclusive lower bound is what does the de-dup — so `C` needs no ordering
+ * instance here, and the ascending-commit-order obligation in the module doc is the
+ * caller's, stated over whatever `C` it chose.
+ *
+ * `initial` is REQUIRED, unlike `subscribeStream`'s `after`. Only a cursor type the
+ * machine knows can have a default — `ORIGIN` for `Position` — and for any other the
+ * caller is the only party that knows what "before everything" is.
+ *
+ * `pollInterval` and `catchUpPageSize` are optional here and fall back to
+ * `DEFAULT_POLL_INTERVAL_MILLIS` and `DEFAULT_CATCH_UP_PAGE_SIZE`: a caller with no
+ * config of its own takes the same figures every engine does, from the same place.
+ * The event-store shape, `SubscriptionDeps`, keeps them required, because every
+ * engine already resolves them from its config and that type is unchanged.
+ */
+export interface ItemSubscriptionDeps<A, C, E> {
+  /**
+   * Page-read the items strictly after `after`, up to `limit`, ascending by cursor.
+   * Returning FEWER than `limit` signals the tail is drained. The exclusive lower
+   * bound IS the de-dup: a re-read from the cursor never returns what was delivered.
+   */
+  readonly fetch: (
+    after: C,
+    limit: ReadLimit,
+  ) => Effect.Effect<ReadonlyArray<A>, E>
+  /** The cursor of a delivered item: the next `fetch` starts strictly after it. */
+  readonly cursorOf: (item: A) => C
+  /** The cursor to start from, exclusive: the first `fetch` reads strictly after it. */
+  readonly initial: C
+  /** The raw wake source — see `SubscriptionDeps.listen`, whose rules it shares. */
+  readonly listen?: Stream.Stream<unknown, E>
+  /**
+   * The poll interval in milliseconds (default `DEFAULT_POLL_INTERVAL_MILLIS`):
+   * finite and greater than 0, or `subscribeItems` dies.
+   */
+  readonly pollInterval?: number
+  /**
+   * The catch-up page size (default `DEFAULT_CATCH_UP_PAGE_SIZE`): an integer
+   * greater than 0, or `subscribeItems` dies.
+   */
+  readonly catchUpPageSize?: number
+  /** The catch-up's chunking (default `'backlog'`); see `SubscriptionEmission`. */
+  readonly emit?: SubscriptionEmission
+}
+
+/**
+ * The two seams the event-store subscribe machine runs over, plus the two tuning
+ * knobs: `ItemSubscriptionDeps` fixed to `SequencedEvent` and `Position`, the shape
+ * every engine's `subscribe` already hands to `subscribeStream`. `E` is the engine's
+ * own infrastructure error — whatever its reads fail with — and it stays on the
+ * channel through to `makeContractStore`, which dies it.
  *
  * ONE type parameter and not two, deliberately: a wake source with an error type of
  * its own is `Stream.orDie`d by the caller, yielding a `Stream<unknown, never>`
@@ -199,19 +326,47 @@ export interface SubscriptionDeps<E> {
 }
 
 /**
- * Build the subscription `Stream`: catch up from `after`, then track the live tail.
- * The cursor threads through the `Stream.paginate` state, starting at `after` and
- * advancing to the last delivered position each batch; the stream never completes
- * on its own (the next state is always `Option.some`), so the enclosing scope
- * interrupts it on close. An `E` mid-stream stays on the channel; `makeContractStore` dies it
- * (subscribe is `E = never` on the contract).
+ * Build a subscription `Stream` over any ascending-cursor source: catch up from
+ * `initial`, then track the live tail. The cursor threads through the
+ * `Stream.paginate` state, starting at `initial` and advancing to the cursor of the
+ * last delivered item each batch; the stream never completes on its own (the next
+ * state is always `Option.some`), so the enclosing scope interrupts it on close. An
+ * `E` mid-stream stays on the channel, for the caller to die or to supervise.
+ *
+ * This is THE delivery algorithm, and `subscribeStream` runs the same machine. Under
+ * `emit: 'backlog'` a batch is everything a drain to the tail returned; under
+ * `'page'` it is one fetched page, so the paginate step and the page coincide and a
+ * full page is followed at once by the next read rather than by a wait.
+ *
+ * A `pollInterval` or `catchUpPageSize` the config fields would refuse is a DEFECT
+ * here, raised by the stream before it reads or forks anything, with a message
+ * naming the field. This caller may have no config whose decode would have caught
+ * it, and unchecked the two failures are silent or late: a page size of 0 gives a
+ * stream that never emits and never fails, and 1.5 dies only at the first read,
+ * inside `ReadLimit`, in words that name neither knob.
  */
-export const subscribeStream = <E>(
-  deps: SubscriptionDeps<E>,
-  after: Position | undefined,
-): Stream.Stream<SequencedEvent, E, Scope.Scope> =>
+export const subscribeItems = <A, C, E>(
+  deps: ItemSubscriptionDeps<A, C, E>,
+): Stream.Stream<A, E, Scope.Scope> => {
+  const fault = tuningFault(deps)
+  return fault === undefined ? machine(deps) : Stream.die(new Error(fault))
+}
+
+/**
+ * The machine itself, with no check of its tuning. `subscribeStream` runs it
+ * directly, and deliberately: the event path stays exactly as it was, with no
+ * second check of figures each SQL engine has already decoded from its config
+ * through the same bounds, and the in-memory store's are a test's own options.
+ */
+const machine = <A, C, E>(
+  deps: ItemSubscriptionDeps<A, C, E>,
+): Stream.Stream<A, E, Scope.Scope> =>
   Stream.unwrap(
     Effect.gen(function* () {
+      const pageSize = deps.catchUpPageSize ?? DEFAULT_CATCH_UP_PAGE_SIZE
+      const pollInterval = deps.pollInterval ?? DEFAULT_POLL_INTERVAL_MILLIS
+      const limit = ReadLimit.make(pageSize)
+
       // The wake primitive: a sliding(1) queue holding at most one pending
       // signal. The wake source `offer`s an opaque wake on every notification;
       // the poll loop `take`s (with a timeout). The lost-wakeup guard is
@@ -242,82 +397,102 @@ export const subscribeStream = <E>(
       // notification is an opaque wake signal.
       yield* listener.pipe(Stream.runDrain, Effect.forkScoped)
 
+      // One page strictly after `from`: the step `drainFrom` loops, and the whole
+      // of a `'page'` read.
+      const fetchPage = (from: C): Effect.Effect<ReadonlyArray<A>, E> =>
+        deps.fetch(from, limit)
+
       // Reconcile: page-read from `from` until a short page, advancing to the last
-      // delivered position. Retries a TRANSIENT poll fault on a capped schedule so
+      // delivered cursor. Retries a TRANSIENT poll fault on a capped schedule so
       // a single connection blip does not kill a days-long subscription; a
-      // persistent fault propagates and dies.
-      const drainFrom = (
-        from: Position,
-      ): Effect.Effect<
-        {
-          readonly events: ReadonlyArray<SequencedEvent>
-          readonly last: Position
-        },
-        E
-      > =>
+      // persistent fault propagates. The retry wraps the WHOLE drain, so a fault on
+      // page three re-reads pages one and two: nothing has been emitted yet, so
+      // that costs reads and never a duplicate.
+      const drainFrom = (from: C): Effect.Effect<ReadonlyArray<A>, E> =>
         Effect.gen(function* () {
-          const collected: Array<SequencedEvent> = []
+          const collected: Array<A> = []
           let cursor = from
           // Loop pages until a short page signals the tail is drained.
           while (true) {
-            const events = yield* deps.fetch(
-              cursor,
-              ReadLimit.make(deps.catchUpPageSize),
-            )
-            for (const event of events) {
-              collected.push(event)
+            const items = yield* fetchPage(cursor)
+            for (const item of items) {
+              collected.push(item)
             }
-            const lastEvent = events.at(-1)
-            if (
-              lastEvent === undefined ||
-              events.length < deps.catchUpPageSize
-            ) {
+            const lastItem = items.at(-1)
+            if (lastItem === undefined || items.length < pageSize) {
               break
             }
-            cursor = lastEvent.position
+            cursor = deps.cursorOf(lastItem)
           }
-          const last = collected.at(-1)?.position ?? from
-          return { events: collected, last }
+          return collected
         }).pipe(Effect.retry(POLL_RETRY_SCHEDULE))
 
+      // The read one paginate step performs, by emission mode. Under `'page'` the
+      // retry wraps the single page, which is the whole of what is unemitted.
+      const read =
+        (deps.emit ?? 'backlog') === 'page'
+          ? (from: C) => fetchPage(from).pipe(Effect.retry(POLL_RETRY_SCHEDULE))
+          : drainFrom
+
       /**
-       * Produce the NEXT non-empty batch from a cursor position, together with
-       * the advanced cursor — a pure `Position → Effect<[events, next]>`. The
-       * `> boundary` de-dup is enforced by `fetch`'s exclusive lower bound, so a
-       * live re-read after a wake never re-emits an already-delivered position.
+       * Produce the NEXT non-empty batch from a cursor, together with the advanced
+       * cursor — a pure `C → Effect<[items, next]>`. The `> boundary` de-dup is
+       * enforced by `fetch`'s exclusive lower bound, so a live re-read after a wake
+       * never re-emits an already-delivered item.
        *
-       * Drain the wake queue BEFORE reading (lost-wakeup guard), then drain from
-       * the cursor. If a drain yields nothing, wait for a wake but only up to the
-       * poll interval, then loop and re-read ANYWAY — the periodic position-based
-       * poll that makes delivery correct independently of any wake. This effect
-       * only completes by returning a non-empty batch; the scope interrupts it on
-       * close.
+       * Drain the wake queue BEFORE reading (lost-wakeup guard), then read from the
+       * cursor. If a read yields nothing, wait for a wake but only up to the poll
+       * interval, then loop and re-read ANYWAY — the periodic cursor-based poll that
+       * makes delivery correct independently of any wake. This effect only
+       * completes by returning a non-empty batch; the scope interrupts it on close.
        */
       const nextBatch = (
-        from: Position,
-      ): Effect.Effect<readonly [ReadonlyArray<SequencedEvent>, Position], E> =>
+        from: C,
+      ): Effect.Effect<readonly [ReadonlyArray<A>, C], E> =>
         Effect.gen(function* () {
           while (true) {
             // `Queue.clear` and not `Queue.takeAll`: the drain must NOT wait, and
             // `takeAll` suspends until at least one signal is present.
             yield* Queue.clear(wake)
-            const { events, last } = yield* drainFrom(from)
-            if (events.length > 0) {
-              return [events, last] as const
+            const items = yield* read(from)
+            const last = items.at(-1)
+            if (last !== undefined) {
+              return [items, deps.cursorOf(last)] as const
             }
             // Nothing new: block on a wake, but only up to the poll interval — on
             // timeout we fall through and the loop re-reads regardless.
-            yield* Effect.ignore(
-              Effect.timeout(Queue.take(wake), deps.pollInterval),
-            )
+            yield* Effect.ignore(Effect.timeout(Queue.take(wake), pollInterval))
           }
         })
 
-      return Stream.paginate(after ?? ORIGIN, (from) =>
+      return Stream.paginate(deps.initial, (from) =>
         Effect.map(
           nextBatch(from),
-          ([events, last]) => [events, Option.some(last)] as const,
+          ([items, next]) => [items, Option.some(next)] as const,
         ),
       )
     }),
   )
+
+/**
+ * Build the event subscription `Stream`: catch up from `after`, then track the live
+ * tail. A THIN wrapper over the machine `subscribeItems` runs, fixing the item to
+ * `SequencedEvent`, the cursor to its `position`, the start to `ORIGIN` when `after`
+ * is absent, and the emission to the default `'backlog'` — so every engine's call
+ * site, and every figure it delivers, is what it was before the machine became
+ * generic. It skips `subscribeItems`' tuning check, for the reason on `machine`. An `E`
+ * mid-stream stays on the channel; `makeContractStore` dies it (subscribe is
+ * `E = never` on the contract).
+ */
+export const subscribeStream = <E>(
+  deps: SubscriptionDeps<E>,
+  after: Position | undefined,
+): Stream.Stream<SequencedEvent, E, Scope.Scope> =>
+  machine<SequencedEvent, Position, E>({
+    fetch: deps.fetch,
+    cursorOf: (event) => event.position,
+    initial: after ?? ORIGIN,
+    ...(deps.listen === undefined ? {} : { listen: deps.listen }),
+    pollInterval: deps.pollInterval,
+    catchUpPageSize: deps.catchUpPageSize,
+  })