@kairos-es/feed 0.0.0-stage → 0.0.0

package/src/inMemorySink.ts

@@ -0,0 +1,215 @@
+/**
+ * The in-memory sink: a `FeedSink` over a `Ref` of documents, with an optional
+ * `FeedResync` and `FeedReconcile`, and the ORACLE the sink law suite holds every
+ * other sink against.
+ *
+ * It keeps the laws by construction: an upsert replaces the whole document, a
+ * delete of an absent key is a no-op, so the same batch applied twice leaves it as
+ * once; a batch is visible the moment `apply` returns, so `confirm` has nothing to
+ * wait for; a resync target is a separate map that `publish` swaps in whole, so a
+ * published resync holds exactly what was loaded into it, and a load into a target
+ * a later `begin` dropped fails rather than recreating it. Each `begin` names its
+ * target by a counter the sink's own state holds, so two workers sharing one sink
+ * value — the way two processes share one real index — never share a target, and
+ * the name is the same on every run of a test.
+ *
+ * Its two optional parts are switched on by option rather than always present,
+ * because which parts a sink has CHANGES what the driver does — a sink with no
+ * `resync` is resynced in place — and a test of either path needs a sink that
+ * lacks the other. It also records every batch it is handed, live or into a
+ * target, so a case can say how many sends a change cost.
+ *
+ * It lives for as long as the value is held, independently of any feed, which is
+ * what lets a test keep one sink across two in-memory feeds and watch the second
+ * feed's fresh generation force a resync.
+ */
+import { Effect, Ref, Stream } from 'effect'
+import type { NonEmptyReadonlyArray } from 'effect/Array'
+import { SinkRetryable } from './errors'
+import type { FeedChange, FeedResync, FeedSink, ResyncTarget } from './FeedSink'
+import type { FeedKey, Generation, SinkName } from './identifiers'
+
+/** What the in-memory sink's `apply` and `load` return: where, and which batch. */
+export interface InMemoryReceipt {
+  /** `'live'`, or the name of the resync target the batch was loaded into. */
+  readonly target: string
+  readonly batch: number
+}
+
+/** One batch the sink was handed, and where it went. */
+export interface InMemorySentBatch<D> {
+  readonly target: string
+  readonly changes: NonEmptyReadonlyArray<FeedChange<D>>
+}
+
+/** Options for {@link makeInMemorySink}. */
+export interface InMemorySinkOptions {
+  readonly name: SinkName
+  /** Give the sink a `FeedResync` (an atomic swap). */
+  readonly resync?: boolean
+  /** Give the sink a `FeedReconcile` (a key listing). */
+  readonly reconcile?: boolean
+}
+
+/** The in-memory sink and what a test reads back from it. */
+export interface InMemorySink<D> {
+  readonly sink: FeedSink<D, InMemoryReceipt>
+  /** The live documents, by key. */
+  readonly documents: Effect.Effect<ReadonlyMap<FeedKey, D>>
+  /** Every batch handed to `apply` or `load`, in order. */
+  readonly batches: Effect.Effect<ReadonlyArray<InMemorySentBatch<D>>>
+  /** The generation of every resync published, in order. */
+  readonly published: Effect.Effect<ReadonlyArray<Generation>>
+}
+
+interface SinkState<D> {
+  readonly live: ReadonlyMap<FeedKey, D>
+  readonly targets: ReadonlyMap<string, ReadonlyMap<FeedKey, D>>
+  readonly batches: ReadonlyArray<InMemorySentBatch<D>>
+  readonly published: ReadonlyArray<Generation>
+  /** How many `begin`s this sink has answered: the next target's attempt number. */
+  readonly attempts: number
+}
+
+const LIVE = 'live'
+
+const applyTo = <D>(
+  documents: ReadonlyMap<FeedKey, D>,
+  changes: NonEmptyReadonlyArray<FeedChange<D>>,
+): ReadonlyMap<FeedKey, D> => {
+  const next = new Map(documents)
+  for (const change of changes) {
+    if (change._tag === 'Upsert') {
+      next.set(change.key, change.doc)
+    } else {
+      next.delete(change.key)
+    }
+  }
+  return next
+}
+
+/** What a load into, or a publish of, a target a later `begin` dropped fails with. */
+const droppedTarget = (name: string): SinkRetryable =>
+  new SinkRetryable({
+    reason: `the resync target '${name}' was dropped by a later begin`,
+  })
+
+/** The resync target's name for one attempt at `generation`; see the module doc. */
+const targetName = (generation: Generation, attempt: number): string =>
+  `${LIVE}__g${generation}_${attempt}`
+
+/** Build an in-memory sink; see the module doc. */
+export const makeInMemorySink = <D>(
+  options: InMemorySinkOptions,
+): Effect.Effect<InMemorySink<D>> =>
+  Effect.gen(function* () {
+    const state = yield* Ref.make<SinkState<D>>({
+      live: new Map(),
+      targets: new Map(),
+      batches: [],
+      published: [],
+      attempts: 0,
+    })
+
+    const record = (
+      target: string,
+      changes: NonEmptyReadonlyArray<FeedChange<D>>,
+    ): Effect.Effect<InMemoryReceipt, SinkRetryable> =>
+      Effect.flatten(
+        Ref.modify(
+          state,
+          (
+            s,
+          ): [Effect.Effect<InMemoryReceipt, SinkRetryable>, SinkState<D>] => {
+            const batches = [...s.batches, { target, changes }]
+            const receipt = { target, batch: batches.length }
+            if (target === LIVE) {
+              return [
+                Effect.succeed(receipt),
+                { ...s, batches, live: applyTo(s.live, changes) },
+              ]
+            }
+            const built = s.targets.get(target)
+            // Dropped by a later begin: recreating it here is what would let a
+            // stale worker's publish swap a fragment in. `FeedResync.load` says so.
+            if (built === undefined) {
+              return [Effect.fail(droppedTarget(target)), s]
+            }
+            return [
+              Effect.succeed(receipt),
+              {
+                ...s,
+                batches,
+                targets: new Map(s.targets).set(
+                  target,
+                  applyTo(built, changes),
+                ),
+              },
+            ]
+          },
+        ),
+      )
+
+    const resync: FeedResync<D, InMemoryReceipt> = {
+      // Every other target is dropped: whatever attempt left it, it will never be
+      // published now, and its own publish must fail rather than swap it in.
+      begin: (generation) =>
+        Ref.modify(state, (s): [ResyncTarget, SinkState<D>] => {
+          const attempts = s.attempts + 1
+          const name = targetName(generation, attempts)
+          return [
+            { generation, name },
+            { ...s, attempts, targets: new Map([[name, new Map()]]) },
+          ]
+        }),
+      load: (target, batch) => record(target.name, batch),
+      publish: (target) =>
+        Effect.flatten(
+          Ref.modify(
+            state,
+            (s): [Effect.Effect<void, SinkRetryable>, SinkState<D>] => {
+              const built = s.targets.get(target.name)
+              if (built === undefined) {
+                return [Effect.fail(droppedTarget(target.name)), s]
+              }
+              const targets = new Map(s.targets)
+              targets.delete(target.name)
+              return [
+                Effect.void,
+                {
+                  ...s,
+                  live: built,
+                  targets,
+                  published: [...s.published, target.generation],
+                },
+              ]
+            },
+          ),
+        ),
+    }
+
+    const sink: FeedSink<D, InMemoryReceipt> = {
+      name: options.name,
+      apply: (batch) => record(LIVE, batch),
+      confirm: () => Effect.void,
+      ...(options.resync === true ? { resync } : {}),
+      ...(options.reconcile === true
+        ? {
+            reconcile: {
+              listKeys: Stream.unwrap(
+                Effect.map(Ref.get(state), (s) =>
+                  Stream.fromIterable(Array.from(s.live.keys())),
+                ),
+              ),
+            },
+          }
+        : {}),
+    }
+
+    return {
+      sink,
+      documents: Effect.map(Ref.get(state), (s) => s.live),
+      batches: Effect.map(Ref.get(state), (s) => s.batches),
+      published: Effect.map(Ref.get(state), (s) => s.published),
+    }
+  })

package/src/index.ts

@@ -0,0 +1,136 @@
+/**
+ * @kairos-es/feed — a state-only change feed: the newest state of each record a
+ * read model touches, written inside that read model's own commit, and copied by
+ * a sync worker to a sink that has no transaction of its own — a search index, a
+ * cache, a mirror (ADR-0019).
+ *
+ * The feed copies STATE. It never tells anyone each thing that happened; that is
+ * `@kairos-es/automation`'s job, and the two are deliberately different shapes: an
+ * automation's only progress record is the log, while a feed keeps a cursor, a
+ * lease and a parked list beside its rows, because what it mirrors is state and
+ * its progress is legitimately a cursor.
+ *
+ * What is here, and where each part's argument lives:
+ *
+ * - The identifiers (`identifiers.ts`) and the error vocabulary (`errors.ts`).
+ * - The feed's write and read sides, `BatchedFeedWriter` / `FeedWriter` and
+ *   `FeedSource` (`Feed.ts`): why the feed is written inside the commit, and what
+ *   "coalesced, latest wins" means for a stored row.
+ * - The sink port, `FeedSink` with its optional `FeedResync` and `FeedReconcile`
+ *   (`FeedSink.ts`): what a sink must keep, and why a sink with neither optional
+ *   part is refused.
+ * - The sync store port, `FeedSyncStore` (`FeedSyncStore.ts`): the cursor, the
+ *   parked list and the worker lease, and the exact contract a backend implements.
+ * - The driver, `runFeedSync` (`runFeedSync.ts`): the lifecycle, the rules it
+ *   keeps, resync, and its own supervision.
+ * - `FeedReadiness` and `materialisationCaughtUp` (`readiness.ts`): why a
+ *   rebuild's publish can wait for the materialisation, and what it waits on.
+ * - `reconcileFeed` (`reconcile.ts`), and `inspectFeed` / `retryParked` /
+ *   `removeSink` (`inspect.ts`).
+ * - The in-memory feed (`inMemoryFeed.ts`) and sink (`inMemorySink.ts`): the parity
+ *   oracles, and what the in-memory feed refuses.
+ *
+ * Wiring, end to end, in memory:
+ *
+ * ```ts
+ * const feed = yield* makeInMemoryFeed({ feed: FeedName.make('course-search'), doc: CourseDoc })
+ * const store = yield* withInMemoryFeed(yield* makeInMemoryProjectionStore, feed, key)
+ *
+ * // In the read model's `apply`, as the LAST step of the batch:
+ * const apply = (batch) =>
+ *   feed.write(Arr.lastNonEmpty(batch).sequenced.position, (writer) =>
+ *     Effect.forEach(touchedCourses(batch), (course) =>
+ *       writer.upsert(FeedKey.make(course.id), toDoc(course)),
+ *     ),
+ *   )
+ *
+ * // Beside the projections, in a fibre of its own:
+ * const { sink } = yield* makeInMemorySink<CourseDoc>({ name: SinkName.make('search'), resync: true })
+ * yield* Effect.forkScoped(runFeedSync({ source: feed.source, sink, store: feed.syncStore }))
+ * ```
+ */
+
+export {
+  CursorSuperseded,
+  FeedSyncStalled,
+  FeedUnreadable,
+  FeedWriteFailed,
+  LeaseHeld,
+  LeaseLost,
+  SinkHalted,
+  SinkRejected,
+  SinkRetryable,
+} from './errors'
+export type {
+  BatchedFeedWriter,
+  FeedRow,
+  FeedSource,
+  FeedWriter,
+} from './Feed'
+export {
+  changeOf,
+  FeedChange,
+  type FeedChangeDefinition,
+  type FeedReconcile,
+  type FeedResync,
+  type FeedSink,
+  type ResyncTarget,
+} from './FeedSink'
+export {
+  type FeedCursor,
+  type FeedLease,
+  type FeedSyncDescription,
+  type FeedSyncStore,
+  type LeaseDescription,
+  type ParkedChange,
+  type ParkedRecord,
+  type SinkSyncState,
+  sameCursor,
+} from './FeedSyncStore'
+export {
+  FEED_ORIGIN,
+  FeedKey,
+  FeedName,
+  FeedSeq,
+  type FeedSinkId,
+  Generation,
+  LeaseHolder,
+  SinkName,
+} from './identifiers'
+export {
+  type InMemoryFeed,
+  type InMemoryFeedOptions,
+  makeInMemoryFeed,
+  withInMemoryFeed,
+} from './inMemoryFeed'
+export {
+  type InMemoryReceipt,
+  type InMemorySentBatch,
+  type InMemorySink,
+  type InMemorySinkOptions,
+  makeInMemorySink,
+} from './inMemorySink'
+export {
+  type FeedInspection,
+  inspectFeed,
+  removeSink,
+  retryParked,
+  type SinkInspection,
+} from './inspect'
+export {
+  type CaughtUpReadModel,
+  type FeedReadiness,
+  type FeedReadinessProbe,
+  materialisationCaughtUp,
+} from './readiness'
+export {
+  type ReconcileFeedOptions,
+  type ReconcileReport,
+  reconcileFeed,
+} from './reconcile'
+export {
+  type FeedSyncRestart,
+  type FeedSyncTuning,
+  type RunFeedSyncOptions,
+  runFeedSync,
+} from './runFeedSync'

package/src/inspect.ts

@@ -0,0 +1,114 @@
+/**
+ * `inspectFeed`, `retryParked` and `removeSink`: an operator's view of a feed's
+ * sinks, and the two repairs an operator makes by hand.
+ *
+ * None of them needs the sink's client, because everything they read or change
+ * lives in the source database beside the feed: an operator can see a sink's lag,
+ * its lease, its parked records and when it last advanced, send a parked record
+ * again, and retire a sink, without the credentials of the index concerned. The
+ * same choice `@kairos-es/automation` made for its own `inspect`. There is no CLI
+ * or UI over any of them; they are effects an application wires into whatever it
+ * already has.
+ *
+ * `lastAdvance` and `removeSink` are a pair. Every sink registered on a feed holds
+ * its tombstone collection back, so a sink the application stopped running and
+ * never removed lets tombstones pile up for good; `lastAdvance` is how such a sink
+ * shows — still, while the head moves — and `removeSink` is what lets collection
+ * proceed without it. `FeedSyncStore.ts` owns both rules.
+ */
+import { type DateTime, Effect, Option } from 'effect'
+import type { FeedUnreadable, FeedWriteFailed, LeaseHeld } from './errors'
+import type { FeedSource } from './Feed'
+import type {
+  FeedCursor,
+  FeedSyncStore,
+  LeaseDescription,
+  ParkedRecord,
+} from './FeedSyncStore'
+import type {
+  FeedKey,
+  FeedName,
+  FeedSeq,
+  Generation,
+  SinkName,
+} from './identifiers'
+
+/** One sink of a feed, as `inspectFeed` reports it. */
+export interface SinkInspection {
+  readonly sink: SinkName
+  /** The generation the sink's cursor was saved at; `None` before its first sync. */
+  readonly generation: Option.Option<Generation>
+  readonly cursor: Option.Option<FeedCursor>
+  /** When the cursor last moved; `None` if it never has. See `SinkSyncState.lastAdvance`. */
+  readonly lastAdvance: Option.Option<DateTime.Utc>
+  /** True when the next pass will resync: no cursor, or one at another generation. */
+  readonly resyncDue: boolean
+  /** Rows the sink has still to receive; see `SinkSyncState.pending`. */
+  readonly lag: number
+  readonly lease: Option.Option<LeaseDescription>
+  readonly parked: ReadonlyArray<ParkedRecord>
+}
+
+/** A feed and its sinks. */
+export interface FeedInspection {
+  readonly feed: FeedName
+  readonly generation: Generation
+  /** The highest `seq` the feed holds. */
+  readonly head: FeedSeq
+  readonly sinks: ReadonlyArray<SinkInspection>
+}
+
+/** Every sink of `source`'s feed: its cursor, lag, lease and parked records. */
+export const inspectFeed = <D>(
+  source: FeedSource<D>,
+  store: FeedSyncStore,
+): Effect.Effect<FeedInspection, FeedUnreadable> =>
+  Effect.gen(function* () {
+    const generation = yield* source.generation
+    const description = yield* store.describe(source.feed)
+    return {
+      feed: source.feed,
+      generation,
+      head: description.head,
+      sinks: description.sinks.map((state) => ({
+        sink: state.sink,
+        generation: Option.map(state.cursor, (c) => c.generation),
+        cursor: state.cursor,
+        lastAdvance: state.lastAdvance,
+        resyncDue: Option.match(state.cursor, {
+          onNone: () => true,
+          onSome: (c) => c.generation !== generation,
+        }),
+        lag: state.pending,
+        lease: state.lease,
+        parked: state.parked,
+      })),
+    }
+  })
+
+/**
+ * Send one parked record again: delete its park and re-stamp its row's `seq`, so
+ * the next pass of every sink on the feed sends it. Succeeds with whether the key
+ * was parked at all. Every sink on the feed receives the record again, not only
+ * `sink`; an idempotent sink absorbs that.
+ */
+export const retryParked = (
+  store: FeedSyncStore,
+  args: {
+    readonly feed: FeedName
+    readonly sink: SinkName
+    readonly key: FeedKey
+  },
+): Effect.Effect<boolean, FeedWriteFailed> => store.retryParked(args)
+
+/**
+ * Retire `sink` from `feed`: delete its cursor, its lease record and its parked
+ * records, so it no longer holds the feed's tombstone collection back. Succeeds
+ * with whether the store held anything for it. Fails `LeaseHeld`, removing
+ * nothing, while a worker for the sink is running: stop the worker first. A worker
+ * started for the sink later registers it again from scratch, and resyncs.
+ */
+export const removeSink = (
+  store: FeedSyncStore,
+  args: { readonly feed: FeedName; readonly sink: SinkName },
+): Effect.Effect<boolean, LeaseHeld | FeedWriteFailed> => store.removeSink(args)

package/src/readiness.ts

@@ -0,0 +1,137 @@
+/**
+ * `FeedReadiness` — when a rebuilt copy may go live — and
+ * `materialisationCaughtUp`, the readiness of the common case.
+ *
+ * ## Why a rebuild has anything to wait for
+ *
+ * A reset empties the feed and moves its generation on, and the materialisation
+ * then replays the log, writing the feed afresh as it goes. The sync worker sees
+ * the new generation within a poll interval and starts its resync at once, but the
+ * feed it pages through is only as complete as the replay is far along. Reaching
+ * the feed's last row says nothing about the replay: it is wherever the replay had
+ * got to. So a resync that publishes at the feed's end swaps a complete live copy
+ * for whatever fraction of the rebuild the replay had written, and search serves
+ * that partial copy until the tail fills the rest in.
+ *
+ * Only the materialisation knows how far its replay has got, and the feed's ports
+ * deliberately name no materialisation, so the driver is TOLD, through an optional
+ * readiness signal on `runFeedSync`, and holds the step that makes the rebuild
+ * final until the signal says yes. With no signal the resync publishes at the
+ * feed's end, as it always did, and a rebuild can serve a partial copy for as long
+ * as the replay takes.
+ *
+ * ## Why it is a function returning an effect
+ *
+ * "Caught up" has to be relative to a fixed point: the log as it stood when the
+ * rebuild began. Judged against the log as it stands at each ask, a read model
+ * under steady writes is always a batch or two behind, and the publish would wait
+ * for a quiet moment that may never come. So the signal is asked ONCE per resync
+ * attempt, as the attempt starts, for the generation that attempt rebuilds, and
+ * returns the PROBE the worker then polls — by which time whatever point the probe
+ * judges by is fixed. An attempt starts after the reset that moved the generation,
+ * so a point taken then is at or after the log as it stood at the reset: the error
+ * is towards waiting slightly longer, never towards publishing early.
+ *
+ * Both halves fail `FeedUnreadable`, the feed's own "a read could not be made":
+ * the worker cannot tell a probe that could not read the materialisation's
+ * progress from a feed it could not read, and needs to do the same thing about
+ * either — end the session and try again after a back-off.
+ */
+import {
+  type AnyProjection,
+  composeProjections,
+  DcbEventStore,
+  DEFAULT_CATCH_UP_PAGE_SIZE,
+  type Position,
+  ReadLimit,
+} from '@kairos-es/core'
+import type { KeyedProjectionStore } from '@kairos-es/read'
+import { Array as Arr, Effect, Option, Stream } from 'effect'
+import { FeedUnreadable } from './errors'
+import type { Generation } from './identifiers'
+
+/**
+ * Polled by the worker, between pages, until it answers `true`: the rebuild this
+ * probe was made for may now go live. See the module doc.
+ */
+export type FeedReadinessProbe = Effect.Effect<boolean, FeedUnreadable>
+
+/**
+ * Asked once per resync attempt, as it starts, for the generation it rebuilds,
+ * and answering with the {@link FeedReadinessProbe} that attempt polls. See the
+ * module doc for why it is two steps.
+ */
+export type FeedReadiness = (
+  generation: Generation,
+) => Effect.Effect<FeedReadinessProbe, FeedUnreadable>
+
+/**
+ * What {@link materialisationCaughtUp} needs of a read model: its bound view store
+ * and its slices. Structural, so any `ReadModel` from `@kairos-es/read` is one.
+ */
+export interface CaughtUpReadModel {
+  readonly store: KeyedProjectionStore
+  readonly slices: Readonly<Record<string, AnyProjection>>
+}
+
+/**
+ * The readiness of the common case: the materialisation's checkpoint has reached
+ * the last event its query selects, as the log stood when the resync attempt
+ * started. Pass the very read model whose `apply` writes the feed, so its query
+ * and its checkpoint are the ones the replay moves.
+ *
+ * The point is the last event the read model's QUERY selects, and not the log's
+ * head, because the runner advances its checkpoint to the positions it is
+ * delivered and is never delivered an event its query does not select: a
+ * checkpoint judged against a head an unrelated event moved would never arrive.
+ * The store hands no cheaper way to that position than reading up to it, so each
+ * attempt starts with one paged read of the query's events past the checkpoint —
+ * the backlog the replay itself is about to read, a page at a time and never held
+ * whole — and each probe after it is one checkpoint read.
+ *
+ * Generation-blind, deliberately: the read model has one checkpoint whatever the
+ * feed's generation, and an attempt whose generation moves on while it waits is
+ * abandoned by the worker's own generation check, so the probe never has to know.
+ */
+export const materialisationCaughtUp = (
+  readModel: CaughtUpReadModel,
+): Effect.Effect<FeedReadiness, never, DcbEventStore> =>
+  Effect.gen(function* () {
+    const log = yield* DcbEventStore
+    const query = composeProjections(readModel.slices).query
+    const limit = ReadLimit.make(DEFAULT_CATCH_UP_PAGE_SIZE)
+    const checkpoint = Effect.mapError(
+      readModel.store.readCheckpoint,
+      (cause) =>
+        new FeedUnreadable({
+          reason: `the checkpoint of projection '${readModel.store.key.projection}' could not be read`,
+          cause,
+        }),
+    )
+
+    // The last position the query selects after `from`, or `from` itself when it
+    // selects none: page by page, so the backlog is never held in memory at once.
+    const lastSelectedAfter = (from: Position): Effect.Effect<Position> =>
+      Effect.gen(function* () {
+        let last = from
+        for (;;) {
+          const response = yield* log.read(query, { after: last, limit })
+          const page = yield* Stream.runCollect(response.events)
+          const final = Arr.last(page)
+          if (Option.isSome(final)) {
+            last = final.value.position
+          }
+          if (page.length < limit) {
+            return last
+          }
+        }
+      })
+
+    const readiness: FeedReadiness = () =>
+      Effect.map(
+        Effect.flatMap(checkpoint, lastSelectedAfter),
+        (target): FeedReadinessProbe =>
+          Effect.map(checkpoint, (reached) => reached >= target),
+      )
+    return readiness
+  })