@kairos-es/feed 0.0.0-stage → 0.0.0

package/src/runFeedSync.ts

@@ -0,0 +1,882 @@
+/**
+ * `runFeedSync` — the sync worker: one per feed and sink, holding the lease,
+ * copying the feed's rows to the sink and moving its cursor only once the sink
+ * confirms them (ADR-0019).
+ *
+ * ## The lifecycle
+ *
+ * A worker runs in SESSIONS. A session takes the lease, then loops:
+ *
+ * 1. Read the cursor and the feed's generation. No cursor, or a cursor at another
+ *    generation, means the sink holds a copy of something the feed no longer is,
+ *    so the session RESYNCS (below) and loops.
+ * 2. Otherwise it TAILS: the rows after the cursor, through core's generic
+ *    subscribe machine with `emit: 'page'`, so each page is sent as it is read and
+ *    the worker holds one page in memory, never the backlog. Each page is checked
+ *    against the generation first, and so is every poll interval on its own, since
+ *    a reset that writes nothing yields no page; a generation that moved since the
+ *    cursor was saved ends the tail, and step 1 sees the mismatch.
+ *
+ * Every page goes through one delivery: send, park the keys the sink refuses with
+ * its reasons, send the rest again, confirm. Then ONE compare-and-set moves the
+ * cursor past the whole page and moves the parked list with it — the refused keys
+ * parked, every key that was taken unparked — and fenced on the lease holder, so a
+ * worker that lost its lease moves nothing. Then tombstones every sink has passed
+ * are collected.
+ *
+ * The rules that sequence keeps, each a consequence of a sink not being
+ * transactional:
+ *
+ * - **The cursor moves only after `confirm`.** A crash between `apply` and
+ *   `confirm` resends the batch; a sink is idempotent, so that costs a resend and
+ *   nothing else.
+ * - **A refused key does not hold the feed up.** It is parked and the cursor moves
+ *   past it. It is retried when its row next changes, because a change gives the
+ *   row a `seq` above the cursor, and it leaves the parked list once it is sent
+ *   successfully — a tombstone for a parked key included, which is how a refused
+ *   record that is then deleted stops being parked without anyone acting.
+ * - **A retryable failure keeps the cursor.** The session ends, and supervision
+ *   (below) starts a new one after a back-off no shorter than the sink's
+ *   `retryAfter`; the new session sends the same page again.
+ *
+ * ## Resync
+ *
+ * Every row, page by page from the origin, goes into `resync.begin`'s fresh
+ * target; then the generation is checked once more, the target is published over
+ * the live copy, and the cursor is saved at the new generation with the parked list
+ * REPLACED by the refusals the resync met. Resync progress is held in memory only,
+ * so a crash restarts it from the origin, with a new `begin` that drops the
+ * half-built target the crash left.
+ *
+ * ### Waiting for the materialisation
+ *
+ * The feed's end is not the rebuild's end. After a reset the materialisation
+ * replays the log and writes the feed as it goes, so a resync that reaches the
+ * feed's last row has reached wherever the replay had got to, and publishing there
+ * swaps a complete live copy for a partial one. `readiness.ts` owns that argument.
+ * So `runFeedSync` takes an optional `readiness`: asked once as each attempt
+ * starts, it hands back a probe, and the attempt does not publish until the probe
+ * says the materialisation has caught up. Until then the attempt polls: once per
+ * poll interval it checks the generation, abandoning the attempt as stale if a
+ * reset moved it again, asks the probe, and reads the feed on from where it left
+ * off, loading what the replay wrote meanwhile. The probe is asked BEFORE that
+ * read, so the read that follows a `true` sees every row the materialisation had
+ * committed by the time it answered, since a feed row commits with the checkpoint
+ * that produced it. The lease is still checked immediately before `publish`, and
+ * the generation once more just before that, however long the wait was.
+ *
+ * With no `readiness` the attempt publishes at the feed's end, which is exactly the
+ * behaviour above, and search can serve a PARTIAL copy for as long as the replay
+ * takes. An in-place rebuild has no publish; there the wait holds the closing
+ * reconcile and the cursor save instead, so the reconcile never deletes a record
+ * the replay has not reached yet. Without a `readiness` an in-place rebuild
+ * reconciles at the feed's end too, and so deletes from the live copy every record
+ * the replay had not yet written again; the tail sends each back as the replay
+ * reaches it.
+ *
+ * The wait is UNBOUNDED. A probe that never says yes holds the attempt for good,
+ * with no error and no restart, because publishing early is the very failure the
+ * wait exists to prevent and no timeout could tell a stuck replay from a long one.
+ * It is made visible instead: `inspectFeed` reports the sink `resyncDue` for as long
+ * as it lasts, since the cursor stays at the old generation, and every
+ * `HELD_WAIT_WARN_POLLS` poll intervals (`tuning.ts`) the attempt logs a warning,
+ * "feed sync rebuild held on readiness", with the feed, the sink, the generation,
+ * the polls so far and how long it has been held.
+ *
+ * Each `begin` names a target of its own, never one an earlier attempt used, and
+ * that is what bounds a lease lapse here. A worker that has lost its lease can
+ * still land one call after the new holder's (`FeedSyncStore.ts` says when). Were
+ * the target named after its generation alone, that call could be a `publish` of
+ * the NEW holder's half-built target, which then goes on loading into a fresh one
+ * holding only the later pages, publishes it, and saves its cursor at the head:
+ * whole pages lost, and nothing to bring them back. With a target per attempt, a
+ * stale `publish` moves only the stale worker's own target — or fails, retryably,
+ * when a newer `begin` has dropped it — and a stale `begin` drops the new holder's
+ * target, whose `publish` then fails retryably and whose resync starts again.
+ * What a stale `publish` of its own target still costs is `FeedSyncStore.ts`'s to
+ * say, with the rest of the lease's window.
+ *
+ * `begin`, `load`, `apply` and `publish` each run behind `ensureHeld`; `confirm`
+ * and a reconcile's key listing change nothing, and do not.
+ *
+ * Tombstones are sent as deletes, although the target starts empty. The feed does
+ * not stand still while a resync pages through it: a key loaded on an early page
+ * can be deleted by the read model before the resync is done, and its tombstone
+ * then sits at a `seq` a LATER page reads. Dropping it would load the key, save
+ * the cursor past its delete, and publish a target that holds a record the feed no
+ * longer has, with nothing ever coming to remove it. A tombstone for a key the
+ * target never held costs a send and changes nothing, since a delete of an absent
+ * key is a no-op by the sink laws.
+ *
+ * The same movement decides the parked list. A key can be met more than once in
+ * one resync — refused on an early page, then changed and met again later — so
+ * the list saved is what each key's LAST change in the resync came to: parked if
+ * that change was refused, absent if it was taken. Keeping every refusal the
+ * resync met would leave parked a key the target holds, or one the feed deleted.
+ *
+ * A sink with no `resync` is resynced IN PLACE: every row is sent to the live copy,
+ * live rows as upserts and tombstones as deletes, and then a reconcile deletes what
+ * the feed no longer holds. The live copy serves a mixture of old and new from the
+ * first page on, which is the price of a target with no atomic swap. Its refusals
+ * land in the parked list it saves like any resync's, with two consequences of
+ * writing over the live copy: a refused upsert leaves the PREVIOUS generation's
+ * document of that key live, and an orphan whose delete the closing reconcile was
+ * refused stays live too. Both are parked with the sink's reason, the orphan at the
+ * origin `seq` since it has no row, and the closing reconcile's report goes to
+ * `onReconcile` like a scheduled one's. A sink with neither part could never be
+ * rebuilt, and is refused at construction.
+ *
+ * ## Supervision
+ *
+ * Every failure a session meets ends it, and the worker's own loop decides what
+ * next. It follows `@kairos-es/automation`'s daemon rather than generalising the
+ * read side's supervisor, whose own doc says a progress signal the caller owns does
+ * not qualify — and the signal here, a cursor advance, is the worker's own.
+ *
+ * - `LeaseHeld`: another worker is running. This one waits a poll interval and
+ *   tries again, and the wait is not a failure: a standby worker is a healthy one,
+ *   and it takes over within a poll interval of the first worker's lease going.
+ * - Anything else: a back-off of `restartMinDelay` doubling to `restartMaxDelay`,
+ *   scaled by a jitter drawn from `Random` so a fleet of workers does not retry in
+ *   lockstep, and never shorter than a `SinkRetryable`'s `retryAfter`. A restart
+ *   that follows no cursor advance since the previous restart counts towards
+ *   `maxNoProgressRestarts`; reaching it ends the worker with `FeedSyncStalled`,
+ *   for the caller to report. Any advance resets the count and the curve.
+ *
+ * Defects are not caught: they are bugs, in the driver or in a sink or store that
+ * should have classified its fault, and supervision must not retry a bug.
+ *
+ * ## The scope
+ *
+ * The worker never returns. Run it in a fibre of its own beside the projections,
+ * and close the scope to stop it: each session runs in a child scope of the
+ * caller's, so closing the caller's scope releases the lease.
+ */
+
+import { type ReadLimit, subscribeItems } from '@kairos-es/core'
+import {
+  Array as Arr,
+  Clock,
+  Data,
+  DateTime,
+  Duration,
+  Effect,
+  Option,
+  Random,
+  Ref,
+  Scope,
+  Stream,
+} from 'effect'
+import type { NonEmptyReadonlyArray } from 'effect/Array'
+import { type Delivered, deliver } from './deliver'
+import {
+  type CursorSuperseded,
+  FeedSyncStalled,
+  type FeedUnreadable,
+  type FeedWriteFailed,
+  type LeaseHeld,
+  type LeaseLost,
+  type SinkHalted,
+  type SinkRejected,
+  type SinkRetryable,
+} from './errors'
+import type { FeedRow, FeedSource } from './Feed'
+import {
+  changeOf,
+  type FeedChange,
+  type FeedReconcile,
+  type FeedResync,
+  type FeedSink,
+} from './FeedSink'
+import type {
+  FeedCursor,
+  FeedLease,
+  FeedSyncStore,
+  ParkedChange,
+  ParkedRecord,
+} from './FeedSyncStore'
+import {
+  FEED_ORIGIN,
+  type FeedKey,
+  type FeedName,
+  type FeedSeq,
+  type Generation,
+  type SinkName,
+} from './identifiers'
+import type { FeedReadiness, FeedReadinessProbe } from './readiness'
+import {
+  type Reconciled,
+  type ReconcileReport,
+  reconcileWith,
+  unparkGone,
+} from './reconcile'
+import {
+  DEFAULT_FEED_MAX_NO_PROGRESS_RESTARTS,
+  DEFAULT_FEED_PAGE_SIZE,
+  DEFAULT_FEED_POLL_INTERVAL,
+  DEFAULT_FEED_RESTART_MAX_DELAY,
+  DEFAULT_FEED_RESTART_MIN_DELAY,
+  HELD_WAIT_WARN_POLLS,
+  positiveDuration,
+  readLimit,
+} from './tuning'
+
+/** What `onRestart` is told before each restart's back-off. */
+export interface FeedSyncRestart {
+  readonly feed: FeedName
+  readonly sink: SinkName
+  /** Consecutive restarts with no cursor advance, this one included. */
+  readonly restarts: number
+  readonly delay: Duration.Duration
+  /** The failure that ended the session. */
+  readonly reason: unknown
+}
+
+/** The worker's knobs. Every one has a default; see `tuning.ts`'s module doc. */
+export interface FeedSyncTuning {
+  /** The upper bound on an idle sleep. Default 1 second. */
+  readonly pollInterval?: Duration.Input
+  /** Rows read, and changes sent, per page. Default core's catch-up page, 512. */
+  readonly pageSize?: number
+  /** The first restart back-off. Default 100 ms. */
+  readonly restartMinDelay?: Duration.Input
+  /** The restart back-off's cap, before jitter. Default 30 seconds. */
+  readonly restartMaxDelay?: Duration.Input
+  /** Consecutive restarts with no cursor advance before `FeedSyncStalled`. Default 40. */
+  readonly maxNoProgressRestarts?: number
+  readonly onRestart?: (info: FeedSyncRestart) => Effect.Effect<void>
+  /**
+   * Told what each reconcile the worker runs did: each scheduled one, and the one
+   * that closes an in-place rebuild.
+   */
+  readonly onReconcile?: (report: ReconcileReport) => Effect.Effect<void>
+}
+
+/** Options for {@link runFeedSync}. */
+export interface RunFeedSyncOptions<D, Receipt> {
+  readonly source: FeedSource<D>
+  readonly sink: FeedSink<D, Receipt>
+  readonly store: FeedSyncStore
+  readonly tuning?: FeedSyncTuning
+  /**
+   * Reconcile the sink against the feed this often, between batches, under the
+   * worker's own lease. Needs the sink's `reconcile`; absent, the worker
+   * reconciles only as part of an in-place resync.
+   */
+  readonly reconcileEvery?: Duration.Input
+  /**
+   * Hold each resync's `publish` — or, in place, its closing reconcile — until the
+   * materialisation writing the feed has caught up with the log, for as long as
+   * that takes; see the module doc's Resync section. Absent, a resync publishes at
+   * the feed's end, and search can be PARTIAL while a rebuild's replay runs.
+   * `materialisationCaughtUp` builds the common case from the read model.
+   */
+  readonly readiness?: FeedReadiness
+}
+
+/** A session's failures: each ends the session, and supervision decides. */
+type SessionFailure =
+  | SinkRetryable
+  | SinkHalted
+  | FeedUnreadable
+  | FeedWriteFailed
+  | CursorSuperseded
+  | LeaseLost
+  | LeaseHeld
+
+/** The tail's signal that the generation moved under it. Never leaves a session. */
+class GenerationMoved extends Data.TaggedError('GenerationMoved') {}
+
+/** The jitter range applied to every restart back-off. */
+const JITTER_LOW = 0.8
+const JITTER_HIGH = 1.2
+
+/**
+ * Run the sync worker for `options.source` and `options.sink`; see the module doc.
+ * Dies at once, before taking the lease, on a sink that has neither `resync` nor
+ * `reconcile`, on a `reconcileEvery` for a sink with no `reconcile`, and on tuning
+ * that is out of range.
+ */
+export const runFeedSync = <D, Receipt>(
+  options: RunFeedSyncOptions<D, Receipt>,
+): Effect.Effect<never, FeedSyncStalled, Scope.Scope> =>
+  Effect.gen(function* () {
+    const config = yield* Effect.sync(() => resolveOptions(options))
+    const parent = yield* Effect.scope
+    const progress = yield* Ref.make(0)
+    const id = { feed: options.source.feed, sink: options.sink.name }
+    let seen = 0
+    let restarts = 0
+    for (;;) {
+      const child = yield* Scope.fork(parent)
+      const failure = yield* Effect.flip(
+        session(options, config, progress).pipe(Scope.use(child)),
+      )
+      if (failure._tag === 'LeaseHeld') {
+        yield* Effect.sleep(config.pollInterval)
+        continue
+      }
+      const advanced = yield* Ref.get(progress)
+      if (advanced !== seen) {
+        seen = advanced
+        restarts = 0
+      }
+      restarts += 1
+      if (restarts >= config.maxNoProgressRestarts) {
+        yield* Effect.logError('feed sync stalled', {
+          feed: id.feed,
+          sink: id.sink,
+          restarts,
+          reason: String(failure),
+        })
+        return yield* new FeedSyncStalled({ ...id, restarts, reason: failure })
+      }
+      const backoff = yield* restartDelay(config, restarts)
+      const delay =
+        failure._tag === 'SinkRetryable' && failure.retryAfter !== undefined
+          ? Duration.max(backoff, failure.retryAfter)
+          : backoff
+      yield* Effect.logWarning('feed sync restarting', {
+        feed: id.feed,
+        sink: id.sink,
+        restarts,
+        delayMillis: Duration.toMillis(delay),
+        reason: String(failure),
+      })
+      if (config.onRestart !== undefined) {
+        yield* config.onRestart({ ...id, restarts, delay, reason: failure })
+      }
+      yield* Effect.sleep(delay)
+    }
+  })
+
+/**
+ * How the sink is rebuilt after a reset, decided ONCE, at construction, from the
+ * parts the sink has. A sink with `resync` is rebuilt by a swap even when it also
+ * has `reconcile`; one with `reconcile` alone is rebuilt in place; one with
+ * neither is refused before this is ever built. So no later step re-asks which
+ * parts the sink has, and none carries a branch for a combination that cannot
+ * reach it.
+ */
+type Rebuild<D, Receipt> =
+  | { readonly _tag: 'Swap'; readonly resync: FeedResync<D, Receipt> }
+  | { readonly _tag: 'InPlace'; readonly reconcile: FeedReconcile }
+
+/** A scheduled reconcile: how often, and the key listing it runs over. */
+interface PeriodicReconcile {
+  readonly every: Duration.Duration
+  readonly reconcile: FeedReconcile
+}
+
+/** The options with every default applied and every value checked. */
+interface ResolvedConfig<D, Receipt> {
+  readonly pollInterval: Duration.Duration
+  readonly pageSize: number
+  readonly limit: ReadLimit
+  readonly restartMinDelay: Duration.Duration
+  readonly restartMaxDelay: Duration.Duration
+  readonly maxNoProgressRestarts: number
+  readonly onRestart: FeedSyncTuning['onRestart']
+  readonly onReconcile: FeedSyncTuning['onReconcile']
+  readonly rebuild: Rebuild<D, Receipt>
+  /** `None` when no `reconcileEvery` was asked for; see `resolveOptions`. */
+  readonly periodic: Option.Option<PeriodicReconcile>
+  /** `None` when no `readiness` was given: a resync publishes at the feed's end. */
+  readonly readiness: Option.Option<FeedReadiness>
+}
+
+const resolveOptions = <D, Receipt>(
+  options: RunFeedSyncOptions<D, Receipt>,
+): ResolvedConfig<D, Receipt> => {
+  const where = `runFeedSync(feed '${options.source.feed}', sink '${options.sink.name}')`
+  const { resync, reconcile } = options.sink
+  const rebuild = rebuildOf(where, resync, reconcile)
+  if (options.reconcileEvery !== undefined && reconcile === undefined) {
+    throw new Error(
+      `${where}: reconcileEvery is set, and the sink has no reconcile`,
+    )
+  }
+  const tuning = options.tuning ?? {}
+  const pageSize = tuning.pageSize ?? DEFAULT_FEED_PAGE_SIZE
+  const limit = readLimit(pageSize)
+  const maxNoProgressRestarts =
+    tuning.maxNoProgressRestarts ?? DEFAULT_FEED_MAX_NO_PROGRESS_RESTARTS
+  if (!Number.isInteger(maxNoProgressRestarts) || maxNoProgressRestarts <= 0) {
+    throw new Error(
+      `${where}: maxNoProgressRestarts must be a positive integer`,
+    )
+  }
+  const pollInterval = positiveDuration(
+    'pollInterval',
+    tuning.pollInterval ?? DEFAULT_FEED_POLL_INTERVAL,
+  )
+  const restartMinDelay = positiveDuration(
+    'restartMinDelay',
+    tuning.restartMinDelay ?? DEFAULT_FEED_RESTART_MIN_DELAY,
+  )
+  const restartMaxDelay = positiveDuration(
+    'restartMaxDelay',
+    tuning.restartMaxDelay ?? DEFAULT_FEED_RESTART_MAX_DELAY,
+  )
+  return {
+    pollInterval,
+    pageSize,
+    limit,
+    restartMinDelay,
+    restartMaxDelay,
+    maxNoProgressRestarts,
+    onRestart: tuning.onRestart,
+    onReconcile: tuning.onReconcile,
+    rebuild,
+    periodic:
+      options.reconcileEvery === undefined || reconcile === undefined
+        ? Option.none()
+        : Option.some({
+            every: positiveDuration('reconcileEvery', options.reconcileEvery),
+            reconcile,
+          }),
+    readiness: Option.fromUndefinedOr(options.readiness),
+  }
+}
+
+/** The sink's {@link Rebuild}, refusing a sink that could never be rebuilt. */
+const rebuildOf = <D, Receipt>(
+  where: string,
+  resync: FeedResync<D, Receipt> | undefined,
+  reconcile: FeedReconcile | undefined,
+): Rebuild<D, Receipt> => {
+  if (resync !== undefined) {
+    return { _tag: 'Swap', resync }
+  }
+  if (reconcile !== undefined) {
+    return { _tag: 'InPlace', reconcile }
+  }
+  throw new Error(
+    `${where}: the sink has neither resync nor reconcile, so it could never be rebuilt after a reset`,
+  )
+}
+
+/**
+ * The delay before restart `n` (1-based): `min(cap, min × 2^(n-1))`, scaled by a
+ * jitter drawn from `Random`, so a seeded `Random` makes it repeat.
+ */
+const restartDelay = <D, Receipt>(
+  config: ResolvedConfig<D, Receipt>,
+  n: number,
+): Effect.Effect<Duration.Duration> =>
+  Effect.map(Random.nextBetween(JITTER_LOW, JITTER_HIGH), (jitter) => {
+    const raw = Duration.toMillis(config.restartMinDelay) * 2 ** (n - 1)
+    const capped = Math.min(raw, Duration.toMillis(config.restartMaxDelay))
+    return Duration.millis(Math.round(capped * jitter))
+  })
+
+/** What a generation check found: the feed's generation now, and whether it moved. */
+interface GenerationCheck {
+  readonly current: Generation
+  /** True when the feed is no longer at the generation held — or none was held. */
+  readonly moved: boolean
+}
+
+/**
+ * Read the feed's generation and say whether it is still `held`. The ONE place the
+ * driver asks whether a reset has happened: before choosing to tail or resync, per
+ * tail page, per poll interval of the tail's watch, per poll interval of a resync
+ * waiting on its readiness, and before a resync publishes.
+ * A `held` of `None` — no cursor yet — counts as moved, since the sink then holds
+ * a copy of no generation at all.
+ */
+const checkGeneration = <D>(
+  source: FeedSource<D>,
+  held: Option.Option<Generation>,
+): Effect.Effect<GenerationCheck, FeedUnreadable> =>
+  Effect.map(source.generation, (current) => ({
+    current,
+    moved: Option.isNone(held) || held.value !== current,
+  }))
+
+/** How the cursor moves at the end of a page or a resync. */
+interface Advance {
+  readonly expected: Option.Option<FeedCursor>
+  readonly next: FeedCursor
+  readonly parked: ParkedChange
+}
+
+/**
+ * Everything one session threads through its tail and resync, built ONCE when the
+ * lease is taken. The lease, the store and the progress counter are closed over by
+ * `deliverRows`, `advance` and `reconcile` rather than handed to each step, so a
+ * step cannot send without the lease check or advance without counting progress.
+ */
+interface SessionContext<D, Receipt> {
+  readonly source: FeedSource<D>
+  readonly sink: FeedSink<D, Receipt>
+  readonly config: ResolvedConfig<D, Receipt>
+  readonly lease: FeedLease
+  /** Deliver `rows` through `send` — the live `apply`, or a resync target's `load`. */
+  readonly deliverRows: (
+    send: (
+      batch: NonEmptyReadonlyArray<FeedChange<D>>,
+    ) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>,
+    rows: NonEmptyReadonlyArray<FeedRow<D>>,
+  ) => Effect.Effect<
+    Delivered,
+    SinkRetryable | SinkHalted | LeaseLost | FeedUnreadable
+  >
+  /**
+   * Move the cursor, fenced on this session's holder, with the parked list; count
+   * the advance as progress; collect the tombstones every sink has now passed.
+   */
+  readonly advance: (
+    args: Advance,
+  ) => Effect.Effect<void, CursorSuperseded | FeedWriteFailed>
+  /** Unpark what a reconcile found gone; see `unparkGone` in `reconcile.ts`. */
+  readonly unparkGone: (isGone: Reconciled['isGone']) => Effect.Effect<void>
+  /** Reconcile the live copy over `reconcile`, under this session's lease. */
+  readonly reconcile: (
+    reconcile: FeedReconcile,
+  ) => Effect.Effect<
+    Reconciled,
+    SinkRetryable | SinkHalted | LeaseLost | FeedUnreadable
+  >
+}
+
+const makeSessionContext = <D, Receipt>(
+  options: RunFeedSyncOptions<D, Receipt>,
+  config: ResolvedConfig<D, Receipt>,
+  progress: Ref.Ref<number>,
+  lease: FeedLease,
+): SessionContext<D, Receipt> => {
+  const { source, sink, store } = options
+  return {
+    source,
+    sink,
+    config,
+    lease,
+    deliverRows: (send, rows) =>
+      deliver({
+        send,
+        confirm: sink.confirm,
+        lease,
+        pending: Arr.map(rows, (row) => ({
+          change: changeOf(row),
+          seq: row.seq,
+        })),
+      }),
+    advance: (args) =>
+      Effect.gen(function* () {
+        yield* store.casCursor({
+          feed: source.feed,
+          sink: sink.name,
+          holder: lease.holder,
+          expected: args.expected,
+          next: args.next,
+          parked: args.parked,
+        })
+        yield* Ref.update(progress, (n) => n + 1)
+        yield* store.collectTombstones(source.feed)
+      }),
+    unparkGone: (isGone) =>
+      unparkGone(store, { feed: source.feed, sink: sink.name }, isGone),
+    reconcile: (reconcile) =>
+      reconcileWith({
+        source,
+        sink,
+        reconcile,
+        lease,
+        pageSize: config.pageSize,
+      }),
+  }
+}
+
+/** One session: the lease, then resync or tail, forever, until something fails. */
+const session = <D, Receipt>(
+  options: RunFeedSyncOptions<D, Receipt>,
+  config: ResolvedConfig<D, Receipt>,
+  progress: Ref.Ref<number>,
+): Effect.Effect<never, SessionFailure, Scope.Scope> =>
+  Effect.gen(function* () {
+    const id = { feed: options.source.feed, sink: options.sink.name }
+    const lease = yield* options.store.acquireLease(id)
+    const ctx = makeSessionContext(options, config, progress, lease)
+    const step = Effect.gen(function* () {
+      const cursor = yield* options.store.readCursor(id)
+      const check = yield* checkGeneration(
+        ctx.source,
+        Option.map(cursor, (c) => c.generation),
+      )
+      if (Option.isSome(cursor) && !check.moved) {
+        return yield* tail(ctx, cursor.value).pipe(
+          Effect.catchTag('GenerationMoved', () => Effect.void),
+        )
+      }
+      return yield* resync(ctx, {
+        generation: check.current,
+        previous: cursor,
+      })
+    })
+    return yield* Effect.forever(step)
+  })
+
+/** The tail: pages after the cursor, each delivered then advanced past. */
+const tail = <D, Receipt>(
+  ctx: SessionContext<D, Receipt>,
+  start: FeedCursor,
+): Effect.Effect<void, SessionFailure | GenerationMoved, Scope.Scope> =>
+  Effect.gen(function* () {
+    const { source, sink, config } = ctx
+    const current = yield* Ref.make(start)
+
+    // Fails `GenerationMoved` once the feed has left the cursor's generation.
+    const ensureGeneration = Effect.gen(function* () {
+      const cursor = yield* Ref.get(current)
+      const check = yield* checkGeneration(
+        source,
+        Option.some(cursor.generation),
+      )
+      if (check.moved) {
+        return yield* new GenerationMoved()
+      }
+      return cursor
+    })
+
+    const onPage = (
+      rows: NonEmptyReadonlyArray<FeedRow<D>>,
+    ): Effect.Effect<void, SessionFailure | GenerationMoved> =>
+      Effect.gen(function* () {
+        const cursor = yield* ensureGeneration
+        const delivered = yield* ctx.deliverRows(sink.apply, rows)
+        const next = {
+          generation: cursor.generation,
+          seq: Arr.lastNonEmpty(rows).seq,
+        }
+        yield* ctx.advance({
+          expected: Option.some(cursor),
+          next,
+          parked: {
+            _tag: 'Amend',
+            park: delivered.parked,
+            unpark: delivered.sent,
+          },
+        })
+        yield* Ref.set(current, next)
+      })
+
+    // A periodic reconcile parks nothing of its own: an orphan it could not delete
+    // is still in the copy, so the next period finds it and tries again, and the
+    // report says so meanwhile. What it does owe the parked list is every park of a
+    // key now gone from the sink, or `inspectFeed` would go on reporting a refusal
+    // for a record that is no longer there; `unparkGone` says why its failure only
+    // warns.
+    const onReconcile = (reconcile: FeedReconcile) =>
+      Effect.gen(function* () {
+        const { report, isGone } = yield* ctx.reconcile(reconcile)
+        if (config.onReconcile !== undefined) {
+          yield* config.onReconcile(report)
+        }
+        yield* ctx.unparkGone(isGone)
+      })
+
+    const pages = subscribeItems<FeedRow<D>, FeedSeq, FeedUnreadable>({
+      fetch: source.changesAfter,
+      cursorOf: (row) => row.seq,
+      initial: start.seq,
+      ...(source.listen === undefined ? {} : { listen: source.listen }),
+      pollInterval: Duration.toMillis(config.pollInterval),
+      catchUpPageSize: config.pageSize,
+      emit: 'page',
+    }).pipe(
+      Stream.chunks,
+      Stream.map((rows) => ({ _tag: 'Page' as const, rows })),
+    )
+
+    // A reset that writes no row would otherwise go unseen: the tail checks the
+    // generation per PAGE, and an emptied feed yields no page. So the generation is
+    // also read once per poll interval on its own, and a move ends the tail.
+    const generationWatch = Stream.tick(config.pollInterval).pipe(
+      Stream.drop(1),
+      Stream.as({ _tag: 'Watch' as const }),
+    )
+    const reconcileTicks = Option.match(config.periodic, {
+      onNone: () => Stream.empty,
+      onSome: ({ every, reconcile }) =>
+        Stream.tick(every).pipe(
+          // `tick` fires once at the start; a reconcile is due one period in.
+          Stream.drop(1),
+          Stream.as({ _tag: 'Reconcile' as const, reconcile }),
+        ),
+    })
+
+    // One consumer, so a page, a watch and a reconcile never overlap: nothing else
+    // sends while a reconcile runs, which is what lets it run under this lease.
+    yield* Stream.runForEach(
+      Stream.merge(pages, Stream.merge(generationWatch, reconcileTicks)),
+      (event) =>
+        event._tag === 'Page'
+          ? onPage(event.rows)
+          : event._tag === 'Watch'
+            ? Effect.asVoid(ensureGeneration)
+            : onReconcile(event.reconcile),
+    )
+  })
+
+/**
+ * One resync attempt's two ends: where its rows are sent, and what makes the
+ * result the copy the sink serves. Built from the session's {@link Rebuild}, so
+ * the resync loop itself never asks which kind of sink it has.
+ */
+interface Rebuilding<D, Receipt> {
+  readonly send: (
+    batch: NonEmptyReadonlyArray<FeedChange<D>>,
+  ) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>
+  /**
+   * Make the result the served copy, and say what that step itself sent and had
+   * refused, so its refusals are parked exactly as a page's are. A swap's
+   * `publish` sends no record and so reports nothing; an in-place rebuild's
+   * closing reconcile sends the deletes of the orphans, and an orphan whose delete
+   * the sink refused is STILL IN THE COPY, so leaving it out of the parked list
+   * would save a cursor at the new generation over a copy that holds a record the
+   * feed deleted, and say nothing about it. The tail's periodic reconcile parks
+   * nothing, since it tries again next period, but it unparks whatever it finds
+   * gone from the sink.
+   */
+  readonly finish: Effect.Effect<Delivered, SessionFailure>
+}
+
+/** What a swap's `publish` sent: no record at all. */
+const NOTHING_SENT: Delivered = { sent: [], parked: [] }
+
+const startRebuild = <D, Receipt>(
+  ctx: SessionContext<D, Receipt>,
+  generation: Generation,
+): Effect.Effect<Rebuilding<D, Receipt>, SessionFailure> => {
+  const { rebuild } = ctx.config
+  if (rebuild._tag === 'InPlace') {
+    return Effect.succeed({
+      send: ctx.sink.apply,
+      finish: Effect.gen(function* () {
+        const { report } = yield* ctx.reconcile(rebuild.reconcile)
+        if (ctx.config.onReconcile !== undefined) {
+          yield* ctx.config.onReconcile(report)
+        }
+        // An orphan has no row, so its park carries the origin as its `seq`: there
+        // is no change of it to point at, and `retryParked` on a key with no row
+        // simply unparks it.
+        const parkedAt = yield* DateTime.now
+        return {
+          sent: report.deleted,
+          parked: report.refused.map(({ key, reason }) => ({
+            key,
+            seq: FEED_ORIGIN,
+            reason,
+            parkedAt,
+          })),
+        }
+      }),
+    })
+  }
+  const { resync } = rebuild
+  return Effect.gen(function* () {
+    // `begin` destroys: it empties a target and drops every other one, which can
+    // be the half-built target of the worker that holds the lease now. So it is a
+    // send like any other, and runs only under a lease just seen held.
+    yield* ctx.lease.ensureHeld
+    const target = yield* resync.begin(generation)
+    return {
+      send: (batch) => resync.load(target, batch),
+      finish: Effect.as(
+        Effect.andThen(ctx.lease.ensureHeld, resync.publish(target)),
+        NOTHING_SENT,
+      ),
+    }
+  })
+}
+
+/** Always ready: the probe of a worker given no `readiness`. */
+const ALWAYS_READY: FeedReadinessProbe = Effect.succeed(true)
+
+/** A full resync at `generation`; see the module doc. */
+const resync = <D, Receipt>(
+  ctx: SessionContext<D, Receipt>,
+  at: {
+    readonly generation: Generation
+    readonly previous: Option.Option<FeedCursor>
+  },
+): Effect.Effect<void, SessionFailure> =>
+  Effect.gen(function* () {
+    const { source, config } = ctx
+    // Asked before anything is read, so the point the probe judges by is fixed
+    // before the first page: see the module doc's "Waiting for the materialisation".
+    const ready = yield* Option.match(config.readiness, {
+      onNone: () => Effect.succeed(ALWAYS_READY),
+      onSome: (readiness) => readiness(at.generation),
+    })
+    const rebuilding = yield* startRebuild(ctx, at.generation)
+    const stale = Effect.map(
+      checkGeneration(source, Option.some(at.generation)),
+      (check) => check.moved,
+    )
+
+    // By key, so a later page's outcome for a key replaces an earlier one's: the
+    // list saved is what each key's LAST change in this resync came to.
+    const parked = new Map<FeedKey, ParkedRecord>()
+    const settle = (delivered: Delivered) => {
+      for (const sent of delivered.sent) {
+        parked.delete(sent)
+      }
+      for (const record of delivered.parked) {
+        parked.set(record.key, record)
+      }
+    }
+    let last: FeedSeq = FEED_ORIGIN
+    const heldSince = yield* Clock.currentTimeMillis
+    let polls = 0
+    for (;;) {
+      // Before the read, so a `true` covers every row the read then finds.
+      const caughtUp = yield* ready
+      for (;;) {
+        const rows = yield* source.changesAfter(last, config.limit)
+        if (!Arr.isReadonlyArrayNonEmpty(rows)) {
+          break
+        }
+        // Tombstones included: see the module doc's Resync section.
+        settle(yield* ctx.deliverRows(rebuilding.send, rows))
+        last = Arr.lastNonEmpty(rows).seq
+        if (rows.length < config.pageSize) {
+          break
+        }
+      }
+      if (caughtUp) {
+        break
+      }
+      yield* Effect.sleep(config.pollInterval)
+      if (yield* stale) {
+        // Reset again while this waited: abandon the attempt, as below.
+        return
+      }
+      polls += 1
+      // The wait has no bound — see the module doc — so a probe that never says
+      // yes must at least be seen not to. Every string or number, never a bigint,
+      // so a logger that JSON-encodes the line keeps it.
+      if (polls % HELD_WAIT_WARN_POLLS === 0) {
+        yield* Effect.logWarning('feed sync rebuild held on readiness', {
+          feed: source.feed,
+          sink: ctx.sink.name,
+          generation: String(at.generation),
+          polls,
+          heldMillis: (yield* Clock.currentTimeMillis) - heldSince,
+        })
+      }
+    }
+
+    if (yield* stale) {
+      // Reset again while this ran: what was loaded is already stale. Leave the
+      // cursor alone, and the session's next step starts the resync over.
+      return
+    }
+    settle(yield* rebuilding.finish)
+    yield* ctx.advance({
+      expected: at.previous,
+      next: { generation: at.generation, seq: last },
+      parked: { _tag: 'Replace', park: Array.from(parked.values()) },
+    })
+  })