@kairos-es/feed 0.0.0-stage → 0.0.0

package/src/inMemoryFeed.ts

@@ -0,0 +1,582 @@
+/**
+ * The in-memory feed: a `BatchedFeedWriter`, a `FeedSource` and a `FeedSyncStore`
+ * over one `Ref`, and the PARITY ORACLE every other feed backend is held against,
+ * as the in-memory event store and projection store are for theirs.
+ *
+ * ## One state, so the store's atomic steps are atomic
+ *
+ * Rows, the sequence, the generation, every sink's cursor, lease holder and parked
+ * list live in ONE `Ref`, and each operation the port calls atomic is one
+ * `Ref.modify` of it. That is what makes this a faithful oracle for a SQL backend
+ * whose same operations are one transaction: a cursor advance and its parked list
+ * cannot be observed half-done, and tombstone collection reads the lowest cursor in
+ * the same step as it deletes.
+ *
+ * ## Used inside an in-memory `ProjectionStore` commit
+ *
+ * `write` buffers its body's upserts and removes and lands them in one
+ * `Ref.update` once the body succeeds, so the feed's half of a batch is a single
+ * atomic effect — what the `ProjectionStore` port asks of a non-transactional
+ * store's per-batch write. Make it the LAST step of the read model's `apply`: the
+ * in-memory store cannot roll back a view written before a feed write that failed,
+ * and a feed write here can fail only by a defect.
+ *
+ * ## Ephemeral, and what that does to the generation
+ *
+ * Nothing here survives the process, so the generation is a fresh value per boot,
+ * drawn from `Random` and never from the host. A sink that outlives the process —
+ * a real search index, or an in-memory sink a test keeps across two feeds — sees a
+ * generation it has no cursor for and resyncs, rather than holding a frozen copy of
+ * a feed that no longer exists. A reset through {@link withInMemoryFeed} moves the
+ * generation on by one.
+ *
+ * For the same reason the feed refuses to be bound to a DURABLE `ProjectionStore`:
+ * the view would survive a restart and the feed would not, the next boot's resync
+ * would publish an empty feed over the sink, and nothing would replay the view to
+ * fill it. That is R2's failure — durability that does not match — in a new place.
+ *
+ * ## Equality
+ *
+ * Whether an upsert changes anything is decided on the document ENCODED through
+ * its `Schema`, compared as JSON with object keys unordered. That is the same
+ * comparison a SQL backend makes on a JSON column, so a document that differs only
+ * in key order is no change in either. A document that fails to encode is a defect
+ * in the read model, not a fault of the feed.
+ */
+import type { Position, ReadLimit } from '@kairos-es/core'
+import type { CheckpointKey, ProjectionStore } from '@kairos-es/read'
+import {
+  DateTime,
+  Effect,
+  Option,
+  PubSub,
+  Random,
+  Ref,
+  Schema,
+  Stream,
+} from 'effect'
+import { CursorSuperseded, LeaseHeld } from './errors'
+import type { BatchedFeedWriter, FeedRow, FeedSource, FeedWriter } from './Feed'
+import {
+  type FeedCursor,
+  type FeedLease,
+  type FeedSyncDescription,
+  type FeedSyncStore,
+  type ParkedChange,
+  type ParkedRecord,
+  type SinkSyncState,
+  sameCursor,
+} from './FeedSyncStore'
+import {
+  FEED_ORIGIN,
+  type FeedKey,
+  type FeedName,
+  FeedSeq,
+  type FeedSinkId,
+  Generation,
+  LeaseHolder,
+  type SinkName,
+} from './identifiers'
+
+/** The in-memory feed; see the module doc. */
+export interface InMemoryFeed<D> extends BatchedFeedWriter<D> {
+  readonly source: FeedSource<D>
+  readonly syncStore: FeedSyncStore
+}
+
+/** A stored row, with its document's canonical encoding (absent for a tombstone). */
+interface StoredRow<D> {
+  readonly row: FeedRow<D>
+  readonly encoded: string | undefined
+}
+
+interface SinkState {
+  readonly cursor: Option.Option<FeedCursor>
+  readonly lastAdvance: Option.Option<DateTime.Utc>
+  /** The lease's one home: taken and released only through `Ref.modify`. */
+  readonly holder: Option.Option<LeaseHolder>
+  readonly parked: ReadonlyMap<FeedKey, ParkedRecord>
+}
+
+interface State<D> {
+  readonly generation: Generation
+  readonly lastSeq: bigint
+  readonly rows: ReadonlyMap<FeedKey, StoredRow<D>>
+  readonly sinks: ReadonlyMap<SinkName, SinkState>
+  readonly boundTo: Option.Option<CheckpointKey>
+}
+
+const EMPTY_SINK: SinkState = {
+  cursor: Option.none(),
+  lastAdvance: Option.none(),
+  holder: Option.none(),
+  parked: new Map(),
+}
+
+/**
+ * A JSON value as a string with every object's keys sorted, so two encodings that
+ * differ only in key order compare equal.
+ */
+const canonicalJson = (value: unknown): string =>
+  JSON.stringify(value, (_key, held: unknown) =>
+    held !== null && typeof held === 'object' && !Array.isArray(held)
+      ? Object.fromEntries(
+          Object.entries(held as Record<string, unknown>).sort(
+            ([left], [right]) => (left < right ? -1 : left > right ? 1 : 0),
+          ),
+        )
+      : held,
+  )
+
+/** The private half of a feed that only {@link withInMemoryFeed} reaches. */
+interface FeedInternals {
+  readonly bind: (key: CheckpointKey) => Effect.Effect<void>
+  readonly reset: Effect.Effect<void>
+}
+
+const internals = new WeakMap<object, FeedInternals>()
+
+/** Options for {@link makeInMemoryFeed}. */
+export interface InMemoryFeedOptions<D, I> {
+  readonly feed: FeedName
+  /** The document's schema: what an upsert encodes, and compares, through. */
+  readonly doc: Schema.Codec<D, I>
+}
+
+/** Build an in-memory feed at a fresh generation; see the module doc. */
+export const makeInMemoryFeed = <D, I>(
+  options: InMemoryFeedOptions<D, I>,
+): Effect.Effect<InMemoryFeed<D>> =>
+  Effect.gen(function* () {
+    const feed = options.feed
+    const initial = yield* Random.nextIntBetween(1, Number.MAX_SAFE_INTEGER)
+    const state = yield* Ref.make<State<D>>({
+      generation: Generation.make(BigInt(initial)),
+      lastSeq: 0n,
+      rows: new Map(),
+      sinks: new Map(),
+      boundTo: Option.none(),
+    })
+    const wake = yield* PubSub.sliding<void>(1)
+    const encode = Schema.encodeEffect(options.doc)
+
+    const requireFeed = (name: FeedName): Effect.Effect<void> =>
+      name === feed
+        ? Effect.void
+        : Effect.die(
+            new Error(
+              `in-memory feed '${feed}' was asked about feed '${name}', which it does not hold`,
+            ),
+          )
+
+    const sinkOf = (s: State<D>, sink: SinkName): SinkState =>
+      s.sinks.get(sink) ?? EMPTY_SINK
+
+    const withSink = (
+      s: State<D>,
+      sink: SinkName,
+      f: (current: SinkState) => SinkState,
+    ): State<D> => ({
+      ...s,
+      sinks: new Map(s.sinks).set(sink, f(sinkOf(s, sink))),
+    })
+
+    // --- the write side ---------------------------------------------------
+
+    const write = <A, E, R>(
+      at: Position,
+      body: (writer: FeedWriter<D>) => Effect.Effect<A, E, R>,
+    ): Effect.Effect<A, E, R> =>
+      Effect.gen(function* () {
+        const buffer = new Map<FeedKey, Option.Option<D>>()
+        const set = (key: FeedKey, doc: Option.Option<D>) =>
+          Effect.sync(() => {
+            buffer.delete(key)
+            buffer.set(key, doc)
+          })
+        const result = yield* body({
+          upsert: (key, doc) => set(key, Option.some(doc)),
+          remove: (key) => set(key, Option.none()),
+        })
+        const staged: Array<{
+          key: FeedKey
+          doc: Option.Option<D>
+          encoded: string | undefined
+        }> = []
+        for (const [key, doc] of buffer) {
+          const encoded = Option.isSome(doc)
+            ? canonicalJson(yield* Effect.orDie(encode(doc.value)))
+            : undefined
+          staged.push({ key, doc, encoded })
+        }
+        const wrote = yield* Ref.modify(state, (s): [boolean, State<D>] => {
+          const rows = new Map(s.rows)
+          let lastSeq = s.lastSeq
+          for (const { key, doc, encoded } of staged) {
+            const stored = rows.get(key)
+            const unchanged =
+              encoded === undefined
+                ? stored === undefined || stored.encoded === undefined
+                : stored !== undefined && stored.encoded === encoded
+            if (unchanged) {
+              continue
+            }
+            lastSeq += 1n
+            rows.set(key, {
+              row: { key, doc, seq: FeedSeq.make(lastSeq), eventPosition: at },
+              encoded,
+            })
+          }
+          return lastSeq === s.lastSeq
+            ? [false, s]
+            : [true, { ...s, rows, lastSeq }]
+        })
+        if (wrote) {
+          yield* PubSub.publish(wake, undefined)
+        }
+        return result
+      })
+
+    // --- the read side ----------------------------------------------------
+
+    const changesAfter = (
+      after: FeedSeq,
+      limit: ReadLimit,
+    ): Effect.Effect<ReadonlyArray<FeedRow<D>>> =>
+      Effect.map(Ref.get(state), (s) =>
+        Array.from(s.rows.values(), (stored) => stored.row)
+          .filter((row) => row.seq > after)
+          .sort((left, right) =>
+            left.seq < right.seq ? -1 : left.seq > right.seq ? 1 : 0,
+          )
+          .slice(0, limit),
+      )
+
+    const source: FeedSource<D> = {
+      feed,
+      generation: Effect.map(Ref.get(state), (s) => s.generation),
+      changesAfter,
+      listen: Stream.fromPubSub(wake),
+    }
+
+    // --- the sync store ---------------------------------------------------
+
+    // Take-if-free and record the holder in ONE `Ref.modify`, so the check and the
+    // take cannot be split by a rival, and the lease has one home that `casCursor`,
+    // `describe` and `removeSink` read. `acquireRelease` runs the take
+    // uninterruptibly, so a lease taken is always paired with its release.
+    const acquireLease = (id: FeedSinkId) =>
+      Effect.gen(function* () {
+        yield* requireFeed(id.feed)
+        const holder = LeaseHolder.make(
+          `in-memory:${yield* Random.nextIntBetween(1, Number.MAX_SAFE_INTEGER)}`,
+        )
+        const take = Effect.flatMap(
+          Ref.modify(state, (s): [boolean, State<D>] =>
+            Option.isSome(sinkOf(s, id.sink).holder)
+              ? [false, s]
+              : [
+                  true,
+                  withSink(s, id.sink, (sink) => ({
+                    ...sink,
+                    holder: Option.some(holder),
+                  })),
+                ],
+          ),
+          (taken) => (taken ? Effect.void : Effect.fail(new LeaseHeld(id))),
+        )
+        // Cleared only while it is still this holder's, so a release can never
+        // take the lease from anyone else.
+        const release = Ref.update(state, (s) => {
+          const current = s.sinks.get(id.sink)
+          return current !== undefined &&
+            Option.isSome(current.holder) &&
+            current.holder.value === holder
+            ? withSink(s, id.sink, (sink) => ({
+                ...sink,
+                holder: Option.none(),
+              }))
+            : s
+        })
+        yield* Effect.acquireRelease(take, () => release)
+        const lease: FeedLease = {
+          holder,
+          ensureHeld: Effect.void,
+        }
+        return lease
+      })
+
+    const casCursor: FeedSyncStore['casCursor'] = (args) =>
+      Effect.flatMap(
+        Effect.zip(requireFeed(args.feed), DateTime.now),
+        ([, now]) =>
+          Effect.flatMap(
+            Ref.modify(state, (s): [boolean, State<D>] => {
+              const current = sinkOf(s, args.sink)
+              const fenced =
+                Option.isSome(current.holder) &&
+                current.holder.value === args.holder
+              if (!fenced || !sameCursor(current.cursor, args.expected)) {
+                return [false, s]
+              }
+              return [
+                true,
+                withSink(s, args.sink, (sink) => ({
+                  ...sink,
+                  cursor: Option.some(args.next),
+                  lastAdvance: Option.some(now),
+                  parked: moveParked(sink.parked, args.parked),
+                })),
+              ]
+            }),
+            (landed) =>
+              landed
+                ? Effect.void
+                : Effect.fail(
+                    new CursorSuperseded({ feed: args.feed, sink: args.sink }),
+                  ),
+          ),
+      )
+
+    const retryParked: FeedSyncStore['retryParked'] = (args) =>
+      Effect.gen(function* () {
+        yield* requireFeed(args.feed)
+        const retried = yield* Ref.modify(state, (s): [boolean, State<D>] => {
+          const sink = sinkOf(s, args.sink)
+          if (!sink.parked.has(args.key)) {
+            return [false, s]
+          }
+          const parked = new Map(sink.parked)
+          parked.delete(args.key)
+          const unparked = withSink(s, args.sink, (current) => ({
+            ...current,
+            parked,
+          }))
+          const stored = s.rows.get(args.key)
+          if (stored === undefined) {
+            return [true, unparked]
+          }
+          const lastSeq = s.lastSeq + 1n
+          return [
+            true,
+            {
+              ...unparked,
+              lastSeq,
+              rows: new Map(s.rows).set(args.key, {
+                ...stored,
+                row: { ...stored.row, seq: FeedSeq.make(lastSeq) },
+              }),
+            },
+          ]
+        })
+        if (retried) {
+          yield* PubSub.publish(wake, undefined)
+        }
+        return retried
+      })
+
+    const collectTombstones: FeedSyncStore['collectTombstones'] = (name) =>
+      Effect.flatMap(requireFeed(name), () =>
+        Ref.modify(state, (s): [number, State<D>] => {
+          // Every sink with a lease or a cursor counts, whatever its cursor's
+          // generation, and one with no cursor blocks collection outright: the
+          // port's doc says why a resyncing sink still needs the tombstones it has
+          // not read, and why an older generation needs no arm of its own.
+          let lowest: bigint | undefined
+          for (const sink of s.sinks.values()) {
+            if (Option.isNone(sink.cursor)) {
+              return [0, s]
+            }
+            if (lowest === undefined || sink.cursor.value.seq < lowest) {
+              lowest = sink.cursor.value.seq
+            }
+          }
+          if (lowest === undefined) {
+            return [0, s]
+          }
+          const rows = new Map(s.rows)
+          let collected = 0
+          for (const [key, stored] of s.rows) {
+            if (stored.encoded === undefined && stored.row.seq <= lowest) {
+              rows.delete(key)
+              collected += 1
+            }
+          }
+          return collected === 0 ? [0, s] : [collected, { ...s, rows }]
+        }),
+      )
+
+    const describe: FeedSyncStore['describe'] = (name) =>
+      Effect.flatMap(requireFeed(name), () =>
+        Effect.map(Ref.get(state), (s): FeedSyncDescription => {
+          let head: bigint = FEED_ORIGIN
+          for (const stored of s.rows.values()) {
+            if (stored.row.seq > head) {
+              head = stored.row.seq
+            }
+          }
+          const sinks = Array.from(
+            s.sinks.entries(),
+            ([sink, state]): SinkSyncState => ({
+              sink,
+              cursor: state.cursor,
+              lastAdvance: state.lastAdvance,
+              lease: Option.map(state.holder, (holder) => ({
+                holder,
+                expiresAt: Option.none(),
+              })),
+              parked: Array.from(state.parked.values()),
+              pending: Array.from(s.rows.values()).filter((stored) =>
+                Option.match(state.cursor, {
+                  onNone: () => stored.encoded !== undefined,
+                  onSome: (cursor) => stored.row.seq > cursor.seq,
+                }),
+              ).length,
+            }),
+          )
+          return { head: FeedSeq.make(head), sinks }
+        }),
+      )
+
+    const removeSink: FeedSyncStore['removeSink'] = (id) =>
+      Effect.flatMap(requireFeed(id.feed), () =>
+        Effect.flatten(
+          Ref.modify(
+            state,
+            (s): [Effect.Effect<boolean, LeaseHeld>, State<D>] => {
+              const current = s.sinks.get(id.sink)
+              if (current === undefined) {
+                return [Effect.succeed(false), s]
+              }
+              if (Option.isSome(current.holder)) {
+                return [Effect.fail(new LeaseHeld(id)), s]
+              }
+              const sinks = new Map(s.sinks)
+              sinks.delete(id.sink)
+              return [Effect.succeed(true), { ...s, sinks }]
+            },
+          ),
+        ),
+      )
+
+    const syncStore: FeedSyncStore = {
+      acquireLease,
+      readCursor: (id) =>
+        Effect.flatMap(requireFeed(id.feed), () =>
+          Effect.map(Ref.get(state), (s) => sinkOf(s, id.sink).cursor),
+        ),
+      casCursor,
+      readParked: (id) =>
+        Effect.flatMap(requireFeed(id.feed), () =>
+          Effect.map(Ref.get(state), (s) =>
+            Array.from(sinkOf(s, id.sink).parked.values()),
+          ),
+        ),
+      retryParked,
+      collectTombstones,
+      removeSink,
+      describe,
+    }
+
+    const result: InMemoryFeed<D> = { feed, write, source, syncStore }
+
+    internals.set(result, {
+      bind: (key) =>
+        Effect.flatMap(
+          Ref.modify(state, (s): [Option.Option<CheckpointKey>, State<D>] =>
+            Option.isNone(s.boundTo)
+              ? [Option.none(), { ...s, boundTo: Option.some(key) }]
+              : [s.boundTo, s],
+          ),
+          (already) =>
+            Option.isSome(already) && !sameKey(already.value, key)
+              ? Effect.die(
+                  new Error(
+                    `withInMemoryFeed: feed '${feed}' is already bound to projection '${already.value.projection}' (partition '${already.value.partition}'), and one feed serves one materialisation`,
+                  ),
+                )
+              : Effect.void,
+        ),
+      reset: Effect.andThen(
+        Ref.update(state, (s) => ({
+          ...s,
+          rows: new Map(),
+          generation: Generation.make(s.generation + 1n),
+        })),
+        PubSub.publish(wake, undefined),
+      ),
+    })
+
+    return result
+  })
+
+const sameKey = (left: CheckpointKey, right: CheckpointKey): boolean =>
+  left.projection === right.projection && left.partition === right.partition
+
+const moveParked = (
+  parked: ReadonlyMap<FeedKey, ParkedRecord>,
+  change: ParkedChange,
+): ReadonlyMap<FeedKey, ParkedRecord> => {
+  const next =
+    change._tag === 'Replace'
+      ? new Map<FeedKey, ParkedRecord>()
+      : new Map(parked)
+  if (change._tag === 'Amend') {
+    for (const key of change.unpark) {
+      next.delete(key)
+    }
+  }
+  for (const record of change.park) {
+    next.set(record.key, record)
+  }
+  return next
+}
+
+/**
+ * `store`, with `resetCheckpoint` for `key` replaced by one that also empties
+ * `feed` and moves its generation on, so every sink resyncs once the read model is
+ * replayed. Every other key passes through untouched, and `readCheckpoint` and
+ * `commit` are the store's own, so the result works through `forKey` exactly as
+ * the store does.
+ *
+ * Binding is checked here, at construction: a feed serves ONE materialisation,
+ * because one writer is what keeps its sequence ascending in commit order, so a
+ * feed already bound to another checkpoint key is refused, as is a durable `store`
+ * (see the module doc). Both refusals are defects.
+ */
+export const withInMemoryFeed = <D>(
+  store: ProjectionStore,
+  feed: InMemoryFeed<D>,
+  key: CheckpointKey,
+): Effect.Effect<ProjectionStore> =>
+  Effect.gen(function* () {
+    const inner = internals.get(feed)
+    if (inner === undefined) {
+      return yield* Effect.die(
+        new Error(
+          `withInMemoryFeed: feed '${feed.feed}' was not made by makeInMemoryFeed`,
+        ),
+      )
+    }
+    if (store.durability === 'durable') {
+      return yield* Effect.die(
+        new Error(
+          `withInMemoryFeed: feed '${feed.feed}' is ephemeral and the store is durable; the view would outlive the feed`,
+        ),
+      )
+    }
+    yield* inner.bind(key)
+    const decorated: ProjectionStore = {
+      durability: store.durability,
+      readCheckpoint: store.readCheckpoint,
+      commit: store.commit,
+      resetCheckpoint: (reset) =>
+        sameKey(reset, key)
+          ? Effect.andThen(
+              store.resetCheckpoint(reset),
+              Effect.uninterruptible(inner.reset),
+            )
+          : store.resetCheckpoint(reset),
+    }
+    return decorated
+  })