@kairos-es/read 0.0.0

package/src/inMemoryProjectionStore.ts

@@ -0,0 +1,163 @@
+/**
+ * The in-memory `ProjectionStore`: a `Ref` of checkpoints plus the exclusive
+ * critical section that makes the guarded advance atomic.
+ *
+ * This is NOT merely a test double. It is load-bearing for the in-process
+ * supervision restart, where the subscription dies (the event store's poll retry
+ * exhausted) but the process and its in-memory view survive: the restart must
+ * resume from the checkpoint rather than replay from `ORIGIN` into an
+ * already-populated view, and an ephemeral checkpoint stored beside an ephemeral
+ * view is exactly what makes that resume correct. It is also the whole
+ * all-in-memory permutation — tests, prototypes, client-side use — and the
+ * durable-log-with-in-memory-view mode's view store (dev, admin and in-flux
+ * slices, rebuilt from `ORIGIN` each boot).
+ *
+ * A factory `Effect`, deliberately not a `Layer`: the view store is selected PER
+ * MATERIALISATION — so a slice may start in memory and GRADUATE to Postgres, which
+ * `ReadModel.store` owns the claim about — and it is therefore a value handed to a
+ * read model, not a context tag that one store could claim process-wide.
+ *
+ * That selection is a FLOOR rather than a ceiling, and this store is on the common
+ * side of it: the two variants above are also available AT ONCE, an in-memory
+ * materialisation of a slice record beside a durable one from the SAME record —
+ * a Postgres table somebody queries with SQL, and an in-process view of the same
+ * slices holding one aggregate's hot figures with no round trip to reach them. That
+ * is `runProjections`, and every call of this factory is an independent store, so
+ * two materialisations wired that way share nothing but their slices. What they do
+ * not share is their FOLDS: each `apply` re-expresses one, so agreement between
+ * their figures is something a test demonstrates rather than something the wiring
+ * gives.
+ *
+ * The critical-section design mirrors `core`'s in-memory event store: one
+ * `Effect.makeSemaphore(1)` serialises the guarded advance, exactly as that store
+ * serialises check-then-append, and for the same reason — the check and the write
+ * must not be separable, and that atomicity IS the concurrency guarantee.
+ */
+import { ORIGIN, type Position } from '@kairos-es/core'
+import { Effect, Ref } from 'effect'
+import type { CheckpointKey, ProjectionStore } from './ProjectionStore'
+import { CheckpointSuperseded } from './ProjectionStore'
+
+/**
+ * Flatten a `CheckpointKey` into a map key.
+ *
+ * The separator is NUL. A `ProjectionId`/`PartitionId` is any non-empty string,
+ * so a printable separator (`:`, `/`, `|`) could be forged inside either half and
+ * two distinct keys would collide onto one checkpoint — one read model silently
+ * reading and advancing another's cursor. NUL cannot appear in a name any human
+ * or config file produces, so the flattening stays injective in practice without
+ * narrowing the id grammar.
+ *
+ * The other route to that same failure is a WIRING rather than a forged id: two
+ * materialisations handed this one store under equal keys, which would be two
+ * runners over one cursor. That one is caught before anything is forked, by the
+ * set-level collision rung of the construction gate in `projectionWiringFault.ts`,
+ * and its sentence names the two fixes. Nothing here can catch it — a store sees
+ * keys, never who is holding it.
+ */
+const mapKey = (key: CheckpointKey): string =>
+  `${key.projection}\u0000${key.partition}`
+
+/**
+ * Build an in-memory `ProjectionStore`.
+ *
+ * Every call is an independent store with its own checkpoints and its own
+ * semaphore, so two read models wired to two calls of this factory cannot
+ * interfere — and a test needing a fresh store just calls it again.
+ */
+export const makeInMemoryProjectionStore: Effect.Effect<ProjectionStore> =
+  Effect.gen(function* () {
+    const checkpoints = yield* Ref.make<ReadonlyMap<string, Position>>(
+      new Map(),
+    )
+    const mutex = yield* Effect.makeSemaphore(1)
+
+    /** The stored position, or `ORIGIN` for a key never committed. */
+    const positionOf = (
+      stored: ReadonlyMap<string, Position>,
+      key: CheckpointKey,
+    ): Position => stored.get(mapKey(key)) ?? ORIGIN
+
+    const readCheckpoint = (
+      key: CheckpointKey,
+    ): Effect.Effect<Position, never> =>
+      // No lock needed: `Ref.get` is a single atomic read, and the guard that
+      // actually protects a commit is re-checked INSIDE the critical section
+      // below — a value read here is only ever an expectation to be guarded on,
+      // never a licence to write.
+      Effect.map(Ref.get(checkpoints), (stored) => positionOf(stored, key))
+
+    const commit = <E, R>(args: {
+      readonly key: CheckpointKey
+      readonly expected: Position
+      readonly next: Position
+      readonly viewWrites: Effect.Effect<void, E, R>
+    }): Effect.Effect<void, E | CheckpointSuperseded, R> =>
+      // The permit is acquired INTERRUPTIBLY (a fibre waiting its turn can still
+      // be torn down), and only the section itself is uninterruptible.
+      mutex.withPermits(1)(
+        Effect.uninterruptible(
+          Effect.gen(function* () {
+            const stored = positionOf(yield* Ref.get(checkpoints), args.key)
+
+            // The guard is checked FIRST, before the view writes run. There is no
+            // transaction here to roll anything back, so the ONLY way a lost
+            // guard can leave the view untouched is to lose before touching it.
+            // Check-then-act is safe despite `viewWrites` suspending in between,
+            // because the section is exclusive: no other fibre can observe or
+            // advance this checkpoint until the permit is released, so the value
+            // read here cannot go stale within the section.
+            //
+            // On a lost guard `viewWrites` was NEVER RUN, which is observably
+            // identical to the SQL store's rollback — the port's abort contract
+            // is met by omission rather than by undo. (The one case omission
+            // cannot cover is a `viewWrites` that fails part-way, which is why
+            // the port requires a non-transactional store's per-batch write to be
+            // a single atomic effect; see `ProjectionStore` and `foldIntoRef`.)
+            if (stored !== args.expected) {
+              return yield* new CheckpointSuperseded({
+                key: args.key,
+                expected: args.expected,
+              })
+            }
+
+            yield* args.viewWrites
+
+            // Uninterruptibility earns its keep here: an interrupt landing
+            // between the view writes and this advance would commit the batch's
+            // effects with the cursor left behind, and the next run would apply
+            // the same batch a second time. Copy-on-write so a concurrent reader
+            // holding the previous map sees a consistent snapshot.
+            yield* Ref.update(checkpoints, (current) =>
+              new Map(current).set(mapKey(args.key), args.next),
+            )
+          }),
+        ),
+      )
+
+    const resetCheckpoint = (key: CheckpointKey): Effect.Effect<void> =>
+      // Under the same permit as `commit`, so a reset can never interleave with a
+      // guarded advance and leave the map half-updated relative to the view.
+      // Deleting rather than storing `ORIGIN` keeps "never committed" and "reset"
+      // one state, which is what the port promises the read-back is.
+      mutex.withPermits(1)(
+        Ref.update(checkpoints, (current) => {
+          const next = new Map(current)
+          next.delete(mapKey(key))
+          return next
+        }),
+      )
+
+    // Annotated at the definition site rather than only through the exported
+    // signature, per the rule on `ProjectionStore` itself.
+    const store: ProjectionStore = {
+      // Dies with the process, which is the R2 input: an ephemeral view is legal
+      // over either an ephemeral or a durable log (it just re-catches-up), so
+      // this value never trips the durability-ordering check.
+      durability: 'ephemeral',
+      readCheckpoint,
+      commit,
+      resetCheckpoint,
+    }
+    return store
+  })

package/src/index.ts

@@ -0,0 +1,218 @@
+/**
+ * @kairos-es/read — the read side: a query-driven projection runner over the
+ * store's `subscribe` contract, and a scalar checkpoint committed atomically with
+ * the read model's own writes (ADR-0007).
+ *
+ * Platform-neutral by design: peers are `effect` plus `@kairos-es/core` and
+ * `@kairos-es/codec` only — the same peer shape as the codec — so the
+ * all-in-memory permutation, a property of one MATERIALISATION's (log, view)
+ * pairing and including client-side use, needs no database
+ * dependency. Every dependency-bearing backend is its own package, which is why
+ * the SQL `ProjectionStore` lives in `@kairos-es/read-postgres` rather than behind
+ * a sub-entry here: a sub-entry shares one dependency surface, whereas a separate
+ * package isolates a new one (`@effect/sql`).
+ *
+ * ## Where each rationale lives
+ *
+ * This module re-exports; it does not argue. Every argument the read side rests
+ * on is written ONCE, at the definition site a reader would reach for anyway,
+ * because a rationale kept in N places is a rationale a correction has to find N
+ * times. What follows is the index, R1 and R2 included: each of those two has a
+ * module that owns it, so each gets a pointer here like everything else. R3 is the
+ * one exception, and the last section says why.
+ *
+ * - **The log seam is `DcbEventStore.subscribe`, and it belongs entirely to the
+ *   event store.** `store-postgres` owns LISTEN/NOTIFY as a wake source over
+ *   `core`'s shared poll machine, `store-sqlite` owns no wake source and rides
+ *   that poll alone, and `core`'s in-memory layer owns its append `PubSub` as a
+ *   wake source over that same machine; all three sit behind the
+ *   identical `subscribe(query, after)` contract, so a projection never learns
+ *   which it is running against. Nothing in this package adds to that contract —
+ *   the one claim below that no other module is placed to make.
+ * - **The view seam is `ProjectionStore`** — one port rather than two, `commit`
+ *   taking the view writes as an ARGUMENT, `CheckpointSuperseded` as an error
+ *   rather than a returned outcome, and `ProjectionStoreError` inverting the store
+ *   contract's defect rule. `ProjectionStore.ts`'s module doc. The multi-key port
+ *   against the keyed façade a read model is handed is on `KeyedProjectionStore`
+ *   and on `forKey` in the same module; the single-atomic-effect requirement a
+ *   NON-TRANSACTIONAL store's caller must honour is on the `ProjectionStore`
+ *   interface itself, and `foldIntoRef.ts` is that shape.
+ * - **R1, atomicity** — a read model's checkpoint lives WITH its view, in a store
+ *   where the two can be committed atomically, which is what `commit` is. The
+ *   argument is on the `ProjectionStore` interface in `ProjectionStore.ts` (R1 is
+ *   what that interface IS), together with what a view offering no transaction at
+ *   all falls back to: at-least-once, with POSITION-KEYED idempotent writes, a
+ *   recommendation because the port cannot check it.
+ * - **R2, durability ordering** — checkpoint durability must not EXCEED event-log
+ *   durability, so a DURABLE view over an EPHEMERAL log is a defect rejected at
+ *   construction rather than a view that silently freezes. The view store declares
+ *   its half on the `ProjectionStore` it implements; the log's half, the silent
+ *   failure the rule prevents, and why the log's class is ONE context tag beside
+ *   the store layer rather than a field per read model are all in
+ *   `EventLogDurability.ts`.
+ * - **How these errors RENDER themselves** — why `ProjectionStoreError` derives its
+ *   `message` from `reason` and `CheckpointSuperseded` derives one from `expected`
+ *   (including the `bigint` argument that once kept the field empty and why it was a
+ *   category error), how far filling that field reaches (every reader of the error,
+ *   not one log line) and what it leaves untouched: the two classes in
+ *   `ProjectionStore.ts`. `PipelineDied` in `ProjectionRunner.ts` and
+ *   `ProjectionStalled` in `superviseOnProgress.ts` are the other two and point
+ *   there.
+ * - **Why the view store is a VALUE and not a `Context.Tag`**, and what a slice's
+ *   graduation from memory to Postgres does and does not change: `ReadModel.store`
+ *   in `runProjection.ts`. It is demonstrated rather than asserted, by
+ *   `examples/course-subscriptions`'s `src/courseRosterSql.ts` and by
+ *   `@kairos-es/read-postgres`'s `test/courseRosterGraduation.test.ts`.
+ * - **The in-memory implementation, and why it is not merely a test double:**
+ *   `inMemoryProjectionStore.ts`.
+ * - **The runner** — the pipeline stage by stage, and why supervision lives next
+ *   door: `ProjectionRunner.ts`, which holds the SUBSTRATE both entry points sit on
+ *   and no entry point of its own. Its two option sets are documented on
+ *   `ProjectionPipelineOptions` there and on `ProjectionSupervisionOptions` in
+ *   `superviseOnProgress.ts`; `ProjectionRunnerOptions` says why their union is
+ *   FLAT and why every figure is bounded rather than merely documented.
+ * - **ONE materialisation of one read model** — `runProjection` and the
+ *   `projectionLayer` over it: `runProjection.ts`, which also says why the singular
+ *   and the plural entry points are two modules over one substrate and why neither
+ *   calls the other.
+ * - **N MATERIALISATIONS of one read model** — one slice record and one
+ *   `ProjectionId` for the call, one view store and one runner each, which is what
+ *   `runProjections` is: `runProjections.ts`. It carries the guarantee at exactly
+ *   its strength (one query, one decode, N applies — never "they cannot drift",
+ *   since a SQL `apply` re-implements the fold and agreement between materialised
+ *   figures is empirical), why the guarantee is construction-scoped, why each entry
+ *   hands over an UNBOUND `ProjectionStore` while the identity is named once for the
+ *   call, why the tuning and the two supervision hooks are PER ENTRY, and why there
+ *   is no `projectionsLayer`. The wiring that is its OWN — two materialisations
+ *   sharing one store under one resolved `CheckpointKey` — is the set-level rung of
+ *   the construction gate below, with the two bounds it deliberately does not reach;
+ *   the gate's other four run in the same pass — two per entry, two once over the
+ *   shared query and slice record — so no refused call forks anything, whichever
+ *   rung and whichever entry it was.
+ * - **The supervisor** — retry while an externally observed signal moves, give up
+ *   when it stops; why a `CheckpointSuperseded` loser resynchronises instead of
+ *   tripping the breaker; why the two hooks are the primary observability channel
+ *   and the log lines a thin lossy default; why it holds no rendering of a
+ *   caller's `E` at all, the label being the fault's own `message`; and the
+ *   wall-clock arithmetic behind the no-progress budget: `superviseOnProgress.ts`.
+ *   `superviseOnProgress` ITSELF is package-internal and deliberately not exported
+ *   below — it is the projection runner's supervisor, not a general combinator, and
+ *   that module records both why and what generalising it would cost. Three
+ *   symbols from it are exported, each because the PUBLIC surface names it:
+ *   `ProjectionStalled` is `ProjectionRunner.fibre`'s error channel,
+ *   `ProjectionRunnerOptions` extends `ProjectionSupervisionOptions`, and tuning
+ *   the no-progress budget means calling `MaxNoProgressRestarts.make(n)`.
+ * - **One writer per MATERIALISATION, and why one read model's stream is never
+ *   sharded:** ADR-0007, with the partition dimension's own half on `PartitionId`.
+ *   Per materialisation rather than per read model, because a read model may have
+ *   several — N of them are N single writers, never shards of one stream.
+ *   Where apply THROUGHPUT rather than isolation is the constraint, ADR-0007 also
+ *   carries the escape hatch — coalesce a batch per target row inside the one
+ *   commit — and there is no library symbol for it, because it is a fold the read
+ *   model performs over its own batch.
+ * - **The tagless-slice rule** — every read-model slice carries at least one tag
+ *   (a `system:…` tag for a genuinely global slice), enforced at construction by
+ *   two checks that cannot stand in for each other, since a lone tagless slice
+ *   composes to a query the store WOULD serve: both are rungs of the wiring gate
+ *   `projectionWiringFault.ts`, which argues the rules and the order they are judged
+ *   in, and which is consulted from the one `prepareProjection` phase both runner
+ *   entry points run before either forks anything.
+ * - **Running a read model in a test** — the `@kairos-es/read/testing` subpath
+ *   (`testing.ts`), which also states its boundary against
+ *   `@kairos-es/projection-store-contract-tests`.
+ * - **The narrative version of all of it** is this package's `README.md`; the
+ *   decision it implements is ADR-0007.
+ *
+ * ## R3, the legal permutations, and several materialisations of one read model
+ *
+ * The one thing this file ARGUES rather than indexes. R1 and R2 each have a module
+ * that owns them, pointed at above; R3 has none, because it is a property of how
+ * read models are composed into a SUBSCRIPTION and no single module holds that.
+ *
+ * **R3, composition bound.** Read models sharing one subscription share one cursor
+ * and one checkpoint, hence by R1 one view store and one durability class. So
+ * per-read-model view selection means one runner per view-store GROUP, not one per
+ * read model — and rebuilding one member of a composed bundle means rebuilding the
+ * bundle (or running a transient dedicated runner from `ORIGIN`). Composing is
+ * still the default, because the NOTIFY channel is per store, not per query.
+ *
+ * **What R3 does NOT say.** It does not bound how many times ONE read model may be
+ * materialised. R3 is about sharing a SUBSCRIPTION, and several materialisations of
+ * one read model deliberately do not share one: they pay N subscriptions, N cursors
+ * and N checkpoints precisely BECAUSE R1 and R2 forbid the sharing — one atomic
+ * `commit` cannot span two stores, and one checkpoint cannot carry two durability
+ * classes. So per-read-model view-store selection is a FLOOR rather than a ceiling.
+ * One slice record may feed a durable Postgres materialisation and an ephemeral
+ * in-memory one AT ONCE, wired by `runProjections`, which holds one `slices` value
+ * and one `ProjectionId` across them so their query and their decode cannot diverge.
+ * What it does not hold is the FOLDS — each `apply` re-expresses the fold against
+ * its own store, and a SQL one writing rows is not `foldIntoRef` — so agreement
+ * between two materialised figures is EMPIRICAL, something a test demonstrates, and
+ * never something the wiring gives you. The runners do not contend either: each
+ * owns its own checkpoint in its own store, so the guarded compare-and-set
+ * arbitrates with no lease, no lock and no coordinator.
+ *
+ * That leaves exactly THREE legal (log, view) permutations: all-in-memory (tests,
+ * prototypes, client-side); a durable log with an in-memory view (dev, low-traffic
+ * and admin views, in-flux slices — rebuilt from `ORIGIN` each boot, which is
+ * affordable at the event counts that are its remit and a reason to graduate a
+ * slice once it bakes); and a durable log with a durable view, the production
+ * default. Each is a property of ONE MATERIALISATION's pairing. Mixing is a
+ * DEPLOYMENT property rather than a fourth permutation: one deployment can run the
+ * second and third side by side against the same log — for two different read
+ * models, which is the case R3 bounds when they try to share a subscription, or for
+ * two materialisations of ONE read model, which R3 never reaches because they never
+ * share one.
+ *
+ * **A third KIND of materialisation sits outside all of this.** A read-time
+ * materialisation — `modelAtHead`, in `@kairos-es/codec` — folds the log per query
+ * and has no table, no checkpoint, no runner and no view store, so no permutation,
+ * no rung of the construction gate and no durability rule applies to it, R2 being
+ * VACUOUS where there is no checkpoint whose durability could outlive a log. It is
+ * consistent as of the `head` its read returned, which trails nothing, so it is the
+ * MOST current view of a slice record and the maintained table beside it is the
+ * stale one. Where a read model is small enough to fold at query time, that is a
+ * reason not to maintain it at all.
+ */
+export {
+  DurableEventLog,
+  EphemeralEventLog,
+  EventLogDurability,
+} from './EventLogDurability'
+export { foldIntoRef } from './foldIntoRef'
+export { makeInMemoryProjectionStore } from './inMemoryProjectionStore'
+export {
+  BatchSize,
+  PipelineDied,
+  type ProjectionPipelineOptions,
+  type ProjectionRunner,
+  type ProjectionRunnerOptions,
+} from './ProjectionRunner'
+export {
+  type CheckpointKey,
+  CheckpointSuperseded,
+  checkpointKey,
+  DEFAULT_PARTITION,
+  type Durability,
+  forKey,
+  type KeyedProjectionStore,
+  PartitionId,
+  ProjectionId,
+  type ProjectionStore,
+  ProjectionStoreError,
+} from './ProjectionStore'
+export {
+  projectionLayer,
+  type ReadModel,
+  runProjection,
+} from './runProjection'
+export {
+  type Materialisation,
+  type MaterialisedProjection,
+  runProjections,
+} from './runProjections'
+export {
+  MaxNoProgressRestarts,
+  ProjectionStalled,
+  type ProjectionSupervisionOptions,
+} from './superviseOnProgress'