@kairos-es/feed 0.0.0-stage → 0.0.0

package/src/deliver.ts

@@ -0,0 +1,108 @@
+/**
+ * One batch to a sink, end to end: send, park what the sink refuses, send the rest
+ * again, then confirm. Package-internal, and the ONE place the driver talks to a
+ * sink's write path, so the steady state, a resync's load and a reconcile's
+ * deletes cannot each grow their own idea of what a refusal means.
+ *
+ * ## Why a refusal re-sends the rest rather than skipping the batch
+ *
+ * A sink that refuses one record of a batch has, by the port's contract, applied
+ * none of it — Algolia, the first sink, rejects a whole `batch` call for one
+ * oversized record. So the records it did NOT name must go again, without the
+ * refused ones, until the sink takes what is left or there is nothing left. Each
+ * round removes at least one key, so the loop ends. A sink that names no key of
+ * the batch would loop forever instead, and that is a sink bug the driver turns
+ * into `SinkHalted` rather than spin on.
+ *
+ * ## The lease, before every send
+ *
+ * `ensureHeld` runs before each `send`, not once per batch: a refusal round is a
+ * send like any other, and a worker that lost its lease between two rounds must
+ * not make the second.
+ */
+import { Array as Arr, DateTime, Effect } from 'effect'
+import type { NonEmptyReadonlyArray } from 'effect/Array'
+import {
+  type FeedUnreadable,
+  type LeaseLost,
+  SinkHalted,
+  type SinkRejected,
+  type SinkRetryable,
+} from './errors'
+import type { FeedChange } from './FeedSink'
+import type { FeedLease, ParkedRecord } from './FeedSyncStore'
+import type { FeedKey, FeedSeq } from './identifiers'
+
+/** One change to send, and the `seq` of the row it came from (for its park). */
+export interface Pending<D> {
+  readonly change: FeedChange<D>
+  readonly seq: FeedSeq
+}
+
+/** What a delivered batch came to: the keys the sink took, and the parks. */
+export interface Delivered {
+  readonly sent: ReadonlyArray<FeedKey>
+  readonly parked: ReadonlyArray<ParkedRecord>
+}
+
+/** The reason recorded for a refused key the sink gave no reason for. */
+export const UNEXPLAINED_REFUSAL = 'refused by the sink, with no reason given'
+
+/**
+ * Send `pending` through `send`, parking each refused key with the sink's reason,
+ * and confirm the receipt of whatever was finally taken. See the module doc.
+ *
+ * @internal
+ */
+export const deliver = <D, Receipt>(args: {
+  readonly send: (
+    batch: NonEmptyReadonlyArray<FeedChange<D>>,
+  ) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>
+  readonly confirm: (
+    receipt: Receipt,
+  ) => Effect.Effect<void, SinkRetryable | SinkHalted>
+  readonly lease: FeedLease
+  readonly pending: NonEmptyReadonlyArray<Pending<D>>
+}): Effect.Effect<
+  Delivered,
+  SinkRetryable | SinkHalted | LeaseLost | FeedUnreadable
+> =>
+  Effect.gen(function* () {
+    let remaining: ReadonlyArray<Pending<D>> = args.pending
+    const parked: Array<ParkedRecord> = []
+    for (;;) {
+      if (!Arr.isReadonlyArrayNonEmpty(remaining)) {
+        return { sent: [], parked }
+      }
+      const batch = Arr.map(remaining, (p) => p.change)
+      yield* args.lease.ensureHeld
+      const sent = yield* args.send(batch).pipe(
+        Effect.map((receipt) => ({ _tag: 'Taken' as const, receipt })),
+        Effect.catchTag('SinkRejected', (rejected) =>
+          Effect.succeed({ _tag: 'Refused' as const, rejected }),
+        ),
+      )
+      if (sent._tag === 'Taken') {
+        yield* args.confirm(sent.receipt)
+        return { sent: remaining.map((p) => p.change.key), parked }
+      }
+      const refused = new Set<FeedKey>(sent.rejected.keys)
+      const named = remaining.filter((p) => refused.has(p.change.key))
+      if (named.length === 0) {
+        return yield* new SinkHalted({
+          reason: `the sink refused keys that are not in the batch (${sent.rejected.keys.join(', ')})`,
+        })
+      }
+      const parkedAt = yield* DateTime.now
+      for (const p of named) {
+        parked.push({
+          key: p.change.key,
+          seq: p.seq,
+          reason:
+            sent.rejected.reasons.get(p.change.key) ?? UNEXPLAINED_REFUSAL,
+          parkedAt,
+        })
+      }
+      remaining = remaining.filter((p) => !refused.has(p.change.key))
+    }
+  })

package/src/errors.ts

@@ -0,0 +1,134 @@
+/**
+ * The change feed's error vocabulary.
+ *
+ * ## The sink's three answers
+ *
+ * A sink classifies every failure into one of three, and the driver's whole
+ * behaviour on failure follows from which. The names borrow the meaning of
+ * `@kairos-es/automation`'s classification without depending on that package: a
+ * feed is not an automation (ADR-0019), and a dependency for three class names
+ * would make it look like one.
+ *
+ * - `SinkRetryable` — the sink could not take the batch NOW (a rate limit, a 5xx, a
+ *   timeout). The cursor stays where it is and the batch is sent again after a
+ *   back-off, never sooner than `retryAfter` when the sink names one.
+ * - `SinkRejected` — the sink will never take THESE records as they stand (too
+ *   large, invalid). The named keys are parked with the sink's reasons, the rest of
+ *   the batch is sent again without them, and the cursor moves past the whole page.
+ *   A parked record is retried when its row next changes.
+ * - `SinkHalted` — something is wrong that retrying will not mend (bad
+ *   credentials, a missing index). The worker stops, and supervision decides.
+ *
+ * ## The feed's own tables
+ *
+ * `FeedWriteFailed` and `FeedUnreadable` are infrastructure faults on the feed's
+ * own durable state, split by direction: a write that did not land, a read that
+ * could not be made. They are tagged errors rather than defects for the reason the
+ * `ProjectionStore` port gives for its own: the driver's supervision has to tell
+ * "back off and try again" from "give up", and only a value can carry that.
+ *
+ * ## The worker's lease and cursor
+ *
+ * `LeaseHeld`, `LeaseLost` and `CursorSuperseded` are the three ways the
+ * single-worker guarantee shows itself: another worker holds the lease, this
+ * worker's lease went while it ran, or the cursor it expected to advance is not
+ * the one stored (a rival moved it, or the lease changed hands).
+ */
+
+import { Data, Duration } from 'effect'
+import type { NonEmptyReadonlyArray } from 'effect/Array'
+import type { FeedKey, FeedName, FeedSinkId, SinkName } from './identifiers'
+
+/** The sink cannot take the batch now: keep the cursor and back off. */
+export class SinkRetryable extends Data.TaggedError('SinkRetryable')<{
+  readonly reason: string
+  /** The least the driver waits before sending again, when the sink knows it. */
+  readonly retryAfter?: Duration.Duration
+}> {
+  override get message(): string {
+    return this.retryAfter === undefined
+      ? this.reason
+      : `${this.reason} (retry after ${Duration.format(this.retryAfter)})`
+  }
+}
+
+/** The sink refuses these records as they stand: park them, send the rest. */
+export class SinkRejected extends Data.TaggedError('SinkRejected')<{
+  readonly keys: NonEmptyReadonlyArray<FeedKey>
+  /** The sink's reason per refused key; a key absent here is parked unexplained. */
+  readonly reasons: ReadonlyMap<FeedKey, string>
+}> {
+  override get message(): string {
+    return `the sink refused ${this.keys.length} record(s): ${this.keys.join(', ')}`
+  }
+}
+
+/** Something retrying will not mend: stop the worker. */
+export class SinkHalted extends Data.TaggedError('SinkHalted')<{
+  readonly reason: string
+}> {
+  override get message(): string {
+    return this.reason
+  }
+}
+
+/** A write to the feed's own tables did not land. */
+export class FeedWriteFailed extends Data.TaggedError('FeedWriteFailed')<{
+  readonly reason: string
+  readonly cause?: unknown
+}> {
+  override get message(): string {
+    return this.reason
+  }
+}
+
+/** A read of the feed's own tables could not be made. */
+export class FeedUnreadable extends Data.TaggedError('FeedUnreadable')<{
+  readonly reason: string
+  readonly cause?: unknown
+}> {
+  override get message(): string {
+    return this.reason
+  }
+}
+
+/** Another worker holds the lease for this feed and sink. */
+export class LeaseHeld extends Data.TaggedError('LeaseHeld')<FeedSinkId> {
+  override get message(): string {
+    return `another worker holds the lease for feed '${this.feed}' and sink '${this.sink}'`
+  }
+}
+
+/** This worker's lease is no longer its own; it must stop sending at once. */
+export class LeaseLost extends Data.TaggedError('LeaseLost')<FeedSinkId> {
+  override get message(): string {
+    return `the lease for feed '${this.feed}' and sink '${this.sink}' was lost`
+  }
+}
+
+/**
+ * The cursor compare-and-set lost: the stored cursor is not the one expected, or
+ * the lease is not held by the worker that asked. Nothing the call carried landed.
+ */
+export class CursorSuperseded extends Data.TaggedError(
+  'CursorSuperseded',
+)<FeedSinkId> {
+  override get message(): string {
+    return `the cursor for feed '${this.feed}' and sink '${this.sink}' was superseded`
+  }
+}
+
+/**
+ * The sync worker gave up: it restarted `restarts` times in a row without moving
+ * its cursor once. `reason` is the failure that ended the last attempt.
+ */
+export class FeedSyncStalled extends Data.TaggedError('FeedSyncStalled')<{
+  readonly feed: FeedName
+  readonly sink: SinkName
+  readonly restarts: number
+  readonly reason: unknown
+}> {
+  override get message(): string {
+    return `the sync of feed '${this.feed}' to sink '${this.sink}' made no progress in ${this.restarts} consecutive restarts`
+  }
+}

package/src/identifiers.ts

@@ -0,0 +1,92 @@
+/**
+ * The change feed's branded identifiers.
+ *
+ * Every one of them is a branded `Schema`, as every domain identifier in this
+ * repository is, so a raw string or `bigint` cannot stand in for one by accident.
+ * Two of them are worth a word, because they look like things the read side
+ * already has and are deliberately not those things.
+ *
+ * - `FeedSeq` is NOT a `Position`. A `Position` is a place in the global event log,
+ *   and one event may fan out into many feed rows (a teacher renamed, fifty courses
+ *   touched). Give those fifty rows the event's position as their cursor and they
+ *   all share one value, so a page boundary that falls among them loses the rest of
+ *   them: the next page asks for rows strictly after a value the lost ones also
+ *   carry. A feed-local sequence gives every row a cursor value of its own. The
+ *   event position is still kept on each row, as `eventPosition`, for the one
+ *   question it answers — "is the sink at least as fresh as P?" — and is never a
+ *   cursor.
+ * - `Generation` is NOT a counter a caller reads meaning into. It is compared for
+ *   EQUALITY only: a sink whose cursor was saved at a generation other than the
+ *   feed's current one resyncs. That is what lets the in-memory feed draw a fresh
+ *   generation from `Random` on every boot while a durable feed counts upwards, and
+ *   both satisfy the one rule the driver needs.
+ */
+import { Schema } from 'effect'
+
+/**
+ * The name of one change feed. A feed is bound to ONE materialisation, because a
+ * feed sequence taken inside a commit is ascending in commit order only while one
+ * transaction at a time writes that feed — which the read side gives per
+ * materialisation and no wider.
+ */
+export const FeedName = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('FeedName'),
+)
+export type FeedName = typeof FeedName.Type
+
+/** A record's key within a feed, and the sink's object id for that record. */
+export const FeedKey = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('FeedKey'),
+)
+export type FeedKey = typeof FeedKey.Type
+
+/**
+ * The feed-local cursor: unique, and ascending in commit order. Compared, never
+ * arithmetic'd, and never assumed contiguous — a sequence shared by several feeds
+ * leaves gaps in each, and a coalesced row gives up its old value when it moves.
+ */
+export const FeedSeq = Schema.BigInt.pipe(
+  Schema.check(Schema.isGreaterThanOrEqualToBigInt(0n)),
+  Schema.brand('FeedSeq'),
+)
+export type FeedSeq = typeof FeedSeq.Type
+
+/** The sentinel feed sequence meaning "before the first row". */
+export const FEED_ORIGIN: FeedSeq = FeedSeq.make(0n)
+
+/**
+ * The feed's generation: it moves on every reset of the materialisation the feed
+ * is bound to, and on every boot of an ephemeral feed. Compared for equality only;
+ * see the module doc.
+ */
+export const Generation = Schema.BigInt.pipe(
+  Schema.check(Schema.isGreaterThanOrEqualToBigInt(0n)),
+  Schema.brand('Generation'),
+)
+export type Generation = typeof Generation.Type
+
+/** The name of one sink. A feed keeps one cursor per sink. */
+export const SinkName = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('SinkName'),
+)
+export type SinkName = typeof SinkName.Type
+
+/**
+ * Who holds a sync worker's lease. Issued by the `FeedSyncStore` when the lease is
+ * taken, and checked again by every cursor compare-and-set, so a worker that has
+ * lost its lease cannot move the cursor even if it is still running.
+ */
+export const LeaseHolder = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('LeaseHolder'),
+)
+export type LeaseHolder = typeof LeaseHolder.Type
+
+/** One sync worker's identity: the feed it reads and the sink it writes. */
+export interface FeedSinkId {
+  readonly feed: FeedName
+  readonly sink: SinkName
+}