@smthrs/journal 0.0.0-stage → 1.0.0-rc.3

package/src/JournalGeneration.ts

@@ -0,0 +1,77 @@
+/**
+ * Durable generations distinguish histories that reuse journal sequences.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Installs the generation table idempotently. Both the SQL journal and time
+ * travel initialize it, including databases whose migration ladder already
+ * passed the journal's reserved block. A read-only client installs nothing.
+ *
+ * @category migrations
+ * @since 1.0.0-rc.0
+ */
+export const initialize = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  // A read-only observer reads the schema it finds and installs nothing.
+  if (Dialect.isReadOnly(sql)) return
+  yield* sql`
+    CREATE TABLE IF NOT EXISTS flows_journal_generations (
+      run_id TEXT PRIMARY KEY NOT NULL CHECK (length(run_id) > 0),
+      generation ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`generation`)
+  } AND generation > 0 AND generation <= 9007199254740991),
+      after_seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`after_seq`)
+  } AND after_seq >= -1 AND after_seq <= 9007199254740991)
+    )
+  `
+})
+
+const journals = new WeakMap<SqlClient.SqlClient, Set<(runIds: ReadonlyArray<string>) => void>>()
+
+/**
+ * Registers a live journal's cache invalidation until its scope closes.
+ * Multiple journals over the same SQL client must observe the same rewind.
+ *
+ * @category lifecycle
+ * @since 1.0.0-rc.0
+ */
+export const onTruncate = (forget: (runIds: ReadonlyArray<string>) => void) =>
+  Effect.gen(function*() {
+    const sql = yield* SqlClient.SqlClient
+    let listeners = journals.get(sql)
+    if (listeners === undefined) {
+      listeners = new Set()
+      journals.set(sql, listeners)
+    }
+    const registered = listeners
+    yield* Effect.acquireRelease(
+      Effect.sync(() => registered.add(forget)),
+      () =>
+        Effect.sync(() => {
+          registered.delete(forget)
+        })
+    )
+  })
+
+/**
+ * Forgets cached identities and allocation floors after a committed truncation.
+ * The rewinder must quiesce producers and flush pending admissions first.
+ * Database rows remain authoritative; this operation changes no durable data.
+ *
+ * @category lifecycle
+ * @since 1.0.0-rc.0
+ */
+export const forget = (runIds: ReadonlyArray<string>) =>
+  Effect.gen(function*() {
+    const sql = yield* SqlClient.SqlClient
+    yield* Effect.sync(() => {
+      for (const invalidate of journals.get(sql) ?? []) invalidate(runIds)
+    })
+  })

package/src/JournalMetrics.ts

@@ -0,0 +1,89 @@
+/**
+ * Standard metric definitions for the journal write path.
+ *
+ * This module only defines the metric handles, following the shape of Effect's
+ * `ClusterMetrics`. `SqlJournal` updates them as emission receipts are
+ * produced, so the counters measure admissions on the hot path rather than
+ * rows read back. No exporter ships in this package; provide one, for
+ * example `@smthrs/observability`, and these counters appear in it.
+ *
+ * @since 0.1.0
+ */
+
+import * as Metric from "effect/Metric"
+
+/**
+ * Counter over journal emissions, dimensioned by `channel` (`durable` or
+ * `lossy`) and `receipt` (`accepted`, `duplicate`, or `dropped`).
+ *
+ * **Details**
+ *
+ * A `durable` emission counts when its receipt is produced, which is after the
+ * write transaction returned; under `Journal.transact` that is still inside
+ * the caller's transaction, so a receipt that later rolls back with the
+ * enclosing transaction has already counted. The counter is throughput
+ * evidence, not commit evidence: the journal rows themselves are the latter.
+ *
+ * @category metrics
+ * @since 0.1.0
+ */
+export const writes = Metric.counter("flows_journal_writes", {
+  description: "Journal emissions by channel and receipt"
+})
+
+/**
+ * `writes` views for the durable channel, keyed by the receipt tag
+ * `Journal.emitDurable` resolves to.
+ *
+ * @category metrics
+ * @since 0.1.0
+ */
+export const durable: {
+  readonly [Tag in "Accepted" | "Duplicate"]: Metric.Metric<number, Metric.CounterState<number>>
+} = {
+  Accepted: Metric.withAttributes(writes, { channel: "durable", receipt: "accepted" }),
+  Duplicate: Metric.withAttributes(writes, { channel: "durable", receipt: "duplicate" })
+}
+
+/**
+ * `writes` views for the lossy channel, keyed by the receipt tag
+ * `Journal.emitLossy` resolves to.
+ *
+ * @category metrics
+ * @since 0.1.0
+ */
+export const lossy: {
+  readonly [Tag in "Accepted" | "Duplicate" | "Dropped"]: Metric.Metric<number, Metric.CounterState<number>>
+} = {
+  Accepted: Metric.withAttributes(writes, { channel: "lossy", receipt: "accepted" }),
+  Duplicate: Metric.withAttributes(writes, { channel: "lossy", receipt: "duplicate" }),
+  Dropped: Metric.withAttributes(writes, { channel: "lossy", receipt: "dropped" })
+}
+
+/**
+ * Counter over lossy entries the writer admitted and then failed to commit,
+ * dimensioned by `channel` (always `lossy`) and the `code` of the
+ * `JournalError` that lost them.
+ *
+ * **Details**
+ *
+ * A lossy producer rarely calls `flush`, so the failure `flush` reports is not
+ * where an operator learns about a lost batch. This counter moves by the number
+ * of entries lost the moment the writer gives them up, whether or not anyone
+ * is waiting on them.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.1
+ */
+export const lostEntries = Metric.counter("flows_journal_lost", {
+  description: "Admitted lossy journal entries the writer failed to commit"
+})
+
+/**
+ * The {@link lostEntries} view for one `JournalError` code.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.1
+ */
+export const lost = (code: string): Metric.Metric<number, Metric.CounterState<number>> =>
+  Metric.withAttributes(lostEntries, { channel: "lossy", code })

package/src/Migrations.ts

@@ -0,0 +1,62 @@
+/**
+ * Journal schema migrations.
+ *
+ * The journal owns four tables: `flows_journal_events` and its event-type
+ * index in `0001_initial`, `flows_journal_checkpoints` in
+ * `0002_checkpoints`, the bounded startup ordering index in
+ * `0003_startup_index`, compacted producer identities in `0004_dedup`, the
+ * per-run exact event-type cursor index in `0005_run_event_type`, and the
+ * `SqlConsensus` lease table `flows_consensus_leases` in `0006_consensus`. Run
+ * and attempt state migrate from `@smthrs/run-store`, the step cache from
+ * `@smthrs/step-cache`, and the durable deferred/clock tables from
+ * `@smthrs/engine-store`; an application composes those sets with this one
+ * through `@smthrs/database`'s `Migrations.layer`.
+ *
+ * @since 0.1.0
+ */
+
+import * as DatabaseMigrations from "@smthrs/database/Migrations"
+import * as Layer from "effect/Layer"
+import { initial } from "./migrations/0001_initial.ts"
+import { checkpoints } from "./migrations/0002_checkpoints.ts"
+import { startupIndex } from "./migrations/0003_startup_index.ts"
+import { dedup } from "./migrations/0004_dedup.ts"
+import { runEventType } from "./migrations/0005_run_event_type.ts"
+import { consensus } from "./migrations/0006_consensus.ts"
+
+/**
+ * The journal's namespaced migration set, for composition with the other
+ * storage packages.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const set: DatabaseMigrations.MigrationSet = {
+  namespace: "journal",
+  idOffset: 0,
+  migrations: {
+    "0001_initial": initial,
+    "0002_checkpoints": checkpoints,
+    "0003_startup_index": startupIndex,
+    "0004_dedup": dedup,
+    "0005_run_event_type": runEventType,
+    "0006_consensus": consensus
+  }
+}
+
+/**
+ * Creates the journal's authoritative durable schema.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const run = DatabaseMigrations.run([set])
+
+/**
+ * Layer that runs journal migrations before exposing the database to journal
+ * services.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = Layer.effectDiscard(run)

package/src/OwnerId.ts

@@ -0,0 +1,42 @@
+/**
+ * The fencing token the journal accepts on durable appends.
+ *
+ * `OwnerId` lives here rather than with the ownership *arbitration* in
+ * `@smthrs/run-store` because the journal is what it fences: `emitDurable`
+ * takes an owner and refuses the append when the persisted fence has moved on.
+ * The journal therefore defines the token, and the run store (which stores it
+ * on runs and arbitrates who holds it) imports it.
+ *
+ * @since 0.1.0
+ */
+
+import * as Schema from "effect/Schema"
+
+/**
+ * A process identity scoped to a host and a unique ownership nonce.
+ *
+ * `pid` is a real operating-system process id, so the schema states that: a
+ * non-negative integer. A fractional, `NaN`, or negative `pid` is a caller
+ * bug, and the fenced methods reject it as `invalid_event` rather than
+ * degrading it into `fence_lost`, which would read as "someone else owns this
+ * run" and send the caller looking for a race that never happened.
+ *
+ * `hostId` and `nonce` stay plain strings. An empty one is a legal value that
+ * simply matches no persisted owner, which is exactly `fence_lost`.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export const OwnerId = Schema.Struct({
+  hostId: Schema.String,
+  pid: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
+  nonce: Schema.String
+})
+
+/**
+ * A process identity scoped to a host and a unique ownership nonce.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type OwnerId = typeof OwnerId.Type

package/src/Projection.ts

@@ -0,0 +1,31 @@
+/**
+ * Reproducible journal projections.
+ *
+ * @since 0.1.0
+ */
+
+import type * as Effect from "effect/Effect"
+import type { Entry } from "./JournalEvent.ts"
+
+/**
+ * An effectful fold over committed journal entries.
+ *
+ * Projections have no independent durable state. Replaying the same entries
+ * through the same reducer must reproduce the same result.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Projection<S, E = never, R = never> {
+  readonly name: string
+  readonly initial: S
+  readonly reduce: (state: S, entry: Entry) => Effect.Effect<S, E, R>
+}
+
+/**
+ * Constructs a reproducible journal projection.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = <S, E = never, R = never>(projection: Projection<S, E, R>): Projection<S, E, R> => projection