@smthrs/journal 0.0.0-stage → 1.0.0-rc.3

package/src/StepFact.ts

@@ -0,0 +1,69 @@
+/** Durable observations owned by an agent checkpoint action.
+ * @since 1.0.0
+ */
+
+import * as Context from "effect/Context"
+import * as Schema from "effect/Schema"
+
+const nonnegative = Schema.Int.check(
+  Schema.isGreaterThanOrEqualTo(0),
+  Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER)
+)
+const positive = Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER))
+
+/** Invocation and retry coordinates remain distinct across replay.
+ * @category schemas
+ * @since 1.0.0
+ */
+export const Step = Schema.Struct({
+  stepId: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)),
+  executionId: Schema.String,
+  action: Schema.String,
+  attempt: positive,
+  ask: Schema.Union([nonnegative, Schema.Literal("repair")]),
+  retry: positive,
+  scope: Schema.String
+})
+
+/** Decoded invocation coordinates.
+ * @category models
+ * @since 1.0.0
+ */
+export type Step = typeof Step.Type
+
+/** The first recorded observation, including its original generation and time.
+ * @category schemas
+ * @since 1.0.0
+ */
+export const Fact = Schema.Struct({
+  version: Schema.Literal(1),
+  step: Step,
+  generation: nonnegative,
+  frame: Schema.Int.check(Schema.isGreaterThanOrEqualTo(-1)),
+  ordinal: nonnegative,
+  cell: Schema.String,
+  at: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)),
+  eventType: Schema.String.check(Schema.isPattern(/^control\.agent\.[a-z-]+$/)),
+  sourceSequence: nonnegative,
+  payload: Schema.Json
+})
+
+/** Decoded checkpoint observation.
+ * @category models
+ * @since 1.0.0
+ */
+export type Fact = typeof Fact.Type
+
+/** Marks a checkpoint whose saved success is the fact to publish.
+ * @category services
+ * @since 1.0.0
+ */
+// The annotation is a marker with no fields; its presence is the entire contract.
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+export class Annotation extends Context.Service<Annotation, {}>()("@smthrs/journal/StepFact/Annotation") {}
+
+/** Native outbox event, committed with its owning checkpoint.
+ * @category events
+ * @since 1.0.0
+ */
+export const eventType = "flows.harness.step-fact.v1"

package/src/index.ts

@@ -0,0 +1,109 @@
+/**
+ * Durable journal services: the immutable event history and nothing else.
+ *
+ * Run and attempt state live in `@smthrs/run-store`, sealed step results in
+ * `@smthrs/step-cache`, and the durable deferred/clock tables in
+ * `@smthrs/engine-store`.
+ *
+ * This entry point is browser-bundleable: every service here is written
+ * against the driver-neutral `@smthrs/database` contract. The test doubles are
+ * kept out of it and live under explicit subpaths; of the two, only
+ * `TestJournal` binds a Node SQLite database:
+ *
+ * ```ts
+ * import { Journal, SqlJournal } from "@smthrs/journal"
+ * import * as TestJournal from "@smthrs/journal/test/TestJournal"
+ * import * as Notifying from "@smthrs/journal/test/Notifying"
+ * ```
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * @category events
+ * @since 0.1.0
+ */
+export * as JournalEvent from "./JournalEvent.ts"
+
+/**
+ * @category events
+ * @since 1.0.0
+ */
+export * as EngineEvent from "./EngineEvent.ts"
+
+/**
+ * @category events
+ * @since 1.0.0
+ */
+export * as ExecutionFact from "./ExecutionFact.ts"
+
+/**
+ * @category events
+ * @since 1.0.0
+ */
+export * as CallFact from "./CallFact.ts"
+
+/**
+ * @since 1.0.0
+ */
+export * as StepFact from "./StepFact.ts"
+
+/**
+ * @category services
+ * @since 0.1.0
+ */
+export * as Journal from "./Journal.ts"
+
+/**
+ * @category layers
+ * @since 0.1.0
+ */
+export * as SqlJournal from "./SqlJournal.ts"
+
+/**
+ * @category services
+ * @since 1.0.0
+ */
+export * as Consensus from "./Consensus.ts"
+
+/**
+ * @category layers
+ * @since 1.0.0
+ */
+export * as SqlConsensus from "./SqlConsensus.ts"
+
+/**
+ * @category metrics
+ * @since 0.1.0
+ */
+export * as JournalMetrics from "./JournalMetrics.ts"
+
+/**
+ * @category redaction
+ * @since 0.1.0
+ */
+export * as Redaction from "./Redaction.ts"
+
+/**
+ * @category redaction
+ * @since 0.1.0
+ */
+export * as RedactedLogger from "./RedactedLogger.ts"
+
+/**
+ * @category projections
+ * @since 0.1.0
+ */
+export * as Projection from "./Projection.ts"
+
+/**
+ * @category models
+ * @since 0.1.0
+ */
+export * as OwnerId from "./OwnerId.ts"
+
+/**
+ * @category migrations
+ * @since 0.1.0
+ */
+export * as Migrations from "./Migrations.ts"

package/src/internal/JournalInternalFault.ts

@@ -0,0 +1,40 @@
+/**
+ * Faults of the journal's own background maintenance. They are logged and
+ * damped, never surfaced to a caller's write, so a log reader classifies one by
+ * `_tag` and `code` instead of a message.
+ *
+ * @since 1.0.0
+ */
+
+import * as Schema from "effect/Schema"
+
+/**
+ * Which background step refused.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export const JournalInternalFaultCode = Schema.Literals(["compaction_capture_timeout"])
+
+/**
+ * The stable code of a {@link JournalInternalFault}.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export type JournalInternalFaultCode = typeof JournalInternalFaultCode.Type
+
+/**
+ * A post-commit maintenance step that refused.
+ *
+ * @category errors
+ * @since 1.0.0
+ */
+export class JournalInternalFault
+  extends Schema.TaggedError<JournalInternalFault>()("@smthrs/journal/JournalInternalFault", {
+    code: JournalInternalFaultCode,
+    message: Schema.String
+  })
+{
+  override readonly name = "JournalInternalFault"
+}

package/src/migrations/0001_initial.ts

@@ -0,0 +1,41 @@
+/**
+ * Initial durable journal event schema.
+ *
+ * @since 0.1.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Creates the `flows_journal_events` table and its event-type index.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const initial: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+
+  yield* sql`CREATE TABLE flows_journal_events (
+    run_id TEXT NOT NULL CHECK (length(run_id) > 0),
+    seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`seq`)
+  } AND seq >= 0 AND seq <= 9007199254740991),
+    event_id TEXT NOT NULL UNIQUE CHECK (length(event_id) > 0),
+    source_id TEXT NOT NULL CHECK (length(source_id) > 0),
+    source_seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`source_seq`)
+  } AND source_seq >= 0 AND source_seq <= 9007199254740991),
+    emitted_at_ms ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`emitted_at_ms`)
+  } AND emitted_at_ms >= 0 AND emitted_at_ms <= 9007199254740991),
+    event_type TEXT NOT NULL CHECK (length(event_type) > 0),
+    payload_json TEXT NOT NULL CHECK (${Dialect.jsonValid(sql, sql`payload_json`)}),
+    meta_json TEXT NOT NULL CHECK (${Dialect.jsonValid(sql, sql`meta_json`)}),
+    PRIMARY KEY (run_id, seq),
+    UNIQUE (run_id, source_id, source_seq)
+  )`
+
+  yield* sql`CREATE INDEX flows_journal_events_event_type_idx ON flows_journal_events (event_type)`
+})

package/src/migrations/0002_checkpoints.ts

@@ -0,0 +1,46 @@
+/**
+ * Durable journal checkpoints and the compaction floor.
+ *
+ * One row per `(run_id, seq)`: the caller-captured state that replays the run
+ * from `seq` without the entries before it. `compacted_at_ms` is NULL until a
+ * compaction has truncated the entries strictly below `seq`; the largest
+ * compacted `seq` per run is the run's compaction floor, and reads whose
+ * cursor starts below it fail with `compacted` instead of returning a
+ * silently shortened history.
+ *
+ * Prior art: Temporal's mutable state is exactly this shape, a durable
+ * snapshot pinned to a history offset (`last_first_event_id` in
+ * `reference/temporal/proto/internal/temporal/server/api/persistence/v1/executions.proto`),
+ * with history below it never replayed.
+ *
+ * @since 0.1.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Creates the `flows_journal_checkpoints` table.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const checkpoints: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+
+  yield* sql`CREATE TABLE flows_journal_checkpoints (
+    run_id TEXT NOT NULL CHECK (length(run_id) > 0),
+    seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`seq`)
+  } AND seq >= 0 AND seq <= 9007199254740991),
+    state_json TEXT NOT NULL CHECK (${Dialect.jsonValid(sql, sql`state_json`)}),
+    created_at_ms ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`created_at_ms`)
+  } AND created_at_ms >= 0 AND created_at_ms <= 9007199254740991),
+    compacted_at_ms ${Dialect.integer(sql)} CHECK (compacted_at_ms IS NULL OR (${
+    Dialect.isInteger(sql, sql`compacted_at_ms`)
+  } AND compacted_at_ms >= 0 AND compacted_at_ms <= 9007199254740991)),
+    PRIMARY KEY (run_id, seq)
+  )`
+})

package/src/migrations/0003_startup_index.ts

@@ -0,0 +1,19 @@
+/**
+ * Index for the bounded journal source-event startup window.
+ *
+ * @since 0.1.0
+ */
+
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Preserves deterministic timestamp/run/sequence ordering without sorting the retained history.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const startupIndex = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  yield* sql`CREATE INDEX flows_journal_events_startup_idx ON flows_journal_events (emitted_at_ms DESC, run_id DESC, seq DESC)`
+})

package/src/migrations/0004_dedup.ts

@@ -0,0 +1,43 @@
+/**
+ * Producer identities retained when compaction deletes event payloads.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Creates the durable compacted-identity index.
+ *
+ * @category migrations
+ * @since 1.0.0-rc.0
+ */
+export const dedup: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  yield* sql`CREATE TABLE flows_journal_dedup (
+    run_id TEXT NOT NULL CHECK (length(run_id) > 0),
+    source_id TEXT NOT NULL CHECK (length(source_id) > 0),
+    source_seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`source_seq`)
+  } AND source_seq >= 0 AND source_seq <= 9007199254740991),
+    event_id TEXT NOT NULL UNIQUE CHECK (length(event_id) > 0),
+    seq ${Dialect.integer(sql)} NOT NULL CHECK (${
+    Dialect.isInteger(sql, sql`seq`)
+  } AND seq >= 0 AND seq <= 9007199254740991),
+    content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
+    PRIMARY KEY (run_id, source_id, source_seq)
+  )`
+  // Enforce the same identity constraints after their event rows are removed.
+  // RAISE(IGNORE) leaves INSERT ... RETURNING empty, so the existing duplicate
+  // path checks the owner fence and then resolves the original receipt.
+  yield* Dialect.trigger(sql, {
+    name: "flows_journal_dedup_insert_guard",
+    table: "flows_journal_events",
+    event: "BEFORE INSERT",
+    when: `EXISTS (SELECT 1 FROM flows_journal_dedup WHERE event_id = NEW.event_id
+      OR (run_id = NEW.run_id AND source_id = NEW.source_id AND source_seq = NEW.source_seq))`,
+    ignore: true
+  })
+})

package/src/migrations/0005_run_event_type.ts

@@ -0,0 +1,19 @@
+/**
+ * Indexed exact event-type reads within a run.
+ *
+ * @since 1.0.0
+ */
+
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Keeps filtered cursor reads proportional to the matching entries.
+ *
+ * @category migrations
+ * @since 1.0.0
+ */
+export const runEventType: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  yield* sql`CREATE INDEX flows_journal_events_run_event_type_idx ON flows_journal_events (run_id, event_type, seq)`
+})

package/src/migrations/0006_consensus.ts

@@ -0,0 +1,83 @@
+/**
+ * The `SqlConsensus` lease table.
+ *
+ * This is the private schema of the default consensus strategy: the owner
+ * tuple, the two-phase claim columns, and the grant timestamp relocated from
+ * `flows_runs`. Lease rows are strategy evidence, never history: heartbeats
+ * and lease state never enter the journal.
+ *
+ * @since 1.0.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Creates the `flows_consensus_leases` table and backfills a lease for every
+ * running or claimed run when `flows_runs` is already present, so an owner
+ * that was driving a run before this migration keeps its fence afterwards.
+ *
+ * @category migrations
+ * @since 1.0.0
+ */
+export const consensus: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+
+  yield* sql`CREATE TABLE flows_consensus_leases (
+    run_id TEXT PRIMARY KEY CHECK (length(run_id) > 0),
+    owner_host_id TEXT,
+    owner_pid ${Dialect.integer(sql)} CHECK (owner_pid IS NULL OR (${
+    Dialect.isInteger(sql, sql`owner_pid`)
+  } AND owner_pid >= 0)),
+    owner_nonce TEXT,
+    granted_at_ms ${Dialect.integer(sql)} CHECK (granted_at_ms IS NULL OR (${
+    Dialect.isInteger(sql, sql`granted_at_ms`)
+  } AND granted_at_ms >= 0)),
+    heartbeat_at_ms ${Dialect.integer(sql)} CHECK (heartbeat_at_ms IS NULL OR (${
+    Dialect.isInteger(sql, sql`heartbeat_at_ms`)
+  } AND heartbeat_at_ms >= 0)),
+    claim_host_id TEXT,
+    claim_pid ${Dialect.integer(sql)} CHECK (claim_pid IS NULL OR (${
+    Dialect.isInteger(sql, sql`claim_pid`)
+  } AND claim_pid >= 0)),
+    claim_nonce TEXT,
+    claimed_at_ms ${Dialect.integer(sql)} CHECK (claimed_at_ms IS NULL OR (${
+    Dialect.isInteger(sql, sql`claimed_at_ms`)
+  } AND claimed_at_ms >= 0))
+  )`
+
+  const tables = yield* Dialect.tables(sql)
+  if (!tables.some((table) => table.name === "flows_runs")) {
+    return
+  }
+
+  yield* sql`
+    INSERT INTO flows_consensus_leases (
+      run_id,
+      owner_host_id,
+      owner_pid,
+      owner_nonce,
+      granted_at_ms,
+      heartbeat_at_ms,
+      claim_host_id,
+      claim_pid,
+      claim_nonce,
+      claimed_at_ms
+    )
+    SELECT
+      run_id,
+      owner_host_id,
+      owner_pid,
+      owner_nonce,
+      CASE WHEN status = 'running' THEN heartbeat_at_ms ELSE NULL END,
+      CASE WHEN status = 'running' THEN heartbeat_at_ms ELSE NULL END,
+      claim_host_id,
+      claim_pid,
+      claim_nonce,
+      claimed_at_ms
+    FROM flows_runs
+    WHERE status = 'running' OR claim_host_id IS NOT NULL
+    ON CONFLICT (run_id) DO NOTHING
+  `
+})

package/src/test/Notifying.ts

@@ -0,0 +1,101 @@
+/**
+ * Effect-native analogue of Bazel Skyframe's `NotifyingHelper` and
+ * `NotifyingInMemoryGraph`: wrap a record-of-Effect-methods service so a listener hook fires around
+ * every operation. The hook runs in the calling fiber, so a hook that dies,
+ * interrupts, or awaits a `Latch` injects crashes, fence loss, and exact
+ * sequencing at any durable transition point, the mechanism that makes
+ * interstitial interleaving bugs testable at all.
+ *
+ * Deliberately NOT ported from Skyframe: `DeterministicHelper`'s
+ * deterministic-scheduler machinery, `ChainedFunction`, and `TrackingAwaiter`.
+ * Effect already gives us deterministic fibers, `TestClock`, `Latch`, and
+ * failure propagation across fiber joins, so those helpers have no residue
+ * here.
+ *
+ * @since 0.1.0
+ */
+
+import type * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+
+/**
+ * Whether the hook is firing before or after the delegated operation.
+ * Skyframe's `NotifyingHelper.Order`, minus its `BEFORE_AND_AFTER` (pass the
+ * same hook body for both firings instead).
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Order = "before" | "after"
+
+/**
+ * Listener invoked around every delegated operation. `op` is the service
+ * method name (Skyframe's `EventType`, but open-world), `args` the call
+ * arguments. Express crash injection as `Effect.die` or `Effect.interrupt`
+ * inside the hook, the hook's type stays `Effect<void>` so the wrapped
+ * service is structurally identical to the real one.
+ *
+ * The `after` firing only happens when the operation succeeds; a failed
+ * operation already is the interesting event.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Hook = (
+  op: string,
+  order: Order,
+  args: ReadonlyArray<unknown>
+) => Effect.Effect<void>
+
+/**
+ * Wraps a service so `hook(op, "before" | "after", args)` fires around every
+ * Effect-valued operation, delegating to the real implementation in between.
+ *
+ * Generic over any record-of-Effect-methods service (`Journal`, `CacheStore`,
+ * `AttemptStore`, `RunStore`, ...): Effect-returning methods and plain Effect
+ * properties are hooked; non-Effect members (e.g. `Stream`-returning methods)
+ * delegate untouched.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const wrap = <S extends object>(service: S, hook: Hook): S => {
+  const wrapped: Record<string, unknown> = {}
+  for (const key of Object.keys(service)) {
+    const value = (service as Record<string, unknown>)[key]
+    if (Effect.isEffect(value)) {
+      wrapped[key] = hook(key, "before", []).pipe(
+        Effect.andThen(value as Effect.Effect<unknown, unknown, unknown>),
+        Effect.tap(() => hook(key, "after", []))
+      )
+    } else if (typeof value === "function") {
+      const method = value as (...args: Array<unknown>) => unknown
+      wrapped[key] = (...args: Array<unknown>) => {
+        const result = method.apply(service, args)
+        return Effect.isEffect(result)
+          ? hook(key, "before", args).pipe(
+            Effect.andThen(result as Effect.Effect<unknown, unknown, unknown>),
+            Effect.tap(() => hook(key, "after", args))
+          )
+          : result
+      }
+    } else {
+      wrapped[key] = value
+    }
+  }
+  return wrapped as S
+}
+
+/**
+ * Layer form of {@link wrap}: reads the real service from the environment and
+ * re-provides the notifying wrapper under the same key, so an existing stack
+ * gains a listener without touching its construction.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = <Id, S extends object>(
+  tag: Context.Key<Id, S>,
+  hook: Hook
+): Layer.Layer<Id, never, Id> => Layer.effect(tag)(Effect.map(tag, (service) => wrap(service, hook)))

package/src/test/TestJournal.ts

@@ -0,0 +1,52 @@
+/**
+ * Deterministic bundle of the production journal service.
+ *
+ * This provides the journal and nothing else. A consumer that also needs run
+ * state or the step cache composes `@smthrs/run-store/test/TestRunStore` and
+ * `@smthrs/step-cache/test/TestCacheStore` over the same database, or takes
+ * the whole engine bundle from `@smthrs/engine-store/test/TestStores`.
+ *
+ * @since 0.1.0
+ */
+
+import * as TestDatabase from "@smthrs/database/test/TestDatabase"
+import * as Layer from "effect/Layer"
+import * as Migrations from "../Migrations.ts"
+import * as SqlJournal from "../SqlJournal.ts"
+
+/**
+ * Options for the deterministic journal bundle.
+ *
+ * The production options with `capacity` and `overflow` made optional. Every
+ * field forwards to {@link SqlJournal.layer} unchanged, so a suite can exercise
+ * index-bound eviction, the entry byte bound, a custom or noop redactor, and
+ * the compaction policy through the bundle instead of hand-assembling the layer
+ * stack the bundle exists to hide.
+ *
+ * The type is derived rather than mirrored because a mirror silently falls
+ * behind: the hand-copied list had already lost `maxEntryBytes`, so the one
+ * production bound a hostile-payload suite needs was unreachable here.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface TestJournalOptions extends Omit<SqlJournal.SqlJournalOptions, "capacity" | "overflow"> {
+  /** Admission-queue bound and `changes` buffer size. Defaults to 1024. */
+  readonly capacity?: number | undefined
+  /** Policy applied when the admission queue is full. Defaults to `reject`. */
+  readonly overflow?: SqlJournal.SqlJournalOptions["overflow"] | undefined
+}
+
+/**
+ * Provides the production SQLite journal over an in-memory database.
+ * Migrations run before the journal is exposed.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = (options?: TestJournalOptions) =>
+  SqlJournal.layer({
+    ...options,
+    capacity: options?.capacity ?? 1024,
+    overflow: options?.overflow ?? "reject"
+  }).pipe(Layer.provide(Layer.provideMerge(Migrations.layer, TestDatabase.layer)))