@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3

package/src/migrations/0008_deferred_consumption.ts

@@ -0,0 +1,27 @@
+/**
+ * Registration recovery only needs completions the run has not observed.
+ *
+ * @since 1.0.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Retains completion values for replay while removing consumed rows from sweeps.
+ * Existing completions stay eligible until first read by the upgraded engine.
+ *
+ * @category migrations
+ * @since 1.0.0
+ */
+export const deferredConsumption: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  yield* sql`ALTER TABLE flows_deferred_completions ADD COLUMN consumed_at_ms ${Dialect.integer(sql)}
+    CHECK (consumed_at_ms IS NULL OR (${
+    Dialect.isInteger(sql, sql`consumed_at_ms`)
+  } AND consumed_at_ms >= 0 AND consumed_at_ms <= 9007199254740991))`
+  yield* sql`CREATE INDEX flows_deferred_completions_unconsumed_idx
+    ON flows_deferred_completions (flow_name, execution_id, deferred_name)
+    WHERE consumed_at_ms IS NULL`
+})

package/src/migrations/0009_deferred_clock_facts.ts

@@ -0,0 +1,163 @@
+/**
+ * Backfills the journal records the deferred/clock fold reads.
+ *
+ * @since 1.0.0
+ */
+
+import { FlowEngine } from "@smthrs/engine"
+import * as JournalEvent from "@smthrs/journal/JournalEvent"
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+import { EventTypes } from "../EventTypes.ts"
+
+interface ClockRow {
+  readonly flow_name: string
+  readonly execution_id: string
+  readonly clock_name: string
+  readonly deferred_name: string
+  readonly due_at_ms: number | string
+  readonly completed_at_ms: number | string | null
+}
+
+interface DeferredRow {
+  readonly flow_name: string
+  readonly execution_id: string
+  readonly deferred_name: string
+  readonly exit_json: string
+  readonly metadata_json: string | null
+  readonly completed_at_ms: number | string
+  readonly consumed_at_ms: number | string | null
+}
+
+const optionalTime = (value: number | string | null): number | null => value === null ? null : Number(value)
+
+/**
+ * Journals every deferred/clock change written before the fold's records
+ * existed: a clock row with no `clock-scheduled` record, a completed clock
+ * with no `clock-completed` record, a completion with no `deferred-completed`
+ * record, and a consumed completion with no `deferred-consumed` record. Without
+ * them a rebuild would drop the clock, or re-arm a completed one and wake a run
+ * for a result it already observed (issue #2037).
+ *
+ * Each backfilled record is appended at the run's next sequence under a
+ * `flows.engine.backfill:` producer whose identity names the row, with the
+ * lineage meta every engine record carries.
+ *
+ * @category migrations
+ * @since 1.0.0
+ */
+export const deferredClockFacts: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  const recorded = new Set<string>()
+  const events = yield* sql<{ readonly run_id: string; readonly event_type: string; readonly payload_json: string }>`
+    SELECT run_id, event_type, payload_json FROM flows_journal_events
+    WHERE ${
+    sql.in("event_type", [
+      EventTypes.clockScheduled,
+      EventTypes.clockCompleted,
+      EventTypes.deferredCompleted,
+      EventTypes.deferredConsumed
+    ])
+  }
+  `
+  for (const event of events) {
+    const payload = JSON.parse(event.payload_json) as Record<string, unknown>
+    const name = event.event_type === EventTypes.clockScheduled || event.event_type === EventTypes.clockCompleted
+      ? payload.clockName
+      : payload.deferredName
+    recorded.add(JSON.stringify([event.event_type, payload.flowName, payload.executionId, name]))
+  }
+  const next = new Map<string, number>()
+  const append = (
+    eventType: string,
+    executionId: string,
+    name: string,
+    emittedAtMs: number,
+    payload: Readonly<Record<string, unknown>>
+  ) =>
+    Effect.gen(function*() {
+      const identity = JSON.stringify([eventType, payload.flowName, executionId, name])
+      if (recorded.has(identity)) return
+      recorded.add(identity)
+      let seq = next.get(executionId)
+      if (seq === undefined) {
+        const rows = yield* sql<{ readonly next: number | string | null }>`
+          SELECT MAX(seq) + 1 AS next FROM flows_journal_events WHERE run_id = ${executionId}
+        `
+        seq = Number(rows[0]?.next ?? 0)
+      }
+      next.set(executionId, seq + 1)
+      const sourceId = `flows.engine.backfill:${JournalEvent.contentDigest(identity)}`
+      yield* sql`
+        INSERT INTO flows_journal_events (
+          run_id, seq, event_id, source_id, source_seq, emitted_at_ms, event_type, payload_json, meta_json
+        ) VALUES (
+          ${executionId},
+          ${seq},
+          ${
+        JournalEvent.makeEventId(
+          executionId as JournalEvent.RunId,
+          sourceId as JournalEvent.SourceId,
+          0 as JournalEvent.SourceSeq
+        )
+      },
+          ${sourceId},
+          ${0},
+          ${emittedAtMs},
+          ${eventType},
+          ${JSON.stringify(payload)},
+          ${JSON.stringify({ lineageId: FlowEngine.Lineage.root(executionId) })}
+        )
+      `
+    })
+
+  const clocks = yield* sql<ClockRow>`
+    SELECT flow_name, execution_id, clock_name, deferred_name, due_at_ms, completed_at_ms
+    FROM flows_clock_deadlines
+    ORDER BY execution_id, flow_name, clock_name
+  `
+  for (const clock of clocks) {
+    const dueAtMs = Number(clock.due_at_ms)
+    const completedAtMs = optionalTime(clock.completed_at_ms)
+    yield* append(EventTypes.clockScheduled, clock.execution_id, clock.clock_name, completedAtMs ?? dueAtMs, {
+      flowName: clock.flow_name,
+      executionId: clock.execution_id,
+      clockName: clock.clock_name,
+      deferredName: clock.deferred_name,
+      dueAtMs
+    })
+    if (completedAtMs !== null) {
+      yield* append(EventTypes.clockCompleted, clock.execution_id, clock.clock_name, completedAtMs, {
+        flowName: clock.flow_name,
+        executionId: clock.execution_id,
+        clockName: clock.clock_name,
+        completedAtMs
+      })
+    }
+  }
+  const deferreds = yield* sql<DeferredRow>`
+    SELECT flow_name, execution_id, deferred_name, exit_json, metadata_json, completed_at_ms, consumed_at_ms
+    FROM flows_deferred_completions
+    ORDER BY execution_id, flow_name, deferred_name
+  `
+  for (const deferred of deferreds) {
+    const completedAtMs = Number(deferred.completed_at_ms)
+    const consumedAtMs = optionalTime(deferred.consumed_at_ms)
+    yield* append(EventTypes.deferredCompleted, deferred.execution_id, deferred.deferred_name, completedAtMs, {
+      flowName: deferred.flow_name,
+      executionId: deferred.execution_id,
+      deferredName: deferred.deferred_name,
+      completedAtMs,
+      exitDigest: JournalEvent.contentDigest(deferred.exit_json),
+      metadataDigest: deferred.metadata_json === null ? null : JournalEvent.contentDigest(deferred.metadata_json)
+    })
+    if (consumedAtMs !== null) {
+      yield* append(EventTypes.deferredConsumed, deferred.execution_id, deferred.deferred_name, consumedAtMs, {
+        flowName: deferred.flow_name,
+        executionId: deferred.execution_id,
+        deferredName: deferred.deferred_name,
+        consumedAtMs
+      })
+    }
+  }
+})

package/src/migrations/0010_pending_cancellation.ts

@@ -0,0 +1,20 @@
+/**
+ * Indexed cancellation cleanup for admissions whose handler is unavailable.
+ *
+ * @since 1.0.0
+ */
+
+import * as Effect from "effect/Effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * Bounds cleanup reads to requested pending rows.
+ *
+ * @category migrations
+ * @since 1.0.0
+ */
+export const pendingCancellation: Effect.Effect<void, unknown, SqlClient.SqlClient> = Effect.gen(function*() {
+  const sql = yield* SqlClient.SqlClient
+  yield* sql`CREATE INDEX flows_runs_pending_cancel_idx ON flows_runs (created_at_ms, run_id)
+    WHERE status = 'pending' AND cancel_requested_at_ms IS NOT NULL`
+})

package/src/test/TestStores.ts

@@ -0,0 +1,145 @@
+/**
+ * Deterministic bundle of every durable store a flow engine composes.
+ *
+ * The stores split into `@smthrs/journal`, `@smthrs/run-store`, and
+ * `@smthrs/step-cache`, each with its own in-memory test layer. An engine
+ * needs all of them over ONE database, which is what this layer builds:
+ * the composed migration sets from `../Migrations.ts` run first, then the four
+ * services bind to the same in-memory SQLite connection.
+ *
+ * Governing design: `packages/smithers/flows/journal/docs/README.md`.
+ *
+ * @since 0.1.0
+ */
+
+import * as DurableWriter from "@smthrs/database/DurableWriter"
+import * as NodeDatabase from "@smthrs/database/node/NodeDatabase"
+import * as TestDatabase from "@smthrs/database/test/TestDatabase"
+import * as SqlJournal from "@smthrs/journal/SqlJournal"
+import * as PlanStore from "@smthrs/plan-store/PlanStore"
+import * as AttemptStore from "@smthrs/run-store/AttemptStore"
+import * as RunStore from "@smthrs/run-store/RunStore"
+import * as CacheStore from "@smthrs/step-cache/CacheStore"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as DurableEngineState from "../DurableEngineState.ts"
+import * as Migrations from "../Migrations.ts"
+import * as OwnerIdentity from "../OwnerIdentity.ts"
+import * as PlanInputStore from "../PlanInputStore.ts"
+import * as PlanMergeStore from "../PlanMergeStore.ts"
+
+/**
+ * Options for the deterministic store bundle.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface TestStoresOptions {
+  readonly capacity?: number
+  readonly overflow?: "reject" | "drop-newest" | "drop-oldest"
+  readonly batchSize?: number
+}
+
+/**
+ * An in-memory SQLite database with the complete durable engine schema
+ * installed, exposed as `SqlClient` and `DurableWriter`.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const database = Layer.provideMerge(Migrations.layer, TestDatabase.layer)
+
+/**
+ * Provides the production SQLite journal, run, attempt, and cache services
+ * over one in-memory database. Migrations run before any durable service is
+ * exposed. The shared writer remains exposed for the engine cancellation
+ * boundary, which must inspect its complete Exit before error normalization.
+ *
+ * `OwnerIdentity.layer` rides along: it is not a store, but it is the other
+ * service `EngineStore.make` requires and has no in-memory variant to choose
+ * between. The default is used rather than a pinned one so a test observes
+ * the same fresh-per-incarnation owner the production composition mints.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = (options?: TestStoresOptions) =>
+  Layer.mergeAll(
+    SqlJournal.layer({
+      capacity: options?.capacity ?? 1024,
+      overflow: options?.overflow ?? "reject",
+      batchSize: options?.batchSize
+    }),
+    RunStore.layer,
+    AttemptStore.layer,
+    CacheStore.layer,
+    PlanStore.layer,
+    PlanInputStore.layer,
+    PlanMergeStore.layer,
+    Layer.effect(DurableWriter.DurableWriter, DurableWriter.DurableWriter)
+  ).pipe(Layer.provide(database), Layer.merge(OwnerIdentity.layer))
+
+/**
+ * The same schema over a named SQLite database or PostgreSQL URL, with the connection exposed.
+ *
+ * {@link database} hides `SqlClient` behind the stores it provisions, which is
+ * right for a case that only needs an engine. Two other shapes need the
+ * connection itself: a composition that adds another SQL-backed service over
+ * the same database (a control runtime, for one), and a case that has to prove
+ * cross-process durability, which needs a persistent identity rather than the private
+ * in-memory database each `:memory:` connection gets to itself.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const databaseAt = (filename: string) => {
+  const url = /^postgres(?:ql)?:/.test(filename) ? new URL(filename) : undefined
+  const schema = url?.searchParams.get("schema") ?? undefined
+  url?.searchParams.delete("schema")
+  const database = url === undefined ?
+    NodeDatabase.layer({ filename }) :
+    Layer.unwrap(
+      Effect.promise(() => import("@smthrs/database/postgres/PostgresDatabase")).pipe(
+        Effect.map((PostgresDatabase) =>
+          PostgresDatabase.layer({
+            url: url.toString(),
+            schema,
+            postgres: { connectTimeout: Infinity, idleTimeout: Infinity, connectionTTL: Infinity }
+          })
+        )
+      )
+    )
+  return Layer.provideMerge(Migrations.layer, Layer.provideMerge(DurableWriter.layer(), database))
+}
+
+/**
+ * Every durable engine service over one named database, connection included.
+ *
+ * `layer` builds the same stores over a private in-memory database and keeps
+ * `SqlClient` to itself. This one takes the database by name and re-exports
+ * the connection, so a case can point two independently constructed bundles at
+ * one file — two connections, two engines, no shared object graph — which is
+ * what a second process actually looks like. `DurableEngineState` rides along
+ * for the same reason: its in-memory variant is a map that a second bundle
+ * would not see.
+ *
+ * Pass `:memory:` for the cheap variant when the connection is still needed.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerAt = (filename: string, options?: TestStoresOptions) =>
+  Layer.mergeAll(
+    SqlJournal.layer({
+      capacity: options?.capacity ?? 1024,
+      overflow: options?.overflow ?? "reject",
+      batchSize: options?.batchSize
+    }),
+    RunStore.layer,
+    AttemptStore.layer,
+    CacheStore.layer,
+    PlanStore.layer,
+    PlanInputStore.layer,
+    PlanMergeStore.layer,
+    DurableEngineState.layer
+  ).pipe(Layer.provideMerge(databaseAt(filename)), Layer.merge(OwnerIdentity.layer))