@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3

package/src/internal/DeferredPersistence.ts

@@ -0,0 +1,459 @@
+/**
+ * Durable deferred delivery and absolute clock scheduling.
+ *
+ * @since 0.1.0
+ */
+
+import { FlowEngine } from "@smthrs/engine"
+import { type DurableClock, type DurableDeferred, type Flow, FlowRuntime } from "@smthrs/flow"
+import { Journal } from "@smthrs/journal"
+import type { Ownership } from "@smthrs/run-store"
+import * as Clock from "effect/Clock"
+import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as FiberMap from "effect/FiberMap"
+import * as Option from "effect/Option"
+import * as Schedule from "effect/Schedule"
+import type * as Scope from "effect/Scope"
+import * as DurableEngineState from "../DurableEngineState.ts"
+import * as JournalRecords from "./JournalRecords.ts"
+import * as StateTransaction from "./StateTransaction.ts"
+
+/**
+ * Reasons passed to the single claim-gated resume scheduler.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type ResumeReason = "deferred" | "clock"
+
+/**
+ * Dependencies for durable deferred and clock persistence.
+ *
+ * `scheduleResume` must enter the run coordinator and ownership claim path. It
+ * must never invoke a flow handler directly.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Dependencies {
+  readonly owner: Ownership.OwnerId
+  readonly journalSource: string
+  readonly scheduleResume: (
+    flowName: string,
+    executionId: string,
+    reason: ResumeReason,
+    sourceId?: string | undefined
+  ) => Effect.Effect<void>
+  /**
+   * Redispatch policy for a durable clock whose fire failed. Defaults to
+   * {@link defaultFireRetryPolicy}; a composition supplies its own the way
+   * the engine's `suspendedRetryPolicy` option is supplied, rather than
+   * editing the constant.
+   */
+  readonly fireRetryPolicy?: FireRetryPolicy | undefined
+}
+
+/**
+ * Encoded deferred completion with optional opaque correlation metadata.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface DeferredDoneOptions {
+  readonly flowName: string
+  readonly executionId: string
+  readonly deferredName: string
+  readonly exit: Exit.Exit<unknown, unknown>
+  readonly metadata?: unknown
+}
+
+/**
+ * Durable deferred and clock operations composed into the encoded engine.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Service {
+  readonly deferredResult: (
+    deferred: DurableDeferred.Any
+  ) => Effect.Effect<
+    Option.Option<Exit.Exit<unknown, unknown>>,
+    never,
+    FlowRuntime.FlowInstance
+  >
+  readonly deferredDone: (options: DeferredDoneOptions) => Effect.Effect<void>
+  readonly deferredDoneIfWaiting: (
+    options: DeferredDoneOptions & {
+      readonly reason: string
+      readonly token: string
+    }
+  ) => Effect.Effect<FlowRuntime.DeferredDoneIfWaitingOutcome>
+  readonly scheduleClock: (
+    flow: Flow.Any,
+    options: {
+      readonly executionId: string
+      readonly clock: DurableClock.DurableClock
+    }
+  ) => Effect.Effect<void>
+  readonly sweepDue: (flowName?: string) => Effect.Effect<void>
+}
+
+const clockKey = (row: DurableEngineState.ClockAddress): string =>
+  JSON.stringify([row.flowName, row.executionId, row.clockName])
+
+/**
+ * A clock-fire redispatch policy. The input is the sandboxed cause of a failed
+ * fire, so any schedule that recurs on an arbitrary input fits.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type FireRetryPolicy = Schedule.Schedule<unknown, unknown>
+
+/**
+ * The default redispatch policy for a durable clock whose fire failed.
+ *
+ * Temporal redispatches timer-queue tasks with backoff until they are acked
+ * (`reference/temporal` `service/history` timer queue + `common/backoff`); we
+ * mirror that here so one transient journal/flush error at fire time cannot
+ * lose the timer until a process restart. Exponential from 100ms, capped at
+ * 30s, forever — the fire is idempotent (first-writer deferred completion,
+ * deduplicated journal admission, CAS clock completion), so retrying an
+ * arbitrary prefix is safe.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const defaultFireRetryPolicy: FireRetryPolicy = Schedule.min([
+  Schedule.exponential("100 millis"),
+  Schedule.spaced("30 seconds")
+])
+
+/**
+ * Constructs durable deferred and clock persistence.
+ *
+ * Delivery ordering is `durable state -> durable journal -> claim-gated
+ * resume`. The delivery operation itself never invokes a flow handler.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const make = (
+  dependencies: Dependencies
+): Effect.Effect<
+  Service,
+  never,
+  DurableEngineState.DurableEngineState | Journal.Journal | Scope.Scope
+> =>
+  Effect.gen(function*() {
+    const state = yield* DurableEngineState.DurableEngineState
+    const journal = yield* Journal.Journal
+    /**
+     * The package's ordered state-and-journal boundary: engine state first,
+     * the journal's write transaction inside it. See
+     * {@link StateTransaction.make} for the order and what it guarantees.
+     */
+    const transactState = StateTransaction.make(state, journal)
+    const timers = yield* FiberMap.make<string>()
+    const fireRetryPolicy = dependencies.fireRetryPolicy ?? defaultFireRetryPolicy
+
+    // The durable channel commits inside `emitDurable`; the flush that
+    // follows only pushes the *lossy* queue. Once the journal's lossy writer
+    // latches a `sink_failed` it stays latched for the process (issue #43),
+    // so treating a flush failure as a defect would turn every committed
+    // durable write into a permanent stall. Log and move on: durable
+    // delivery must never be coupled to lossy-channel health.
+    const flushLossy = journal.flush.pipe(
+      Effect.catchCause((cause) =>
+        Effect.logWarning(
+          "engine-store: lossy journal flush failed after a committed durable write; continuing",
+          cause
+        )
+      )
+    )
+
+    const persistDeferred = (
+      options: DeferredDoneOptions,
+      reason: ResumeReason = "deferred",
+      expectedWaiting?: { readonly reason: string; readonly token: string } | undefined,
+      alongside: Effect.Effect<void> = Effect.void
+    ): Effect.Effect<FlowRuntime.DeferredDoneIfWaitingOutcome> =>
+      Effect.gen(function*() {
+        const completedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
+        // The completion row and the record describing it commit as one unit:
+        // a crash between them left a resumable deferred the journal never
+        // announced. The lossy flush below stays outside — it waits on the
+        // journal's writer fiber, which would deadlock against the write
+        // transaction this holds.
+        const admission = yield* transactState(
+          Effect.gen(function*() {
+            if (expectedWaiting !== undefined) {
+              const waiting = yield* state.waiting(options.executionId)
+              if (
+                Option.isNone(waiting) ||
+                waiting.value.reason !== expectedWaiting.reason ||
+                waiting.value.token !== expectedWaiting.token
+              ) {
+                return { _tag: "NotWaiting" as const }
+              }
+            }
+            const completion = yield* state.completeDeferred({
+              flowName: options.flowName,
+              executionId: options.executionId,
+              deferredName: options.deferredName,
+              exit: options.exit,
+              ...(options.metadata === undefined ? {} : { metadata: options.metadata }),
+              completedAtMs
+            })
+            const row = completion.row
+
+            // Durable channel: a deferred completion is a lifecycle record and
+            // must never take the droppable lossy queue. It is unfenced by
+            // design — external-trigger admissions are first-writer-wins
+            // regardless of who owns the run (issue #10), which is exactly the
+            // admission `emitDurableUnfenced` exists for. The record names the
+            // row and the digests of its value; the value stays in the row,
+            // because the journal is redacted and replayed (issue #72).
+            yield* journal.emitDurableUnfenced(
+              JournalRecords.deferredCompleted({
+                runId: options.executionId,
+                lineageId: FlowEngine.Lineage.root(options.executionId),
+                sourceId: `${dependencies.journalSource}:deferred:${
+                  JSON.stringify([options.flowName, options.executionId, options.deferredName])
+                }`,
+                sourceSeq: 0
+              }, {
+                flowName: row.flowName,
+                executionId: row.executionId,
+                deferredName: row.deferredName,
+                completedAtMs: row.completedAtMs,
+                exitDigest: completion.digest.exit,
+                metadataDigest: completion.digest.metadata
+              })
+            ).pipe(Effect.orDie)
+            yield* alongside
+            return { _tag: "Admitted" as const, completion }
+          })
+        ).pipe(Effect.orDie)
+        if (admission._tag === "NotWaiting") return "NotWaiting"
+        const row = admission.completion.row
+        yield* flushLossy
+        yield* dependencies.scheduleResume(
+          row.flowName,
+          row.executionId,
+          reason
+        )
+        return admission.completion._tag
+      })
+
+    const completeDeferred = (
+      options: DeferredDoneOptions,
+      reason: ResumeReason = "deferred"
+    ): Effect.Effect<void> => persistDeferred(options, reason).pipe(Effect.asVoid)
+
+    const fireClock = (row: DurableEngineState.ClockRow): Effect.Effect<void> =>
+      Effect.gen(function*() {
+        const completedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
+        // The clock completes in the deferred's transaction, and its record
+        // with it: a completion the journal never recorded is a clock a
+        // rebuilt deadline index would arm again (issue #2037).
+        yield* persistDeferred(
+          {
+            flowName: row.flowName,
+            executionId: row.executionId,
+            deferredName: row.deferredName,
+            exit: Exit.void,
+            metadata: {
+              clockName: row.clockName,
+              dueAtMs: row.dueAtMs,
+              completedAtMs
+            }
+          },
+          "clock",
+          undefined,
+          Effect.flatMap(
+            state.completeClock(row, completedAtMs),
+            (completed) =>
+              completed._tag === "Completed" ? emitClockCompleted(completed.row, completedAtMs) : Effect.void
+          )
+        )
+      })
+
+    const armClock = (row: DurableEngineState.ClockRow): Effect.Effect<void> =>
+      // A replayed sleep re-schedules its clock and gets the existing row
+      // back. A fired clock whose completion stands is not armed again: firing
+      // it would re-admit that completion and wake the run, whose replay would
+      // re-schedule the clock again, re-driving the parked run forever. A
+      // rewind can drop the completion and keep the fired row; that clock
+      // still has to fire.
+      (row.completedAtMs === null
+        ? Effect.succeed(false)
+        : Effect.map(state.deferred(row), Option.isSome)).pipe(
+          Effect.flatMap((fired) => fired ? Effect.void : armTimer(row))
+        )
+
+    const armTimer = (row: DurableEngineState.ClockRow): Effect.Effect<void> =>
+      Clock.currentTimeMillis.pipe(Effect.map(Math.floor)).pipe(
+        Effect.flatMap((nowMs) =>
+          fireClock(row).pipe(
+            // A failed fire (journal emit/flush defect included) must not kill
+            // the one-shot timer fiber: expose the full cause and redispatch
+            // with backoff until the fire is durably acked.
+            Effect.sandbox,
+            Effect.retry(fireRetryPolicy),
+            Effect.orDie,
+            Effect.delay(Duration.millis(Math.max(0, row.dueAtMs - nowMs))),
+            FiberMap.run(timers, clockKey(row), { onlyIfMissing: true })
+          )
+        ),
+        Effect.asVoid
+      )
+
+    /** Records that a clock stopped being pending, in the caller's transaction. */
+    const emitClockCompleted = (
+      row: DurableEngineState.ClockAddress,
+      completedAtMs: number
+    ): Effect.Effect<void> =>
+      journal.emitDurableUnfenced(
+        JournalRecords.clockCompletedFor(
+          dependencies.journalSource,
+          FlowEngine.Lineage.root(row.executionId),
+          row,
+          completedAtMs
+        )
+      ).pipe(Effect.asVoid, Effect.orDie)
+
+    /**
+     * Durable channel: clock schedule records are lifecycle evidence and must
+     * never be droppable (issue #10). Unfenced: registration-time sweeps
+     * re-record clocks the current process does not own.
+     */
+    const emitClockScheduled = (
+      row: DurableEngineState.ClockRow
+    ): Effect.Effect<void> =>
+      journal.emitDurableUnfenced(
+        JournalRecords.clockScheduled({
+          runId: row.executionId,
+          lineageId: FlowEngine.Lineage.root(row.executionId),
+          sourceId: `${dependencies.journalSource}:clock:${
+            JSON.stringify([row.flowName, row.executionId, row.clockName])
+          }`,
+          sourceSeq: 0
+        }, {
+          flowName: row.flowName,
+          executionId: row.executionId,
+          clockName: row.clockName,
+          deferredName: row.deferredName,
+          dueAtMs: row.dueAtMs
+        })
+      ).pipe(Effect.asVoid, Effect.orDie)
+
+    /** Re-announces an already persisted clock row (the sweep path). */
+    const recordClockScheduled = (
+      row: DurableEngineState.ClockRow
+    ): Effect.Effect<void> => emitClockScheduled(row).pipe(Effect.andThen(flushLossy))
+
+    const scheduleClock: Service["scheduleClock"] = Effect.fn("DeferredPersistence.scheduleClock")((
+      flow,
+      options
+    ) =>
+      Effect.gen(function*() {
+        const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
+        // The clock row and its schedule record commit as one unit, so a
+        // crash between them can no longer arm a durable timer the journal
+        // never announced (or announce one that was rolled back).
+        const scheduled = yield* transactState(
+          Effect.gen(function*() {
+            const scheduled = yield* state.scheduleClock({
+              flowName: flow._tag,
+              executionId: options.executionId,
+              clockName: options.clock.name,
+              deferredName: options.clock.deferred.name,
+              dueAtMs: nowMs + Duration.toMillis(options.clock.duration),
+              completedAtMs: null
+            }, dependencies.owner)
+            yield* emitClockScheduled(scheduled.row)
+            return scheduled
+          })
+        ).pipe(Effect.orDie)
+        yield* flushLossy
+        yield* armClock(scheduled.row)
+      })
+    )
+
+    const sweepDue: Service["sweepDue"] = Effect.fn("DeferredPersistence.sweepDue")((flowName) =>
+      Effect.gen(function*() {
+        const rows = yield* state.pendingClocks(flowName === undefined ? {} : { flowName })
+        yield* Effect.forEach(
+          rows,
+          (row) => recordClockScheduled(row).pipe(Effect.andThen(armClock(row))),
+          { discard: true }
+        )
+        if (flowName !== undefined) {
+          const completions = yield* state.completedDeferreds(flowName)
+          yield* Effect.forEach(
+            completions,
+            (address) =>
+              dependencies.scheduleResume(
+                address.flowName,
+                address.executionId,
+                "deferred",
+                `${dependencies.journalSource}:wake:${
+                  JSON.stringify([
+                    address.flowName,
+                    address.executionId,
+                    address.deferredName
+                  ])
+                }`
+              ),
+            { discard: true }
+          )
+        }
+      })
+    )
+
+    return {
+      deferredResult: Effect.fn("DeferredPersistence.deferredResult")(function*(deferred) {
+        const instance = yield* FlowRuntime.FlowInstance
+        const address = {
+          flowName: instance.flow._tag,
+          executionId: instance.executionId,
+          deferredName: deferred.name
+        }
+        const row = yield* state.deferred(address)
+        if (Option.isSome(row)) {
+          // Keep the result for replay, but stop registration from waking a
+          // run that already observed it and may now be parked on another wait.
+          // The first consumption is recorded in the same transaction.
+          const consumedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
+          yield* transactState(
+            Effect.flatMap(state.consumeDeferred(address, consumedAtMs), (consumed) =>
+              consumed
+                ? journal.emitDurableUnfenced(
+                  JournalRecords.deferredConsumed({
+                    runId: address.executionId,
+                    lineageId: FlowEngine.Lineage.root(address.executionId),
+                    sourceId: `${dependencies.journalSource}:deferred-consumed:${
+                      JSON.stringify([address.flowName, address.executionId, address.deferredName])
+                    }`,
+                    sourceSeq: 0
+                  }, { ...address, consumedAtMs })
+                ).pipe(Effect.asVoid)
+                : Effect.void)
+          ).pipe(Effect.orDie)
+        }
+        return Option.map(row, (value) => value.exit as Exit.Exit<unknown, unknown>)
+      }),
+      deferredDone: Effect.fn("DeferredPersistence.deferredDone")(completeDeferred),
+      deferredDoneIfWaiting: Effect.fn("DeferredPersistence.deferredDoneIfWaiting")((options) =>
+        persistDeferred(options, "deferred", {
+          reason: options.reason,
+          token: options.token
+        })
+      ),
+      scheduleClock,
+      sweepDue
+    }
+  })

package/src/internal/EffectRecords.ts

@@ -0,0 +1,136 @@
+/**
+ * Effect-boundary evidence, emitted by the engine at the point an irreversible
+ * action crosses into the outside world.
+ *
+ * Time travel (`packages/smithers/flows/time-travel/docs/README.md`) requires one durable
+ * journal entry recorded **before the adapter begins**, whose status then moves
+ * only forward (`intended` → `succeeded` | `unknown`). Rewind reads exactly
+ * these records to classify the doomed suffix as blocking, revertible, or
+ * warning; nothing else in the journal can answer that question, because an
+ * attempt row says an attempt happened, not that an effect left the machine.
+ *
+ * ## Why the shape lives here and not in `@smthrs/time-travel`
+ *
+ * `@smthrs/time-travel` depends on `@smthrs/engine-store` (it decodes
+ * `RunState`), so the arrow cannot be reversed without a cycle. The record is
+ * therefore a **wire contract**, not a shared type: this module writes the
+ * shape `EffectBoundary.decodeEntry` reads, under the event type that module
+ * owns, and `packages/smithers/flows/time-travel/test/EngineIntegration.test.ts` pins the two
+ * together by rewinding across a boundary record this module produced.
+ *
+ * ## One producer identity, two channels that must not double-count
+ *
+ * An irreversible dispatch now writes on two channels: the attempt lifecycle
+ * (`attempt-started` / `attempt-finished`), which says a durable attempt row
+ * moved, and this boundary (`intended` / terminal), which says an external
+ * effect was begun and how it settled. They are deliberately NOT merged —
+ * an attempt is retried under new numbers while an effect keeps one identity —
+ * and they cannot double-count, because a rewind's suffix scan matches only
+ * {@link eventType}. The two are tied by construction: the effect id IS the
+ * attempt identity (`runId:keyDigest:attempt`), so every boundary record names
+ * the attempt row that produced it and the audit reads as one story.
+ *
+ * @since 0.1.0
+ */
+
+import { JournalEvent } from "@smthrs/journal"
+
+/**
+ * The event type `@smthrs/time-travel` decodes boundary evidence from.
+ *
+ * Duplicated deliberately rather than imported: see the module note.
+ *
+ * @since 0.1.0
+ * @category constants
+ */
+export const eventType = "flows.time-travel.effect-boundary"
+
+/**
+ * Monotonic completion evidence recorded around an effect.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type Status = "intended" | "succeeded" | "unknown"
+
+/**
+ * What the engine knows about an effect before it begins.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Descriptor {
+  readonly id: string
+  readonly kind: string
+  readonly tier: "sealed" | "compensable" | "irreversible"
+  readonly runId: string
+  readonly lineageId: string
+  readonly sourceId: string
+  readonly attempt: number
+  readonly idempotencyKey?: string | undefined
+  readonly cacheKey?: string | undefined
+  readonly changeId?: string | undefined
+  /**
+   * The stable compensation descriptor the adapter owns, when it declares one.
+   * Time travel puts it in the entry so a
+   * rewind's handler resolution is a lookup over recorded evidence rather than
+   * a guess from the effect kind alone.
+   */
+  readonly compensation?: string | undefined
+  readonly residue?: string | undefined
+}
+
+/**
+ * Builds one boundary record for `descriptor` at `status`.
+ *
+ * Every effect and status pair gets its own producer identity: the status is
+ * appended to `descriptor.sourceId`, and every boundary record carries
+ * sourceSeq 0. Re-emitting the same status for the same effect therefore
+ * reuses that `(sourceId, sourceSeq)` pair, so a replay collapses into a
+ * journal `Duplicate` instead of appending a second boundary row, the same
+ * convergence the attempt lifecycle records rely on.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const boundary = (
+  descriptor: Descriptor,
+  status: Status,
+  output?: unknown
+): JournalEvent.Input =>
+  new JournalEvent.Input({
+    runId: descriptor.runId as JournalEvent.RunId,
+    sourceId: `${descriptor.sourceId}:effect:${descriptor.id}:${status}` as JournalEvent.SourceId,
+    sourceSeq: 0 as JournalEvent.SourceSeq,
+    eventType,
+    payload: {
+      version: 1,
+      effect: {
+        id: descriptor.id,
+        kind: descriptor.kind,
+        tier: descriptor.tier,
+        status,
+        runId: descriptor.runId,
+        lineageId: descriptor.lineageId,
+        attempt: descriptor.attempt,
+        durableBoundary: true,
+        providerStream: false,
+        ...(output === undefined ? {} : { output }),
+        ...(descriptor.idempotencyKey === undefined ? {} : { idempotencyKey: descriptor.idempotencyKey }),
+        ...(descriptor.cacheKey === undefined ? {} : { cacheKey: descriptor.cacheKey }),
+        ...(descriptor.changeId === undefined ? {} : { changeId: descriptor.changeId }),
+        ...(descriptor.compensation === undefined ? {} : { compensation: descriptor.compensation }),
+        ...(descriptor.residue === undefined ? {} : { residue: descriptor.residue })
+      }
+    },
+    meta: {
+      lineageId: descriptor.lineageId,
+      ...(descriptor.cacheKey === undefined ? {} : { cacheKey: descriptor.cacheKey }),
+      timeTravel: {
+        effectId: descriptor.id,
+        kind: descriptor.kind,
+        tier: descriptor.tier,
+        status
+      }
+    }
+  })

package/src/internal/EngineJj.ts

@@ -0,0 +1,30 @@
+/**
+ * Engine-private Jujutsu authority.
+ *
+ * Action bodies resolve the public `Jj` tag from their guarded host context.
+ * Snapshot, restore, and diff bookkeeping resolve this distinct tag instead,
+ * so an engine operation never grants the same authority to user code.
+ *
+ * @since 1.0.0
+ */
+
+import { Jj } from "@smthrs/kernel"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+
+/**
+ * Engine-owned repository operations, absent from public action context.
+ *
+ * @since 1.0.0
+ * @category services
+ */
+export class EngineJj extends Context.Service<EngineJj, Jj.Jj>()("@smthrs/engine-store/internal/EngineJj") {}
+
+/**
+ * Captures the current public Jj implementation under the private engine tag.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerFromJj: Layer.Layer<EngineJj, never, Jj.Jj> = Layer.effect(EngineJj, Effect.map(Jj.Jj, EngineJj.of))