@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/src/Inconsistency.ts

@@ -0,0 +1,253 @@
+/**
+ * Receiver for detected engine inconsistencies.
+ *
+ * Modeled on Skyframe's `GraphInconsistencyReceiver`: the store layer detects
+ * an invariant violation — today, a content-addressed cache key that produced
+ * a different result than the recorded row — and this service decides whether
+ * the run fails or tolerates it. Either way the conflict is journaled, so the
+ * detector is never wired to /dev/null.
+ *
+ * The `note` signature deliberately matches the plugin spec's
+ * `cacheInconsistency` hook (`(event) => Effect<InconsistencyVerdict>`); the
+ * strict layer is the future default plugin.
+ *
+ * The core default is strict so an invariant violation cannot pass silently.
+ *
+ * Governing designs: `docs/concepts/attempts-and-replay.md` and
+ * `docs/concepts/cache-admission.md`.
+ *
+ * @since 0.1.0
+ */
+
+import { FlowEngine } from "@smthrs/engine"
+import { Journal } from "@smthrs/journal"
+import type { Ownership } from "@smthrs/run-store"
+import type { CacheStore } from "@smthrs/step-cache"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as JournalRecords from "./internal/JournalRecords.ts"
+
+/**
+ * Decision returned by an inconsistency receiver.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type InconsistencyVerdict = "fail" | "tolerate"
+
+/**
+ * A content-addressed cache key that produced a result different from the
+ * recorded row — a hermeticity/determinism violation.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface CacheConflict {
+  readonly key: string
+  readonly existing: CacheStore.CacheEntry | undefined
+  readonly attempted: CacheStore.CacheEntry
+}
+
+/**
+ * Recorded boundary evidence whose blob no longer hashes to its recorded
+ * digest — on-disk corruption of the content-addressed store, detected while
+ * replaying a verified cache hit or a succeeded attempt (issue #150).
+ * Distinct from a transient host failure, which merely refuses the replay
+ * and stays retryable.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface BlobCorruption {
+  /** The run that observed the corruption; the journaling target. */
+  readonly runId: string
+  readonly keyDigest: string
+  readonly path: string
+  readonly recordedDigest: string
+  readonly measuredDigest: string
+  /**
+   * Provenance of the corrupt evidence when it came from a shared cache row;
+   * absent when it came from this run's own succeeded attempt row.
+   */
+  readonly recordedRunId?: string | undefined
+  readonly recordedEventSeq?: number | undefined
+}
+
+/**
+ * Inconsistency receiver operations.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Service {
+  readonly note: (event: CacheConflict) => Effect.Effect<InconsistencyVerdict, Journal.JournalError>
+  /**
+   * Observes on-disk corruption of recorded boundary evidence (issue #150).
+   * `"fail"` fails the dispatch; `"tolerate"` lets it fall back to a real
+   * execution (which re-captures and heals the corrupt address).
+   */
+  readonly noteCorruption: (event: BlobCorruption) => Effect.Effect<InconsistencyVerdict, Journal.JournalError>
+}
+
+/**
+ * Service tag for the engine inconsistency receiver.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class Inconsistency extends Context.Service<Inconsistency, Service>()("@smthrs/engine-store/Inconsistency") {}
+
+/**
+ * Options for constructing a journaling inconsistency receiver.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MakeOptions {
+  readonly journal: Journal.Service
+  readonly verdict: InconsistencyVerdict
+  /**
+   * The owner the conflict record is fenced to: a reclaimed run cannot append
+   * hermeticity evidence it no longer owns. Required — every composer of this
+   * receiver is a run's owner.
+   */
+  readonly owner: Ownership.OwnerId
+}
+
+/**
+ * Builds a receiver that journals every conflict under the run that attempted
+ * the write and returns a fixed verdict.
+ *
+ * The record goes through the journal's durable channel: a cache conflict is
+ * hermeticity evidence, and a `tolerate` verdict that silently dropped its
+ * only record would wire the detector to /dev/null (issue #10).
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (options: MakeOptions): Service => ({
+  note: Effect.fn("Inconsistency.note")((event) =>
+    Effect.as(
+      options.journal.emitDurable(
+        JournalRecords.cacheConflict(
+          {
+            runId: event.attempted.recordedRunId,
+            sourceId: "flows/engine-store/inconsistency",
+            lineageId: FlowEngine.Lineage.root(event.attempted.recordedRunId)
+          },
+          {
+            // `@smthrs/journal` redacts the credential word `key`. A durable
+            // content identity must use the specific, non-credential name.
+            cacheKey: event.key,
+            verdict: options.verdict,
+            existing: event.existing === undefined ? null : {
+              recordedRunId: event.existing.recordedRunId,
+              recordedEventSeq: event.existing.recordedEventSeq,
+              createdAtMs: event.existing.createdAtMs
+            },
+            attempted: {
+              recordedRunId: event.attempted.recordedRunId,
+              recordedEventSeq: event.attempted.recordedEventSeq,
+              createdAtMs: event.attempted.createdAtMs
+            }
+          }
+        ),
+        options.owner
+      ),
+      options.verdict
+    )
+  ),
+  noteCorruption: Effect.fn("Inconsistency.noteCorruption")((event) =>
+    Effect.as(
+      options.journal.emitDurable(
+        JournalRecords.cacheCorruption(
+          {
+            runId: event.runId,
+            lineageId: FlowEngine.Lineage.root(event.runId),
+            // The producer identity is the corruption evidence plus the
+            // recorded row generation (issues #156, #172): a caller
+            // re-observing the same corrupt row re-emits an exact producer
+            // duplicate the journal collapses, while an identically corrupt
+            // row recorded after eviction and healing has new provenance and
+            // lands as a distinct incident. Attempt-row evidence has no cache
+            // provenance, so its explicit local markers retain same-row
+            // convergence. A record already landed by another lineage under
+            // this identity surfaces as an idempotency conflict; either way
+            // evidence for that row generation exists exactly once.
+            sourceId:
+              `flows/engine-store/inconsistency:corruption:${event.keyDigest}:${event.path}:${event.recordedDigest}:${event.measuredDigest}:${
+                event.recordedRunId ?? "local"
+              }:${event.recordedEventSeq ?? "local"}`,
+            sourceSeq: 0
+          },
+          {
+            keyDigest: event.keyDigest,
+            verdict: options.verdict,
+            path: event.path,
+            recordedDigest: event.recordedDigest,
+            measuredDigest: event.measuredDigest,
+            recordedRunId: event.recordedRunId ?? null,
+            recordedEventSeq: event.recordedEventSeq ?? null
+          }
+        ),
+        options.owner
+      ).pipe(
+        Effect.catch((error) => error.code === "idempotency_conflict" ? Effect.succeed(undefined) : Effect.fail(error))
+      ),
+      options.verdict
+    )
+  )
+})
+
+/**
+ * Builds a receiver from overrides. The default notes nothing and tolerates.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeNoop = (overrides: Partial<Service> = {}): Service => ({
+  note: Effect.fn("Inconsistency.note")(() => Effect.succeed("tolerate" as const)),
+  noteCorruption: Effect.fn("Inconsistency.noteCorruption")(() => Effect.succeed("tolerate" as const)),
+  ...overrides
+})
+
+/**
+ * Provides a receiver that never journals and always tolerates.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<Inconsistency> =>
+  Layer.succeed(Inconsistency)(makeNoop(overrides))
+
+/**
+ * Journals every cache conflict and fails the run. This is the default
+ * receiver for engine wiring: a non-hermetic sealed hard-boundary action is
+ * a defect, not a condition to paper over.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerStrict = (owner: Ownership.OwnerId): Layer.Layer<Inconsistency, never, Journal.Journal> =>
+  Layer.effect(Inconsistency)(
+    Effect.gen(function*() {
+      const journal = yield* Journal.Journal
+      return make({ journal, verdict: "fail", owner })
+    })
+  )
+
+/**
+ * Journals every cache conflict and continues, preserving the first-recorded
+ * row (Skyframe's tolerant production configuration).
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerTolerant = (owner: Ownership.OwnerId): Layer.Layer<Inconsistency, never, Journal.Journal> =>
+  Layer.effect(Inconsistency)(
+    Effect.gen(function*() {
+      const journal = yield* Journal.Journal
+      return make({ journal, verdict: "tolerate", owner })
+    })
+  )

package/src/Migrations.ts

@@ -0,0 +1,95 @@
+/**
+ * Durable engine schema migrations.
+ *
+ * Engine-store owns persisted deferred/clock state, execution selections,
+ * and append-only plan-input observations. These are authoritative execution
+ * records, distinct from journal projections and evictable step-cache rows.
+ * It reserves migration id block `3000`.
+ *
+ * Because engine-store composes the journal, the run store, and the step
+ * cache, {@link sets} is also the full durable schema an engine needs, and
+ * {@link layer} installs it in dependency order.
+ *
+ * Derived contracts: `docs/api.md` ("Migrations") and
+ * `packages/smithers/flows/journal/docs/api.md`.
+ *
+ * @since 0.1.0
+ */
+
+import * as DatabaseMigrations from "@smthrs/database/Migrations"
+import * as JournalMigrations from "@smthrs/journal/Migrations"
+import * as PlanMigrations from "@smthrs/plan-store/Migrations"
+import * as RunStoreMigrations from "@smthrs/run-store/Migrations"
+import * as StepCacheMigrations from "@smthrs/step-cache/Migrations"
+import * as Layer from "effect/Layer"
+import { initial } from "./migrations/0001_initial.ts"
+import { selectionStore } from "./migrations/0002_selection_store.ts"
+import { planInputs } from "./migrations/0003_plan_inputs.ts"
+import { planEnvironment } from "./migrations/0004_plan_environment.ts"
+import { planMerges } from "./migrations/0005_plan_merges.ts"
+import { executionListing } from "./migrations/0006_execution_listing.ts"
+import { runParentSequence } from "./migrations/0007_run_parent_sequence.ts"
+import { deferredConsumption } from "./migrations/0008_deferred_consumption.ts"
+import { deferredClockFacts } from "./migrations/0009_deferred_clock_facts.ts"
+import { pendingCancellation } from "./migrations/0010_pending_cancellation.ts"
+
+/**
+ * Engine-store's own namespaced migration set.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const set: DatabaseMigrations.MigrationSet = {
+  namespace: "engine-store",
+  idOffset: DatabaseMigrations.idBlock * 3,
+  migrations: {
+    "0001_initial": initial,
+    "0002_selection_store": selectionStore,
+    "0003_plan_inputs": planInputs,
+    "0004_plan_environment": planEnvironment,
+    "0005_plan_merges": planMerges,
+    "0006_execution_listing": executionListing,
+    "0007_run_parent_sequence": runParentSequence,
+    "0008_deferred_consumption": deferredConsumption,
+    "0009_deferred_clock_facts": deferredClockFacts,
+    "0010_pending_cancellation": pendingCancellation
+  }
+}
+
+/**
+ * Every migration set a durable engine needs, in dependency order: journal
+ * events, run and attempt state, the step cache, the engine's own deferred and
+ * clock, selection, and input tables, then the persisted plan.
+ *
+ * The plan set has the highest id block (`4000`). The database loader applies
+ * forward additions within already installed blocks transactionally before
+ * the ordinary migration pass (for example, engine-store `3003` after plan
+ * `4003`). Earlier holes and entirely new lower blocks remain refusals.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const sets: ReadonlyArray<DatabaseMigrations.MigrationSet> = [
+  JournalMigrations.set,
+  RunStoreMigrations.set,
+  StepCacheMigrations.set,
+  set,
+  PlanMigrations.set
+]
+
+/**
+ * Creates the complete durable engine schema.
+ *
+ * @category migrations
+ * @since 0.1.0
+ */
+export const run = DatabaseMigrations.run(sets)
+
+/**
+ * Layer that installs the complete durable engine schema before exposing the
+ * database to any durable service.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = Layer.effectDiscard(run)

package/src/OwnerIdentity.ts

@@ -0,0 +1,125 @@
+/**
+ * The source of the fencing identity a store incarnation runs under.
+ *
+ * Minting an {@link OwnerId} is an act of nondeterminism against the host: it
+ * reads a process identifier and draws a fresh nonce. Both belong behind a
+ * port rather than in the composition itself — the closed-host doctrine in
+ * `@smthrs/kernel`'s `HostServices` admits no ambient `process` and no static
+ * `node:crypto` import, and browser support is a hard requirement, so a store
+ * that reached for either directly could not be composed in a tab at all.
+ *
+ * Governing design: `docs/concepts/ownership-and-fencing.md`.
+ *
+ * @since 0.1.0
+ */
+
+import type { OwnerId } from "@smthrs/journal/OwnerId"
+import * as Context from "effect/Context"
+import * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Random from "effect/Random"
+
+/**
+ * Owner identity operations.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Service {
+  /**
+   * Mints the identity this incarnation fences its durable writes with. The
+   * host supplies the incarnation; the caller supplies the host id, because
+   * which host a store speaks for is a composition decision, not a host fact.
+   */
+  readonly ownerId: (hostId: string) => Effect.Effect<OwnerId>
+}
+
+/**
+ * Service tag for the owner identity source.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class OwnerIdentity extends Context.Service<OwnerIdentity, Service>()("@smthrs/engine-store/OwnerIdentity") {}
+
+/**
+ * Constructs an owner identity source from an implementation.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (service: Service): Service => OwnerIdentity.of(service)
+
+/**
+ * The host's process identifier, when the platform has one.
+ *
+ * Read off `globalThis` rather than through a bare `process` reference so the
+ * module carries no Node binding at all: a browser bundle sees `undefined`
+ * here and falls through to a drawn incarnation number instead. The read
+ * happens at layer construction — {@link layer} — so importing the module
+ * touches nothing.
+ */
+const hostProcessId = (): number | undefined =>
+  (globalThis as { readonly process?: { readonly pid?: number } }).process?.pid
+
+/**
+ * Builds the standard identity source over an explicitly supplied process id
+ * and the injected `Crypto` service.
+ *
+ * On node and bun `pid` is the real process id, so the minted token is exactly
+ * what the store produced before this service existed. A browser tab has no
+ * process, and no stable per-tab integer that means the same thing, so
+ * `undefined` draws an incarnation number instead: the component's only job is
+ * to distinguish concurrent incarnations on one host, and `Random` is the
+ * sanctioned port for drawing one. Effect's own cluster storage does the same
+ * where a backend exposes no connection pid (`reference/effect`
+ * `unstable/cluster/SqlRunnerStorage.ts`).
+ *
+ * The nonce is a UUIDv4 drawn from the injected `effect/Crypto` service — the
+ * closed-host seam every composition already provides for digesting — so no
+ * ambient `crypto` global is reached from a hot path. A host whose crypto
+ * cannot mint a UUID cannot fence anything, so that refusal is a defect.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeIncarnation = (pid: number | undefined, crypto: Crypto.Crypto): Service => {
+  const incarnationId: Effect.Effect<number> = pid === undefined
+    ? Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })
+    : Effect.succeed(pid)
+  return make({
+    ownerId: Effect.fn("OwnerIdentity.ownerId")(function*(hostId) {
+      yield* Effect.annotateCurrentSpan({ hostId })
+      return {
+        hostId,
+        pid: yield* incarnationId,
+        nonce: yield* crypto.randomUUIDv4.pipe(Effect.orDie)
+      } satisfies OwnerId
+    })
+  })
+}
+
+/**
+ * Provides {@link makeIncarnation} over whatever process id the running
+ * platform reports and the composition's `Crypto` service. This is what
+ * engine composition supplies unless a host has a better answer.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer: Layer.Layer<OwnerIdentity, never, Crypto.Crypto> = Layer.effect(
+  OwnerIdentity,
+  Effect.map(Crypto.Crypto, (crypto) => makeIncarnation(hostProcessId(), crypto))
+)
+
+/**
+ * Provides a fixed identity. A test — or a host that derives ownership from
+ * something outside this process, such as a lease it already holds — pins the
+ * whole token instead of letting the default draw one.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerConstant = (owner: OwnerId): Layer.Layer<OwnerIdentity> =>
+  Layer.succeed(OwnerIdentity)(make({ ownerId: () => Effect.succeed(owner) }))