@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/src/StepCacheFold.ts

@@ -0,0 +1,244 @@
+/**
+ * The step-cache fold: `flows_step_cache` heads rebuilt from the journal.
+ *
+ * A head exists only because a `CacheStore.put` inserted it, and
+ * `ActionPersistence` journals exactly that insertion as a
+ * `flows.engine.cache-provenance` entry with action `admitted`, in the same
+ * `journal.transact` as the `put`. A duplicate or a losing first-writer race
+ * writes a ledger row but no head, so it journals no `admitted` entry. Every
+ * engine eviction is journalled before the fenced `CacheStore.evict` that
+ * performs it: `expired`, `stale_read_set`, and `replay_failed` with reason
+ * `corruption`, each naming the row it retires. The fold retires a row on any
+ * `replay_failed` that names it: a producer slot keeps the first reason it
+ * journalled, so a corruption can hide behind an earlier host failure. A row
+ * whose replay failed only on the host is then a miss.
+ *
+ * The fold keeps the admitted rows no eviction retired. A key with one such
+ * row gets it back as its head. A key with several lost at least one of them
+ * without a journal entry (`sweepExpired`, retention, an unfenced `evict`), and
+ * the journal cannot say which one the store still held, so it gets no head.
+ *
+ * Cache results stay in the append-only `flows_step_cache_recorded` ledger
+ * that the `admitted` entry addresses. The journal is redacted and replayed to
+ * sync subscribers, so it carries the decision and the provenance, never the
+ * executable result (issue #72).
+ *
+ * A rebuild writes a subset of the live heads, apart from rows removed without
+ * a journal entry, whose restored hits still pass the engine's age, read-set,
+ * and digest checks. Shared-tier write-backs have no local entry and are not
+ * rebuilt. Compaction and rewind delete eviction entries, so a store holding
+ * a compacted run, or one where any run was ever rewound, gets no heads at all.
+ * Retention is not yet covered: it deletes a run's entries, and with them the
+ * compaction floor and any eviction that run journalled of another run's row,
+ * so a rebuild after retention can restore a row the engine evicted (issue
+ * #2053). A convergence re-record that reuses an evicted row's provenance stays
+ * retired, so that key misses.
+ *
+ * @since 1.0.0
+ */
+
+import { Journal, type JournalEvent, Projection } from "@smthrs/journal"
+import { CacheStore } from "@smthrs/step-cache"
+import * as Effect from "effect/Effect"
+import * as HashMap from "effect/HashMap"
+import * as HashSet from "effect/HashSet"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * The journal event type the fold reads.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const eventType = "flows.engine.cache-provenance"
+
+/**
+ * Provenance actions that retire the row they name.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const retiringActions: ReadonlyArray<string> = ["expired", "stale_read_set", "replay_failed"]
+
+/**
+ * Admitted rows folded so far, and the ones the journal retired.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface State {
+  /** Every row a `put` inserted as a head, by {@link refId}. */
+  readonly admitted: HashMap.HashMap<string, CacheStore.RecordedRef>
+  /** Rows an engine eviction retired, by {@link refId}. */
+  readonly retired: HashSet.HashSet<string>
+}
+
+/**
+ * The stable identity of a recorded row.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export const refId = (ref: CacheStore.RecordedRef): string =>
+  JSON.stringify([ref.keyDigest, ref.recordedRunId, ref.recordedEventSeq])
+
+const record = (value: unknown): Readonly<Record<string, unknown>> | undefined =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+    ? value as Readonly<Record<string, unknown>>
+    : undefined
+
+/** The row a payload names by `keyDigest`/`recordedRunId`/`recordedEventSeq`, if it names one. */
+const named = (payload: Readonly<Record<string, unknown>>): CacheStore.RecordedRef | undefined =>
+  typeof payload.keyDigest === "string" && typeof payload.recordedRunId === "string" &&
+    typeof payload.recordedEventSeq === "number"
+    ? {
+      keyDigest: payload.keyDigest,
+      recordedRunId: payload.recordedRunId,
+      recordedEventSeq: payload.recordedEventSeq
+    }
+    : undefined
+
+const retires = (payload: Readonly<Record<string, unknown>>): boolean =>
+  retiringActions.includes(payload.action as string)
+
+/**
+ * Folds one committed journal entry.
+ *
+ * @category folds
+ * @since 1.0.0
+ */
+export const reduce = (state: State, entry: JournalEvent.Entry): State => {
+  const payload = record(entry.payload)
+  const ref = entry.eventType === eventType && payload !== undefined ? named(payload) : undefined
+  if (ref === undefined) return state
+  if (payload!.action === "admitted") return { ...state, admitted: HashMap.set(state.admitted, refId(ref), ref) }
+  return retires(payload!) ? { ...state, retired: HashSet.add(state.retired, refId(ref)) } : state
+}
+
+/**
+ * The empty fold.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const initial: State = { admitted: HashMap.empty(), retired: HashSet.empty() }
+
+/**
+ * The fold as a journal projection, for `Journal.project` over one run.
+ *
+ * @category folds
+ * @since 1.0.0
+ */
+export const projection: Projection.Projection<State> = Projection.make({
+  name: "flows.engine.step-cache",
+  initial,
+  reduce: (state, entry) => Effect.succeed(reduce(state, entry))
+})
+
+/**
+ * The head each key may be rebuilt with: its one admitted row no eviction
+ * retired. A key with more than one such row gets none.
+ *
+ * @category folds
+ * @since 1.0.0
+ */
+export const candidates = (state: State): ReadonlyArray<CacheStore.RecordedRef> => {
+  const byKey = new Map<string, Array<CacheStore.RecordedRef>>()
+  for (const [id, ref] of state.admitted) {
+    if (HashSet.has(state.retired, id)) continue
+    const refs = byKey.get(ref.keyDigest)
+    if (refs === undefined) byKey.set(ref.keyDigest, [ref])
+    else refs.push(ref)
+  }
+  return [...byKey.values()].flatMap((refs) => refs.length === 1 ? refs : [])
+}
+
+/**
+ * What a {@link rebuild} read and wrote.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Rebuilt {
+  /** Runs with cache provenance entries. */
+  readonly runs: number
+  /** Runs with a compaction floor. Any makes the rebuild write no heads. */
+  readonly compacted: number
+  /** Runs a rewind truncated. Any makes the rebuild write no heads. */
+  readonly rewound: number
+  readonly admitted: number
+  readonly retired: number
+  readonly heads: number
+}
+
+const pageSize = 512
+
+/** Folds one run's cache provenance entries into `state`, in sequence order. */
+const foldRun = (journal: Journal.Service, runId: string, start: State) =>
+  Effect.gen(function*() {
+    let state = start
+    let after: JournalEvent.Seq | undefined
+    while (true) {
+      const page = yield* journal.entries({
+        runId: runId as JournalEvent.RunId,
+        eventTypes: [eventType],
+        limit: pageSize,
+        ...(after === undefined ? {} : { after })
+      })
+      for (const entry of page.entries) {
+        state = reduce(state, entry)
+        after = entry.seq
+      }
+      if (!page.hasMore) return state
+    }
+  })
+
+/**
+ * Rebuilds every `flows_step_cache` head from the journal.
+ *
+ * Reads each run's cache provenance entries in sequence order, folds them,
+ * and replaces the heads through `CacheStore.rebuildHeads` in one writer
+ * transaction. A store with a compacted or rewound run clears the heads
+ * instead: the eviction that retired a row may be gone. A journal read failure, including
+ * a compaction that lands during the read, leaves the heads untouched.
+ *
+ * Run it on a quiescent store, as a restore does: a cache write that commits
+ * between the journal read and the rewrite is dropped from the heads (a miss,
+ * never a wrong hit).
+ *
+ * @category operations
+ * @since 1.0.0
+ */
+export const rebuild = Effect.fn("StepCacheFold.rebuild")(function*() {
+  const journal = yield* Journal.Journal
+  const sql = yield* SqlClient.SqlClient
+  const floors = yield* sql<{ readonly compacted: number | string }>`
+    SELECT COUNT(DISTINCT run_id) AS compacted FROM flows_journal_checkpoints
+    WHERE compacted_at_ms IS NOT NULL
+  `
+  const compacted = Number(floors[0]!.compacted)
+  const generations = yield* sql<{ readonly rewound: number | string }>`
+    SELECT COUNT(*) AS rewound FROM flows_journal_generations
+  `
+  const rewound = Number(generations[0]!.rewound)
+  const runs = yield* sql<{ readonly run_id: string }>`
+    SELECT DISTINCT run_id FROM flows_journal_events
+    WHERE event_type = ${eventType}
+    ORDER BY run_id
+  `
+  let state = initial
+  if (compacted === 0 && rewound === 0) {
+    for (const { run_id } of runs) {
+      state = yield* foldRun(journal, run_id, state)
+    }
+  }
+  const heads = yield* CacheStore.rebuildHeads(candidates(state))
+  return {
+    runs: runs.length,
+    compacted,
+    rewound,
+    admitted: HashMap.size(state.admitted),
+    retired: HashSet.size(state.retired),
+    heads
+  } satisfies Rebuilt
+})

package/src/StepSandbox.ts

@@ -0,0 +1,110 @@
+/**
+ * Injectable, scope-safe step sandboxing over the workspace transaction.
+ *
+ * @since 0.1.0
+ */
+
+import type * as ArtifactStore from "@smthrs/artifacts/ArtifactStore"
+import type * as KernelWorkspace from "@smthrs/kernel/Workspace"
+import * as Context from "effect/Context"
+import type * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import type * as FileSystem from "effect/FileSystem"
+import * as Layer from "effect/Layer"
+import type * as Path from "effect/Path"
+import * as Schema from "effect/Schema"
+import * as StepBoundary from "./StepBoundary.ts"
+import * as WorkspaceSandbox from "./WorkspaceSandbox.ts"
+
+/**
+ * A hermetic body attempted to read a path outside its declared read set.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export class UndeclaredRead extends Schema.TaggedError<UndeclaredRead>()(
+  "@smthrs/engine-store/UndeclaredRead",
+  {
+    code: Schema.Literal("undeclared_read"),
+    paths: Schema.Array(Schema.String),
+    diffIdentity: Schema.NonEmptyString
+  }
+) {}
+
+/**
+ * A sandbox that can acquire an isolated workspace for one step.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Service {
+  readonly open: Effect.Effect<WorkspaceSandbox.Service, StepBoundary.UnsupportedBoundary>
+}
+
+/**
+ * Injectable step-sandbox service.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export const StepSandbox: Context.Service<Service, Service> = Context.Service<Service>(
+  "@smthrs/engine-store/StepSandbox"
+)
+
+/**
+ * Constructs a step sandbox around a workspace transaction backend.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (workspace: WorkspaceSandbox.Service): Service =>
+  StepSandbox.of({
+    open: Effect.succeed(workspace)
+  })
+
+/**
+ * Filesystem-backed sandbox layer. The workspace transaction seeds only the
+ * declared reads, exposes its scoped FileSystem to the body, observes every
+ * mutation, and performs atomic copy-back on release. Built from
+ * `WorkspaceSandbox.layerFileSystem`, so it refuses a path-based host the same way.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer: Layer.Layer<
+  Service,
+  WorkspaceSandbox.WorkspaceError,
+  FileSystem.FileSystem | Path.Path | ArtifactStore.ArtifactStore | KernelWorkspace.Workspace
+> = Layer.effect(
+  StepSandbox,
+  Effect.map(WorkspaceSandbox.WorkspaceSandbox, make)
+).pipe(Layer.provide(WorkspaceSandbox.layerFileSystem()))
+
+/**
+ * Deterministic in-memory sandbox layer for tests.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerTest = (
+  initialFiles: WorkspaceSandbox.InitialFiles = {}
+): Layer.Layer<Service, WorkspaceSandbox.WorkspaceError, Crypto.Crypto> =>
+  Layer.effect(StepSandbox, Effect.map(WorkspaceSandbox.makeMemory(initialFiles), (sandbox) => make(sandbox.service)))
+
+const unsupported = new StepBoundary.UnsupportedBoundary({
+  code: "unsupported_boundary",
+  message: "this host cannot provide a hermetic step sandbox"
+})
+
+/**
+ * Fail-closed host layer for environments that cannot sandbox (for example browsers).
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop: Layer.Layer<Service> = Layer.succeed(
+  StepSandbox,
+  StepSandbox.of({
+    open: Effect.fail(unsupported)
+  })
+)

package/src/WakeBus.ts

@@ -0,0 +1,167 @@
+/**
+ * The in-process wake bus: durable writes that make a run runnable — a
+ * deferred completed, a durable clock fired, an operator resume — announce
+ * the run id here, and in-process waiters parked on that run resume
+ * immediately instead of waiting out a poll tick.
+ *
+ * The bus is EDGE-TRIGGERED and MISS-TOLERANT by design. A wake published
+ * while nobody waits is dropped, not queued: durable state already records
+ * that the run is runnable, and the engine's suspension polling schedule
+ * (`suspendedRetryPolicy`) remains the bounded fallback that re-drives any
+ * run whose wake the bus missed. Temporal's scheduled task queue takes the
+ * same position — `NotifyNewTasks` is a coalesced, non-blocking channel send
+ * and `MaxPollInterval` bounds the polling fallback (`reference/temporal`
+ * `service/history/queues/queue_scheduled.go`) — because a lossless bus
+ * would have to be durable, and the durable record already exists. Waiter
+ * registration follows opencode's in-process job registry: one `Deferred`
+ * per waiter, succeeded on completion, removed when the waiter's own scope
+ * closes (`reference/opencode` `packages/smithers/flows/core/src/background-job.ts`).
+ *
+ * Cross-process wakes never reach this bus: another process's deferred
+ * completion is observed through the store, on the polling schedule and the
+ * heartbeat sweeps. That split is deliberate v1 scope — an event-driven
+ * cross-process wake needs a committed journal subscription, which is a
+ * separate seam.
+ *
+ * The bus holds no host access at all — a `Map` of `Deferred`s — so it is
+ * browser-safe without a platform layer.
+ *
+ * @since 0.1.0
+ */
+
+import * as Context from "effect/Context"
+import * as Deferred from "effect/Deferred"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+
+/**
+ * Wake bus operations.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Service {
+  /**
+   * Announces that a durable write made the run runnable. Every waiter
+   * currently parked on `executionId` resumes; with no waiters the wake is
+   * dropped and the polling fallback covers the run.
+   */
+  readonly wake: (executionId: string) => Effect.Effect<void>
+  /**
+   * Parks until the next {@link wake} for `executionId`. Registration is
+   * removed when the waiting fiber is interrupted — including scope closure
+   * and losing a race against the polling sleep — so an abandoned wait
+   * leaks nothing.
+   *
+   * The wait is keyed by execution id alone: execution ids are unique
+   * across flows, so the flow tag adds no discrimination.
+   */
+  readonly awaitWake: (executionId: string) => Effect.Effect<void>
+  /**
+   * How many waiters are currently parked on `executionId`. This is
+   * observability for tests and diagnostics, not a coordination primitive:
+   * a count observed now says nothing about a moment later.
+   */
+  readonly waiters: (executionId: string) => Effect.Effect<number>
+}
+
+/**
+ * Service tag for the in-process wake bus.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class WakeBus extends Context.Service<WakeBus, Service>()("@smthrs/engine-store/WakeBus") {}
+
+/**
+ * Constructs a wake bus. Registration and delivery are plain in-memory
+ * operations on the constructing process; nothing is durable.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeUnsafe = (): Service => {
+  const parked = new Map<string, Set<Deferred.Deferred<void>>>()
+  return WakeBus.of({
+    wake: (executionId) =>
+      Effect.suspend(() => {
+        const waiters = parked.get(executionId)
+        if (waiters === undefined) return Effect.void
+        // Snapshot before succeeding: each woken waiter's release removes it
+        // from the live set, and delivery must not observe that mutation.
+        return Effect.forEach(
+          Array.from(waiters),
+          (waiter) => Deferred.succeed(waiter, undefined),
+          { discard: true }
+        )
+      }),
+    awaitWake: (executionId) =>
+      Effect.acquireUseRelease(
+        Effect.sync(() => {
+          const waiter = Deferred.makeUnsafe<void>()
+          let waiters = parked.get(executionId)
+          if (waiters === undefined) {
+            waiters = new Set()
+            parked.set(executionId, waiters)
+          }
+          waiters.add(waiter)
+          return { waiter, waiters }
+        }),
+        ({ waiter }) => Deferred.await(waiter),
+        // The registration's own set is closed over rather than re-fetched:
+        // the map entry can only be deleted by the release that empties it,
+        // so the set this waiter joined is the set it must leave.
+        ({ waiter, waiters }) =>
+          Effect.sync(() => {
+            waiters.delete(waiter)
+            if (waiters.size === 0) parked.delete(executionId)
+          })
+      ),
+    waiters: (executionId) => Effect.sync(() => parked.get(executionId)?.size ?? 0)
+  })
+}
+
+/**
+ * Constructs a wake bus effectfully. Prefer this in compositions;
+ * {@link makeUnsafe} exists for construction outside a fiber.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make: Effect.Effect<Service> = Effect.sync(makeUnsafe)
+
+/**
+ * A bus that never delivers: wakes are dropped and waiters park forever.
+ * With it in place every resume travels the polling fallback — which is
+ * exactly what a test asserting the fallback provides it for.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeNoop = (overrides?: Partial<Service>): Service =>
+  WakeBus.of({
+    wake: () => Effect.void,
+    awaitWake: () => Effect.never,
+    waiters: () => Effect.succeed(0),
+    ...overrides
+  })
+
+/**
+ * Provides a fresh wake bus. An engine composition resolves the bus
+ * optionally, so providing this layer is how a host shares one bus between
+ * the engine and its own wake sources; a composition given none builds a
+ * private bus.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer: Layer.Layer<WakeBus> = Layer.sync(WakeBus)(makeUnsafe)
+
+/**
+ * Provides {@link makeNoop}.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop = (overrides?: Partial<Service>): Layer.Layer<WakeBus> =>
+  Layer.sync(WakeBus)(() => makeNoop(overrides))