@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/src/internal/RunCatalogOps.ts

@@ -0,0 +1,174 @@
+/**
+ * The workspace run catalog read.
+ *
+ * `@smthrs/sync` serves a workspace subscription over a `RunCatalog`, and
+ * every implementation of that service was static (`layerStatic`,
+ * `layerNoop`) or in-process (`makeMemory`, which hears only about runs the
+ * same process registered). A follower composed against any of them saw the
+ * runs that existed when it started and never learned of another run, so a
+ * second engine writing to the same workspace was invisible to it. This
+ * module is the durable half of the fix: the read that says which runs the
+ * workspace has right now, taken from `flows_runs` itself.
+ *
+ * The read is a full set rather than a cursor tail, for one reason: retention
+ * deletes runs. A follower that only ever appended what appeared after a
+ * cursor would keep every collected run in its view forever and hold a
+ * journal stream open for each one. Reading the set makes both directions
+ * work, and the workspace's own bound keeps the cost of it flat.
+ *
+ * Nothing here polls. The interval belongs to the catalog that consumes this
+ * read (`RunCatalog.makePolling` in `@smthrs/sync`), so the durable side has
+ * no policy and no fiber of its own.
+ *
+ * @since 0.1.0
+ */
+
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+import * as RunListing from "./RunListing.ts"
+import { RunCatalogError } from "./RunListingError.ts"
+
+/**
+ * Stable error codes returned by the catalog read.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export { RunCatalogErrorCode } from "./RunListingError.ts"
+
+/**
+ * A catalog read that could not complete.
+ *
+ * `list_failed` means the workspace's run set could not be read. The caller
+ * keeps whatever view it already had; the read is idempotent, so the next one
+ * converges once the cause is fixed.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export { RunCatalogError } from "./RunListingError.ts"
+
+/**
+ * Options for one catalog read.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface ListOptions {
+  /**
+   * Largest number of runs the read returns. Defaults to
+   * {@link defaultLimit}. A workspace with more runs than the bound is
+   * followed by its most recent ones; an older run stays readable through a
+   * run-scoped subscription, which names it directly.
+   */
+  readonly limit?: number | undefined
+}
+
+/**
+ * The workspace's run set.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Service {
+  /** Revision-consistent, filtered keyset page of execution rows. */
+  readonly listRuns: RunListing.Service["listRuns"]
+  /**
+   * Every run the workspace has, oldest first, bounded by the read. "Oldest"
+   * is the run table's own row order, which is the order `RunStore.create`
+   * inserted them: SQLite hands a new row an identifier above every one in
+   * the table, and reuses one only after the highest is deleted, so deletion
+   * by retention does not reorder what is left.
+   */
+  readonly listRunIds: (options?: ListOptions) => Effect.Effect<ReadonlyArray<string>, RunCatalogError>
+}
+
+/**
+ * Service tag for the workspace run catalog read.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class RunCatalogRead extends Context.Service<RunCatalogRead, Service>()(
+  "@smthrs/engine-store/RunCatalogRead"
+) {}
+
+/**
+ * Runs one read returns when the caller names no bound. Large enough that an
+ * ordinary workspace is returned whole, small enough that a workspace which
+ * has never been collected does not hand a follower an unbounded list.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const defaultLimit = 10_000
+
+/**
+ * Builds the catalog read over the composition's own run table.
+ *
+ * The read goes to `flows_runs` directly, the way `DurableEngineState`,
+ * `ArtifactGc`, and `RetentionOps` do: this package composes the run-store
+ * migrations, so the schema is its own.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (): Effect.Effect<Service, never, SqlClient.SqlClient> =>
+  Effect.gen(function*() {
+    const sql = yield* Effect.service(SqlClient.SqlClient)
+    const listing = yield* RunListing.make()
+
+    // Newest first by `rowid`, then reversed. `rowid` is the order the
+    // workspace gained its runs, and reading it backwards walks the table's
+    // own b-tree, so the bound costs one seek rather than a sort of every
+    // run: `flows_runs` carries no index on `created_at_ms`, and two runs
+    // created in the same millisecond do not order by it at all.
+    //
+    // `flows_runs` has a TEXT primary key, so its `rowid` is not an alias of
+    // a declared column and SQLite documents that a rebuild may reassign it.
+    // The order the rebuild leaves is what this read depends on, not the
+    // numbers, and `VACUUM INTO` — the hot backup this release ships, and the
+    // only rebuild in the supported path (the release policy) — preserves
+    // it. `RunCatalogCatchUp.test.ts` pins that across a real backup and
+    // reopen, including after retention has deleted rows out of the middle
+    // and the end.
+    const listRunIds: Service["listRunIds"] = Effect.fn("RunCatalog.listRunIds")((options?: ListOptions) =>
+      Effect.gen(function*() {
+        const limit = options?.limit ?? defaultLimit
+        if (!Number.isSafeInteger(limit) || limit < 0) {
+          return yield* Effect.fail(
+            new RunCatalogError({
+              code: "invalid_options",
+              message: "the run catalog limit must be a non-negative safe integer"
+            })
+          )
+        }
+        return yield* sql<{ readonly run_id: string }>`
+          SELECT run_id FROM flows_runs
+          ORDER BY rowid DESC
+          LIMIT ${limit}
+        `.pipe(
+          Effect.map((rows) => rows.map((row) => row.run_id).reverse()),
+          Effect.mapError((cause) =>
+            new RunCatalogError({
+              code: "list_failed",
+              message: "the workspace run set could not be read",
+              cause
+            })
+          )
+        )
+      })
+    )
+
+    return { listRunIds, ...listing }
+  })
+
+/**
+ * Provides the workspace run catalog read.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer: Layer.Layer<RunCatalogRead, never, SqlClient.SqlClient> = Layer.effect(RunCatalogRead)(make())

package/src/internal/RunCoordinator.ts

@@ -0,0 +1,215 @@
+/**
+ * Coordinates keyed drain executions.
+ *
+ * Adapted nearly verbatim from
+ * `reference/opencode/packages/smithers/flows/core/src/session/run-coordinator.ts`:
+ * `Deferred` joins, `FiberSet` ownership, coalesced wakes, and
+ * `uninterruptibleMask` preserve its start-or-join semantics. This adaptation
+ * drops the source's `force` drain argument because no engine-store drain has
+ * a second execution mode; the successor flag carries coordinator scheduling.
+ *
+ * @since 0.1.0
+ */
+
+import { Cause, Deferred, Effect, Exit, Fiber, FiberSet, type Scope } from "effect"
+
+/**
+ * Serializes drain execution for each key while allowing distinct keys to run
+ * concurrently.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface RunCoordinator<Key, E> {
+  /**
+   * Snapshots keys with an execution owned by this coordinator.
+   *
+   * @since 0.1.0
+   * @category operations
+   */
+  readonly active: Effect.Effect<ReadonlySet<Key>>
+  /**
+   * Starts a drain while idle or joins the active drain for the key.
+   *
+   * @since 0.1.0
+   * @category operations
+   */
+  readonly run: (key: Key) => Effect.Effect<void, E>
+  /**
+   * Joins the active drain for the key, and starts none while idle.
+   *
+   * @since 1.0.0
+   * @category operations
+   */
+  readonly join: (key: Key) => Effect.Effect<void, E>
+  /**
+   * Starts an idle key without joining or requesting another active drain.
+   *
+   * @since 1.0.0
+   * @category operations
+   */
+  readonly schedule: (key: Key) => Effect.Effect<void>
+  /** Reserves an idle key, then starts its drain only while the supplied durable readiness holds. */
+  readonly scheduleIf: (key: Key, ready: Effect.Effect<boolean>) => Effect.Effect<void>
+  /**
+   * Ensures one coalesced drain follows the active drain for the key.
+   *
+   * @since 0.1.0
+   * @category operations
+   */
+  readonly wake: (key: Key) => Effect.Effect<void>
+  /**
+   * Interrupts the active drain for the key and waits for its cleanup.
+   *
+   * @since 0.1.0
+   * @category operations
+   */
+  readonly interrupt: (key: Key) => Effect.Effect<void>
+  /** Sends interruption without awaiting cleanup; safe for a short commit publication. */
+  readonly requestInterrupt: (key: Key) => Effect.Effect<void>
+}
+
+type Entry<E> = {
+  readonly done: Deferred.Deferred<void, E>
+  owner?: Fiber.Fiber<void, never>
+  pendingWake: boolean
+  stopping: boolean
+}
+
+/**
+ * Creates a scoped coordinator for keyed drain effects.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const make = <Key, E, R>(options: {
+  readonly drain: (key: Key) => Effect.Effect<void, E, R>
+}): Effect.Effect<RunCoordinator<Key, E>, never, Scope.Scope | R> =>
+  Effect.gen(function*() {
+    const active = new Map<Key, Entry<E>>()
+    const fork = yield* FiberSet.makeRuntime<R, void, never>()
+
+    const makeEntry = (): Entry<E> => ({
+      done: Deferred.makeUnsafe<void, E>(),
+      pendingWake: false,
+      stopping: false
+    })
+
+    const start = (
+      key: Key,
+      entry: Entry<E>,
+      successor = false,
+      onlyIf: Effect.Effect<boolean> = Effect.succeed(true)
+    ): void => {
+      const ready = Deferred.makeUnsafe<void>()
+      const owner = fork(
+        (successor ? Effect.yieldNow : Deferred.await(ready)).pipe(
+          Effect.andThen(onlyIf),
+          Effect.flatMap((ready) => ready ? Effect.suspend(() => options.drain(key)) : Effect.void),
+          Effect.onError((cause) =>
+            Cause.hasInterruptsOnly(cause)
+              ? Effect.void
+              : Effect.logWarning(`engine-store: coordinated drain failed for ${String(key)}`, cause)
+          ),
+          Effect.onExit((exit) => Effect.sync(() => settle(key, entry, exit))),
+          Effect.exit,
+          Effect.asVoid
+        )
+      )
+      entry.owner = owner
+      if (!successor) Deferred.doneUnsafe(ready, Effect.void)
+    }
+
+    const settle = (key: Key, entry: Entry<E>, exit: Exit.Exit<void, E>): void => {
+      if (Exit.isSuccess(exit) && !entry.stopping && entry.pendingWake) {
+        entry.pendingWake = false
+        start(key, entry, true)
+        return
+      }
+
+      const successor = entry.pendingWake ? makeEntry() : undefined
+      if (successor === undefined) active.delete(key)
+      else {
+        active.set(key, successor)
+        start(key, successor, true)
+      }
+      Deferred.doneUnsafe(entry.done, exit)
+    }
+
+    const startIdle = (key: Key, onlyIf?: Effect.Effect<boolean>): Entry<E> => {
+      const next = makeEntry()
+      active.set(key, next)
+      start(key, next, false, onlyIf)
+      return next
+    }
+
+    const run = (key: Key): Effect.Effect<void, E> =>
+      Effect.uninterruptibleMask((restore) => {
+        const entry = active.get(key)
+        if (entry !== undefined) {
+          /* v8 ignore next -- stopping is a cooperative-scheduler handoff covered by the cleanup race test */
+          if (entry.stopping) return restore(Deferred.await(entry.done).pipe(Effect.andThen(run(key))))
+          return restore(Deferred.await(entry.done))
+        }
+
+        return restore(Deferred.await(startIdle(key).done))
+      })
+
+    const join = (key: Key): Effect.Effect<void, E> =>
+      Effect.suspend(() => {
+        const entry = active.get(key)
+        return entry === undefined ? Effect.void : Deferred.await(entry.done)
+      })
+
+    const schedule = (key: Key): Effect.Effect<void> =>
+      Effect.sync(() => {
+        if (!active.has(key)) startIdle(key)
+      })
+
+    const scheduleIf = (key: Key, ready: Effect.Effect<boolean>): Effect.Effect<void> =>
+      Effect.sync(() => {
+        if (!active.has(key)) startIdle(key, ready)
+      })
+
+    const wake = (key: Key): Effect.Effect<void> =>
+      Effect.sync(() => {
+        const entry = active.get(key)
+        if (entry !== undefined) {
+          entry.pendingWake = true
+          return
+        }
+
+        startIdle(key)
+      })
+
+    const interrupt = (key: Key): Effect.Effect<void> =>
+      Effect.suspend(() => {
+        const entry = active.get(key)
+        if (entry?.owner === undefined) return Effect.void
+        entry.stopping = true
+        entry.pendingWake = false
+        return Fiber.interrupt(entry.owner)
+      })
+
+    const requestInterrupt = (key: Key): Effect.Effect<void> =>
+      Effect.withFiber((fiber) =>
+        Effect.sync(() => {
+          const entry = active.get(key)
+          if (entry?.owner === undefined) return
+          entry.stopping = true
+          entry.pendingWake = false
+          entry.owner.interruptUnsafe(fiber.id)
+        })
+      )
+
+    return {
+      active: Effect.fn("RunCoordinator.active")(() => Effect.sync(() => new Set(active.keys())))(),
+      run: Effect.fn("RunCoordinator.run")(run),
+      join: Effect.fn("RunCoordinator.join")(join),
+      schedule: Effect.fn("RunCoordinator.schedule")(schedule),
+      scheduleIf: Effect.fn("RunCoordinator.scheduleIf")(scheduleIf),
+      wake: Effect.fn("RunCoordinator.wake")(wake),
+      interrupt: Effect.fn("RunCoordinator.interrupt")(interrupt),
+      requestInterrupt: Effect.fn("RunCoordinator.requestInterrupt")(requestInterrupt)
+    }
+  })