effect 4.0.5 → 4.0.7

package/src/cluster/ClusterCron.ts

@@ -7,9 +7,15 @@
  * for distributed maintenance work where the job should be owned by the cluster
  * rather than by every runner independently.
  *
+ * With `calculateNextRunFromPrevious`, every run is processed by a single
+ * entity per cron job, so runs never overlap and are caught up in order. The
+ * scheduled time of the current run is available through
+ * `CurrentScheduledTime`.
+ *
  * @stability unstable
  * @since 4.0.0
  */
+import * as Context from "../Context.ts"
 import * as Cron from "../Cron.ts"
 import * as DateTime from "../DateTime.ts"
 import * as Duration from "../Duration.ts"
@@ -29,6 +35,23 @@ import * as Entity from "./Entity.ts"
 import type { Sharding } from "./Sharding.ts"
 import * as Singleton from "./Singleton.ts"
 
+/**
+ * Service tag for the scheduled time of the cron run currently executing.
+ *
+ * **When to use**
+ *
+ * Use to read the time the current run was scheduled for, inside the
+ * `execute` effect of `make`.
+ *
+ * @stability unstable
+ * @category services
+ * @since 4.0.0
+ */
+export class CurrentScheduledTime extends Context.Service<
+  CurrentScheduledTime,
+  DateTime.Utc
+>()("effect/cluster/ClusterCron/CurrentScheduledTime") {}
+
 /**
  * Creates a layer that runs a cron job through the cluster sharding system.
  *
@@ -60,6 +83,11 @@ export const make = <E, R>(options: {
    *
    * Defaults to `false`, meaning the next run will be calculated from the
    * current time.
+   *
+   * When enabled, all runs are processed one at a time by a single entity, so
+   * missed runs are caught up in order without overlapping. A run that fails
+   * is retried in place and blocks later runs until it succeeds or is older
+   * than `skipIfOlderThan`.
    */
   readonly calculateNextRunFromPrevious?: boolean | undefined
 
@@ -76,26 +104,42 @@ export const make = <E, R>(options: {
    * Defaults to "1 day".
    */
   readonly skipIfOlderThan?: Duration.Input | undefined
-}): Layer.Layer<never, never, Sharding | Exclude<R, Scope>> => {
+}): Layer.Layer<never, never, Sharding | Exclude<Exclude<R, CurrentScheduledTime>, Scope>> => {
+  const fromPrevious = options.calculateNextRunFromPrevious === true
+  const nextRun = (after: DateTime.Utc) => DateTime.fromDateUnsafe(Cron.next(options.cron, after))
+
   const CronEntity = Entity.make(`ClusterCron/${options.name}`, [
     Rpc.make("run", {
       payload: CronPayload
     })
       .annotate(Persisted, true)
-      .annotate(Uninterruptible, true)
+      .annotate(Uninterruptible, true),
+    Rpc.make("start").annotate(Persisted, true)
   ])
     .annotate(ClusterSchema.ShardGroup, () => options.shardGroup ?? "default")
     .annotate(ClusterSchema.ClientTracingEnabled, false)
 
+  // with `calculateNextRunFromPrevious`, every run shares the executor address
+  const send = (dateTime: DateTime.Utc) =>
+    Effect.flatMap(CronEntity.client, (makeClient) =>
+      fromPrevious
+        ? makeClient(executorId).run(new CronPayload({ dateTime, executor: true }), { discard: true })
+        : makeClient(DateTime.formatIso(dateTime)).run(new CronPayload({ dateTime }), { discard: true }))
+
+  const schedule = (dateTime: DateTime.Utc) => send(dateTime).pipe(withRetry("Failed to schedule next run, retrying"))
+
   const InitialRun = Singleton.make(
     `ClusterCron/${options.name}`,
-    Effect.gen(function*() {
-      const now = yield* DateTime.now
-      const next = DateTime.fromDateUnsafe(Cron.next(options.cron, now))
-      const entityId = options.calculateNextRunFromPrevious ? "initial" : DateTime.formatIso(next)
-      const client = (yield* CronEntity.client)(entityId)
-      yield* client.run({ dateTime: next }, { discard: true })
-    }),
+    fromPrevious
+      // the executor schedules the next run if its chain has been lost.
+      // `start` has no primary key, so each retry is a new message
+      ? Effect.flatMap(CronEntity.client, (makeClient) => makeClient(executorId).start()).pipe(
+        withRetry("Failed to start cron job, retrying")
+      )
+      : Effect.flatMap(
+        DateTime.now,
+        (now) => send(nextRun(now)).pipe(withRetry("Failed to schedule initial run, retrying"))
+      ),
     { shardGroup: options.shardGroup }
   )
 
@@ -104,61 +148,76 @@ export const make = <E, R>(options: {
     Option.getOrElse(() => Duration.days(1))
   )
 
-  const effect = Effect.fnUntraced(function*(dateTime: DateTime.Utc) {
+  const execute = Effect.fnUntraced(function*(dateTime: DateTime.Utc) {
     const now = yield* DateTime.now
     if (DateTime.isLessThan(dateTime, DateTime.subtractDuration(now, skipIfOlderThan))) {
       return
     }
-    return yield* options.execute
+    return yield* Effect.provideService(options.execute, CurrentScheduledTime, dateTime)
   }, Effect.orDie)
 
   const EntityLayer = CronEntity.toLayer(Effect.gen(function*() {
-    const makeClient = yield* CronEntity.client
+    const address = yield* Entity.CurrentAddress
+    // runs are processed one at a time, so a run processed before `start`
+    // means the chain is alive
+    let processedRun = false
     return {
-      run: (request) =>
-        Effect.onExitPrimitive(
-          effect(request.payload.dateTime),
-          Effect.fnUntraced(function*(exit) {
-            if (Exit.isFailure(exit)) {
-              yield* Effect.logWarning(exit.cause)
-            }
-            const now = yield* DateTime.now
-            const next = DateTime.fromDateUnsafe(Cron.next(
-              options.cron,
-              options.calculateNextRunFromPrevious ? request.payload.dateTime : now
-            ))
-            const client = makeClient(DateTime.formatIso(next))
-            return yield* client.run({ dateTime: next }, { discard: true }).pipe(
-              Effect.tapCause((cause) => Effect.logWarning("Failed to schedule next run, retrying", cause)),
-              Effect.sandbox,
-              Effect.retry(retryPolicy),
-              Effect.orDie
-            )
-          }),
+      run: (request) => {
+        const dateTime = request.payload.dateTime
+        if (fromPrevious && address.entityId !== executorId) {
+          // a run scheduled by an earlier release joins the executor's chain
+          return schedule(dateTime)
+        }
+        processedRun = true
+        return Effect.onExitPrimitive(
+          execute(dateTime),
+          (exit) =>
+            Effect.andThen(
+              Exit.isFailure(exit) ? Effect.logWarning(exit.cause) : Effect.void,
+              fromPrevious ? schedule(nextRun(dateTime)) : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now)))
+            ),
           true
         ).pipe(
           Effect.annotateLogs({
             module: "effect/cluster/ClusterCron",
             name: options.name,
-            dateTime: request.payload.dateTime
+            dateTime
           })
         )
+      },
+      start: () =>
+        processedRun
+          ? Effect.void
+          // deduplicated against the pending run if the chain is alive
+          : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now)))
     }
   }))
 
   return Layer.merge(InitialRun, EntityLayer)
 }
 
+const executorId = "executor"
+
 const retryPolicy = Schedule.min([
   Schedule.exponential(200, 1.5),
   Schedule.spaced("1 minute")
 ])
 
+const withRetry = (message: string) => <A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, never, R> =>
+  self.pipe(
+    Effect.tapCause((cause) => Effect.logWarning(message, cause)),
+    Effect.sandbox,
+    Effect.retry(retryPolicy),
+    Effect.orDie
+  )
+
 class CronPayload extends Schema.Class<CronPayload>("effect/cluster/ClusterCron/CronPayload")({
-  dateTime: Schema.DateTimeUtc
+  dateTime: Schema.DateTimeUtc,
+  // set for runs sent to the executor, which share one entity address
+  executor: Schema.optional(Schema.Boolean)
 }) {
   [PrimaryKey.symbol]() {
-    return ""
+    return this.executor ? DateTime.formatIso(this.dateTime) : ""
   }
   [DeliverAt.symbol]() {
     return this.dateTime

package/src/cluster/ClusterMetrics.ts

@@ -1,10 +1,11 @@
 /**
  * Standard metric definitions used by the unstable cluster runtime. The gauges
- * track runner-local entities, singleton processes, registered runners,
- * healthy runners, and acquired shards.
+ * track runner-local entities, entity mailbox sizes, singleton processes,
+ * registered runners, healthy runners, acquired shards, and shards being
+ * released. The timer tracks shard release duration.
  *
  * This module only defines the metric handles. The runtime components that
- * manage entities, runners, singletons, and sharding update these gauges while
+ * manage entities, runners, singletons, and sharding update these metrics while
  * the cluster is running.
  *
  * @stability unstable
@@ -23,13 +24,13 @@ import * as Metric from "../Metric.ts"
  *
  * **Details**
  *
- * Bigint gauge named `effect_cluster_entities`, updated with the entity type as
- * a metric tag.
+ * Bigint gauge named `effect_cluster_entities`, tagged with the entity type.
+ * Each entity manager registers a series that reads its active entity count
+ * when metrics are snapshotted, and removes it when the manager shuts down.
  *
  * **Gotchas**
  *
- * This gauge is runner-local and sampled by the entity manager loop. Aggregate
- * across runners and expect up to roughly one polling interval of lag.
+ * This gauge is runner-local. Aggregate across runners.
  *
  * @see {@link singletons} for singleton process counts on the current runner
  * @see {@link shards} for shard ownership on the current runner
@@ -40,6 +41,86 @@ import * as Metric from "../Metric.ts"
  */
 export const entities = Metric.gauge("effect_cluster_entities", { bigint: true })
 
+/**
+ * Represents a gauge tracking the mailbox size of each entity resident on the
+ * current runner.
+ *
+ * **When to use**
+ *
+ * Use to find entities whose mailboxes back up, before sends start failing
+ * with `MailboxFull`.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size`, with one series per
+ * resident entity tagged with `type`, `id` and `shard`. The value is the
+ * number the mailbox capacity check compares against: requests admitted to the
+ * entity, whether queued or in flight, excluding internal keep-alive requests.
+ *
+ * The value is read when metrics are snapshotted, so it adds no work per
+ * message. Each series is removed when its entity is released.
+ *
+ * **Gotchas**
+ *
+ * Off by default. Enable it with `ShardingConfig.entityMailboxMetrics` or the
+ * `mailboxMetrics` option when registering an entity. One series per resident
+ * entity can mean many series on busy runners.
+ *
+ * An entity keeps reporting while it drains in-flight requests during
+ * shutdown. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entities} for the number of resident entities per type
+ * @see {@link entityMailboxSizeMax} for the always-on per-type maximum
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSize = Metric.gauge("effect_cluster_entity_mailbox_size", { bigint: true })
+
+/**
+ * Represents a gauge tracking the largest mailbox size among the entities of
+ * each type resident on the current runner.
+ *
+ * **When to use**
+ *
+ * Use to alert on entity types whose mailboxes back up, before sends start
+ * failing with `MailboxFull`, without paying for a series per entity.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size_max`, with one series
+ * per registered entity type tagged with `type`. The value is the largest
+ * mailbox size that {@link entityMailboxSize} would report for the type,
+ * including entities draining in-flight requests during shutdown, or `0` when
+ * the type has no resident entities.
+ *
+ * The value is computed when metrics are snapshotted, by visiting each
+ * resident entity of the type, so it adds no work per message. Entities
+ * draining during shutdown keep their slot until they finish, so the visit is
+ * bounded by `ShardingConfig.maxResidentEntities` across all types, unless it
+ * is `"unbounded"`. The series is removed when the entity type shuts down.
+ *
+ * **Gotchas**
+ *
+ * Always on. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entityMailboxSize} for the opt-in per-entity mailbox sizes
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSizeMax = Metric.gauge("effect_cluster_entity_mailbox_size_max", { bigint: true })
+
 /**
  * Creates a gauge tracking the number of singleton processes currently running on the
  * current runner.
@@ -116,3 +197,64 @@ export const runnersHealthy = Metric.gauge("effect_cluster_runners_healthy", { b
  * @since 4.0.0
  */
 export const shards = Metric.gauge("effect_cluster_shards", { bigint: true })
+
+/**
+ * Represents a gauge tracking the number of shards the current runner is still
+ * releasing.
+ *
+ * **When to use**
+ *
+ * Use to detect shard releases that stall during rebalancing, shutdown handoff
+ * or lock storage failures.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_shards_releasing`, updated with the number
+ * of distinct shards being released. A shard counts from the moment it is no
+ * longer acquired until its entities and singletons have stopped and its lock
+ * is released, including waits for lock storage to recover and release
+ * retries.
+ *
+ * **Gotchas**
+ *
+ * Positive values are normal while shards move; alert only when the value
+ * stays positive. It does not reflect lock ownership, and reacquisition can
+ * stay blocked briefly after it returns to zero, while a forced release
+ * finishes its bookkeeping.
+ *
+ * Like the other runner gauges it assumes one runner per process: runners
+ * sharing a registry without distinct metric attributes overwrite each other's
+ * value.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.1.0
+ */
+export const shardsReleasing = Metric.gauge("effect_cluster_shards_releasing", { bigint: true })
+
+/**
+ * Represents a timer tracking how long the current runner takes to release a
+ * shard after reassignment, lock loss or shutdown handoff.
+ *
+ * **When to use**
+ *
+ * Use to observe shard handoff latency during rebalancing, shutdown, or lock
+ * storage failures.
+ *
+ * **Details**
+ *
+ * Millisecond timer named `effect_cluster_shard_release`. Each sample covers
+ * entity teardown, retries and any wait for shard lock storage to recover.
+ *
+ * **Gotchas**
+ *
+ * Only completed releases are recorded. Interrupted releases and forced
+ * releases after lock storage becomes unhealthy are not.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.0
+ */
+export const shardReleaseDuration = Metric.timer("effect_cluster_shard_release", {
+  description: "Time taken to release a shard, including entity teardown, retries and lock storage recovery"
+})