effect 4.0.4 → 4.0.6

package/src/cluster/ClusterMetrics.ts

@@ -1,10 +1,11 @@
 /**
  * Standard metric definitions used by the unstable cluster runtime. The gauges
- * track runner-local entities, singleton processes, registered runners,
- * healthy runners, and acquired shards.
+ * track runner-local entities, entity mailbox sizes, singleton processes,
+ * registered runners, healthy runners, acquired shards, and shards being
+ * released. The timer tracks shard release duration.
  *
  * This module only defines the metric handles. The runtime components that
- * manage entities, runners, singletons, and sharding update these gauges while
+ * manage entities, runners, singletons, and sharding update these metrics while
  * the cluster is running.
  *
  * @stability unstable
@@ -23,13 +24,13 @@ import * as Metric from "../Metric.ts"
  *
  * **Details**
  *
- * Bigint gauge named `effect_cluster_entities`, updated with the entity type as
- * a metric tag.
+ * Bigint gauge named `effect_cluster_entities`, tagged with the entity type.
+ * Each entity manager registers a series that reads its active entity count
+ * when metrics are snapshotted, and removes it when the manager shuts down.
  *
  * **Gotchas**
  *
- * This gauge is runner-local and sampled by the entity manager loop. Aggregate
- * across runners and expect up to roughly one polling interval of lag.
+ * This gauge is runner-local. Aggregate across runners.
  *
  * @see {@link singletons} for singleton process counts on the current runner
  * @see {@link shards} for shard ownership on the current runner
@@ -40,6 +41,86 @@ import * as Metric from "../Metric.ts"
  */
 export const entities = Metric.gauge("effect_cluster_entities", { bigint: true })
 
+/**
+ * Represents a gauge tracking the mailbox size of each entity resident on the
+ * current runner.
+ *
+ * **When to use**
+ *
+ * Use to find entities whose mailboxes back up, before sends start failing
+ * with `MailboxFull`.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size`, with one series per
+ * resident entity tagged with `type`, `id` and `shard`. The value is the
+ * number the mailbox capacity check compares against: requests admitted to the
+ * entity, whether queued or in flight, excluding internal keep-alive requests.
+ *
+ * The value is read when metrics are snapshotted, so it adds no work per
+ * message. Each series is removed when its entity is released.
+ *
+ * **Gotchas**
+ *
+ * Off by default. Enable it with `ShardingConfig.entityMailboxMetrics` or the
+ * `mailboxMetrics` option when registering an entity. One series per resident
+ * entity can mean many series on busy runners.
+ *
+ * An entity keeps reporting while it drains in-flight requests during
+ * shutdown. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entities} for the number of resident entities per type
+ * @see {@link entityMailboxSizeMax} for the always-on per-type maximum
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSize = Metric.gauge("effect_cluster_entity_mailbox_size", { bigint: true })
+
+/**
+ * Represents a gauge tracking the largest mailbox size among the entities of
+ * each type resident on the current runner.
+ *
+ * **When to use**
+ *
+ * Use to alert on entity types whose mailboxes back up, before sends start
+ * failing with `MailboxFull`, without paying for a series per entity.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size_max`, with one series
+ * per registered entity type tagged with `type`. The value is the largest
+ * mailbox size that {@link entityMailboxSize} would report for the type,
+ * including entities draining in-flight requests during shutdown, or `0` when
+ * the type has no resident entities.
+ *
+ * The value is computed when metrics are snapshotted, by visiting each
+ * resident entity of the type, so it adds no work per message. Entities
+ * draining during shutdown keep their slot until they finish, so the visit is
+ * bounded by `ShardingConfig.maxResidentEntities` across all types, unless it
+ * is `"unbounded"`. The series is removed when the entity type shuts down.
+ *
+ * **Gotchas**
+ *
+ * Always on. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entityMailboxSize} for the opt-in per-entity mailbox sizes
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSizeMax = Metric.gauge("effect_cluster_entity_mailbox_size_max", { bigint: true })
+
 /**
  * Creates a gauge tracking the number of singleton processes currently running on the
  * current runner.
@@ -116,3 +197,64 @@ export const runnersHealthy = Metric.gauge("effect_cluster_runners_healthy", { b
  * @since 4.0.0
  */
 export const shards = Metric.gauge("effect_cluster_shards", { bigint: true })
+
+/**
+ * Represents a gauge tracking the number of shards the current runner is still
+ * releasing.
+ *
+ * **When to use**
+ *
+ * Use to detect shard releases that stall during rebalancing, shutdown handoff
+ * or lock storage failures.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_shards_releasing`, updated with the number
+ * of distinct shards being released. A shard counts from the moment it is no
+ * longer acquired until its entities and singletons have stopped and its lock
+ * is released, including waits for lock storage to recover and release
+ * retries.
+ *
+ * **Gotchas**
+ *
+ * Positive values are normal while shards move; alert only when the value
+ * stays positive. It does not reflect lock ownership, and reacquisition can
+ * stay blocked briefly after it returns to zero, while a forced release
+ * finishes its bookkeeping.
+ *
+ * Like the other runner gauges it assumes one runner per process: runners
+ * sharing a registry without distinct metric attributes overwrite each other's
+ * value.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.1.0
+ */
+export const shardsReleasing = Metric.gauge("effect_cluster_shards_releasing", { bigint: true })
+
+/**
+ * Represents a timer tracking how long the current runner takes to release a
+ * shard after reassignment, lock loss or shutdown handoff.
+ *
+ * **When to use**
+ *
+ * Use to observe shard handoff latency during rebalancing, shutdown, or lock
+ * storage failures.
+ *
+ * **Details**
+ *
+ * Millisecond timer named `effect_cluster_shard_release`. Each sample covers
+ * entity teardown, retries and any wait for shard lock storage to recover.
+ *
+ * **Gotchas**
+ *
+ * Only completed releases are recorded. Interrupted releases and forced
+ * releases after lock storage becomes unhealthy are not.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.0
+ */
+export const shardReleaseDuration = Metric.timer("effect_cluster_shard_release", {
+  description: "Time taken to release a shard, including entity teardown, retries and lock storage recovery"
+})

package/src/cluster/ClusterWorkflowEngine.ts

@@ -24,7 +24,7 @@ import * as Rpc from "../rpc/Rpc.ts"
 import { ClientAbort } from "../rpc/RpcSchema.ts"
 import * as Schedule from "../Schedule.ts"
 import * as Schema from "../Schema.ts"
-import type * as Scope from "../Scope.ts"
+import * as Scope from "../Scope.ts"
 import * as Semaphore from "../Semaphore.ts"
 import * as Activity from "../workflow/Activity.ts"
 import * as DurableClock from "../workflow/DurableClock.ts"
@@ -135,11 +135,21 @@ export const make = Effect.gen(function*() {
   // Rebuilds share an activation; overlapping activations have separate results.
   // Weak keys do not retain scopes, but contexts captured by clients, activities,
   // or finalizers can keep a closed scope (and its results) reachable until later.
-  const pendingByActivation = new WeakMap<Scope.Scope, {
-    readonly workflowName: string
-    readonly executionId: string
-    readonly results: Map<string, Exit.Exit<unknown, unknown>>
-  }>()
+  const pendingByActivation = new WeakMap<Scope.Scope, ActivationState>()
+  // Closes the scopes retained by suspended runs. The latest run already closed
+  // its own scope with its real exit, so earlier finalizers observe success and
+  // skip superseded compensations. Abandonment releases owner-local resources
+  // without running durable finalizers, so the run can replay elsewhere.
+  const closeRuns = (state: ActivationState, abandoned: boolean) =>
+    Effect.suspend(() => {
+      const scope = state.scope
+      if (abandoned) {
+        for (const instance of state.instances) instance.abandoned = true
+      }
+      state.instances.clear()
+      state.scope = Scope.makeUnsafe()
+      return Scope.close(scope, Exit.void)
+    })
   const clients = yield* RcMap.make({
     lookup: Effect.fnUntraced(function*(workflowName: string) {
       const entity = entities.get(workflowName)
@@ -282,7 +292,7 @@ export const make = Effect.gen(function*() {
     yield* storage.clearAddress(clockAddress)
   })
 
-  const resume = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string) {
+  const resume = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string, mode: WakeMode) {
     const maybeReply = yield* requestReply({
       workflow,
       entityType: `Workflow/${workflow._tag}`,
@@ -294,6 +304,8 @@ export const make = Effect.gen(function*() {
     const maybeSuspended = Option.filter(maybeReply, isSuspendedReply)
 
     if (Option.isNone(maybeSuspended)) return
+    // Replies without `failed` retain automatic wakes for compatibility.
+    if (mode === "automatic" && isFailedReply(maybeSuspended.value)) return
     if (
       !(yield* sharding.reset(
         Snowflake.Snowflake(maybeSuspended.value.requestId),
@@ -305,6 +317,15 @@ export const make = Effect.gen(function*() {
     yield* sharding.pollStorage
   })
 
+  // Activations load a persisted interrupt when they start, so the per-call
+  // check reads only memory.
+  const interruptPending = (instance: WorkflowEngine.WorkflowInstance["Service"]) =>
+    Effect.map(Effect.serviceOption(CurrentActivationScope), (activation) => {
+      const pending = Option.isSome(activation) ? pendingByActivation.get(activation.value) : undefined
+      return pending?.workflowName === instance.workflow._tag && pending.executionId === instance.executionId &&
+        pending.results.has(InterruptSignal.name)
+    })
+
   const sendResumeParent = Effect.fnUntraced(function*(options: {
     readonly workflowName: string
     readonly executionId: string
@@ -363,26 +384,25 @@ export const make = Effect.gen(function*() {
   // `Suspended`. Replies are only observed on the runner that saves them, so
   // callers on other runners, and any failure here, leave delivery to the
   // caller's retry schedule.
-  const awaitComplete = (workflow: Workflow.Any, executionId: string) =>
-    Effect.gen(function*() {
-      const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId })
-      if (!sharding.hasShardId(address.shardId)) return yield* Effect.never
-      const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" })
-      if (Option.isNone(requestId)) return yield* Effect.never
-      const request = Envelope.makeRequest<any>({
-        requestId: requestId.value,
-        address,
-        tag: "run",
-        payload: {},
-        headers: Headers.empty
-      })
-      // A replay that suspends again re-arms the wait, so it does not spend a
-      // step of the caller's retry schedule.
-      let settled = false
-      while (!settled) {
-        settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply))
-      }
-    }).pipe(Effect.catchCause(() => Effect.never))
+  const awaitComplete = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string) {
+    const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId })
+    if (!sharding.hasShardId(address.shardId)) return yield* Effect.never
+    const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" })
+    if (Option.isNone(requestId)) return yield* Effect.never
+    const request = Envelope.makeRequest<any>({
+      requestId: requestId.value,
+      address,
+      tag: "run",
+      payload: {},
+      headers: Headers.empty
+    })
+    // A replay that suspends again re-arms the wait, so it does not spend a
+    // step of the caller's retry schedule.
+    let settled = false
+    while (!settled) {
+      settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply))
+    }
+  }, Effect.catchCause(() => Effect.never))
 
   const interrupt = Effect.fnUntraced(
     function*(workflow: Workflow.Any, executionId: string) {
@@ -436,7 +456,32 @@ export const make = Effect.gen(function*() {
             const activation = yield* CurrentActivationScope
             let pending = pendingByActivation.get(activation)
             if (!pending) {
-              pending = { workflowName: workflow._tag, executionId, results: new Map() }
+              const results = new Map<string, Exit.Exit<unknown, unknown>>()
+              // A new owner must see an interrupt persisted before it took over.
+              // Publish the cache only once loaded, so a retried build repeats
+              // a failed lookup.
+              const interrupt = yield* Effect.orDie(requestReply({
+                workflow,
+                entityType: `Workflow/${workflow._tag}`,
+                executionId,
+                tag: "deferred",
+                id: InterruptSignal.name
+              }))
+              if (Option.isSome(interrupt)) {
+                const decoded = decodeDeferredWithExit(interrupt.value as any)
+                results.set(
+                  InterruptSignal.name,
+                  (decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit) as Exit.Exit<unknown, unknown>
+                )
+              }
+              pending = {
+                workflowName: workflow._tag,
+                executionId,
+                results,
+                instances: new Set(),
+                scope: Scope.makeUnsafe()
+              }
+              yield* Scope.addFinalizer(activation, closeRuns(pending, true))
               pendingByActivation.set(activation, pending)
             }
             // Retain awaited names through reply persistence, including suspension.
@@ -444,25 +489,40 @@ export const make = Effect.gen(function*() {
             let currentRun: {
               readonly request: Entity.Request<any>
               readonly instance: WorkflowEngine.WorkflowInstance["Service"]
+              // Activity awaits must wake the run after suspension without preempting it.
+              readonly activityAwaits: Map<string, number>
             } | undefined
-            // Concurrent wakes share one wait for the current run to publish its reply.
+            // Concurrent automatic wakes share one wait for the current run to
+            // publish its reply. Explicit wakes cannot be dropped, because an
+            // automatic wake skips the reset of a failure suspension, so they
+            // wait outside the gate. Operator resume does not use the gate.
             const resumeGate = Semaphore.makeUnsafe(1)
-            const resumeCurrentRun = Effect.suspend(() =>
-              resumeGate.withPermitsIfAvailable(1)(
-                currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
-              )
-            ).pipe(
-              // Release the gate before reset can start another run, so later
-              // completions can wake that replay.
-              Effect.flatMap((waited) => Option.isSome(waited) ? resume(workflow, executionId) : Effect.void),
-              ensureSuccess
+            const waitForCurrentRunReply = Effect.suspend(() =>
+              currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
             )
+            const resumeCurrentRun = (mode: WakeMode) =>
+              ensureSuccess(
+                mode === "explicit"
+                  ? Effect.andThen(waitForCurrentRunReply, resume(workflow, executionId, mode))
+                  : resumeGate.withPermitsIfAvailable(1)(waitForCurrentRunReply).pipe(
+                    // Release the gate before reset can start another run, so later
+                    // completions can wake that replay.
+                    Effect.flatMap((waited) =>
+                      Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void
+                    )
+                  )
+              )
             return {
               run: (request: Entity.Request<any>) => {
-                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId)
-                currentRun = { request, instance }
+                // Each run gets a fresh scope under the activation's parent, so
+                // replays rebuild compensations while suspended runs keep theirs.
+                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId, {
+                  scope: Scope.forkUnsafe(pending.scope)
+                })
+                pending.instances.add(instance)
+                currentRun = { request, instance, activityAwaits: new Map() }
                 const parent = (request.payload as any)[payloadParentKey] as
-                  | { workflowName: string; executionId: string }
+                  | { workflowName: string; executionId: string; detached?: boolean }
                   | undefined
                 return execute(workflow.payloadSchema.make(request.payload) as object, executionId).pipe(
                   Effect.onExit((exit) => {
@@ -479,7 +539,9 @@ export const make = Effect.gen(function*() {
                       !instance.abandoned &&
                       !(suspendOnFailure && exit._tag === "Failure")
                     ) {
-                      return parent ? ensureSuccess(sendResumeParent(parent, executionId)) : Effect.void
+                      return parent && !parent.detached
+                        ? ensureSuccess(sendResumeParent(parent, executionId))
+                        : Effect.void
                     }
                     return engine.deferredResult(InterruptSignal).pipe(
                       Effect.flatMap((maybeExit) => {
@@ -497,6 +559,11 @@ export const make = Effect.gen(function*() {
                     )
                   }),
                   Workflow.intoResult,
+                  Effect.onExit((exit) =>
+                    Exit.isSuccess(exit) && exit.value._tag === "Suspended"
+                      ? Effect.void
+                      : closeRuns(pending, instance.abandoned)
+                  ),
                   (effect) => deferredState.trackRun(instance, effect),
                   // Request middleware and local callers may carry another entity's context.
                   Effect.provideService(CurrentActivationScope, activation)
@@ -506,7 +573,9 @@ export const make = Effect.gen(function*() {
               activity(request: Entity.Request<any>) {
                 const payload = request.payload as { name: string; attempt: number }
                 const activityId = `${executionId}/${payload.name}`
-                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId)
+                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId, {
+                  awaitedDeferreds: currentRun?.activityAwaits
+                })
                 interruptedActivities.delete(activityId)
                 const entry = activities.get(activityId)
                 if (!entry) {
@@ -546,18 +615,26 @@ export const make = Effect.gen(function*() {
               deferred: (request: Entity.Request<any>) => {
                 const payload = request.payload as any
                 pending.results.set(payload.name, payload.exit)
+                const mode: WakeMode = payload.name === InterruptSignal.name ? "explicit" : "automatic"
                 // Await registers before reading the cache. If this run has not
                 // registered yet, a later await will see the result just stored.
                 // Check when the wake runs and skip preemption too: interrupting
                 // a later await would require waiting for its reply before reset.
                 // A run completing its own deferred also reads it from the cache
-                // when it awaits later, so that completion never wakes the run.
+                // when it awaits later, unless an activity is already parked on it.
+                // A run only reads InterruptSignal while suspending, so an interrupt
+                // always waits for the current run's reply before it resets the run.
                 const wake = Effect.suspend(() =>
-                  currentRun && !currentRun.instance.awaitedDeferreds.has(payload.name)
-                    ? currentRun.instance.completedDeferreds.has(payload.name)
+                  currentRun && mode === "automatic" && !currentRun.instance.awaitedDeferreds.has(payload.name)
+                    ? currentRun.activityAwaits.has(payload.name)
+                      // Hold an activity wake until the run has published its reply.
+                      ? resumeCurrentRun(mode)
+                      : currentRun.instance.completedDeferreds.has(payload.name)
                       ? Effect.void
-                      : ensureSuccess(resume(workflow, executionId))
-                    : deferredState.deferredDone(executionId, payload.name).pipe(Effect.andThen(resumeCurrentRun))
+                      : ensureSuccess(resume(workflow, executionId, mode))
+                    : deferredState.deferredDone(executionId, payload.name).pipe(
+                      Effect.andThen(resumeCurrentRun(mode))
+                    )
                 )
                 // An asynchronous reply releases the RPC concurrency permit while
                 // the wake waits; the pending request still occupies mailbox capacity.
@@ -572,7 +649,7 @@ export const make = Effect.gen(function*() {
                 )
               },
 
-              resume: () => resumeCurrentRun.pipe(Rpc.wrap({ fork: true, uninterruptible: true }))
+              resume: () => resumeCurrentRun("automatic").pipe(Rpc.wrap({ fork: true, uninterruptible: true }))
             }
           }),
           // Alongside the run, let deferred handlers briefly acquire a permit to
@@ -589,7 +666,7 @@ export const make = Effect.gen(function*() {
         ).pipe(Effect.andThen(registration))
       }),
 
-    execute: (workflow, { discard, executionId, parent, payload }) => {
+    execute: (workflow, { detached, discard, executionId, parent, payload }) => {
       ensureEntity(workflow)
       return RcMap.get(clients, workflow._tag).pipe(
         Effect.flatMap((make) =>
@@ -597,7 +674,10 @@ export const make = Effect.gen(function*() {
             parent ?
               {
                 ...payload,
-                [payloadParentKey]: { workflowName: parent.workflow._tag, executionId: parent.executionId }
+                // A detached child keeps its parent for observability only.
+                [payloadParentKey]: detached
+                  ? { workflowName: parent.workflow._tag, executionId: parent.executionId, detached: true }
+                  : { workflowName: parent.workflow._tag, executionId: parent.executionId }
               } :
               payload,
             { discard }
@@ -649,7 +729,7 @@ export const make = Effect.gen(function*() {
       ))
     }),
 
-    resume: (workflow, executionId) => ensureSuccess(resume(workflow, executionId)),
+    resume: (workflow, executionId) => ensureSuccess(resume(workflow, executionId, "explicit")),
 
     awaitComplete,
 
@@ -659,6 +739,21 @@ export const make = Effect.gen(function*() {
         const instance = Context.get(services, WorkflowEngine.WorkflowInstance)
         yield* Effect.annotateCurrentSpan("executionId", instance.executionId)
         const activityId = `${instance.executionId}/${activity.name}`
+        // A pending interrupt replays only to rebuild compensations, so it must
+        // not run new activities. Finalizers run once `interrupted` is set.
+        if (!instance.interrupted && (yield* interruptPending(instance))) {
+          const reply = yield* Effect.orDie(requestReply({
+            workflow: instance.workflow,
+            entityType: `Workflow/${instance.workflow._tag}`,
+            executionId: instance.executionId,
+            tag: "activity",
+            id: activityPrimaryKey(activity.name, attempt)
+          }))
+          // Suspend so `onExit` applies the interrupt to the run.
+          if (Option.isNone(reply) || isSuspendedReply(reply.value)) {
+            return new Workflow.Suspended({})
+          }
+        }
         const client = (yield* RcMap.get(clientsPartial, instance.workflow._tag))(instance.executionId)
         while (true) {
           if (!activities.has(activityId)) {
@@ -845,9 +940,24 @@ const DeferredRpc = Rpc.make("deferred", {
   .annotate(ClusterSchema.Persisted, true)
   .annotate(ClusterSchema.Uninterruptible, true)
 
+type WakeMode = "explicit" | "automatic"
+
+interface ActivationState {
+  readonly workflowName: string
+  readonly executionId: string
+  readonly results: Map<string, Exit.Exit<unknown, unknown>>
+  // Every run forks its scope from here, so runs that suspended keep their
+  // resources and durable finalizers until the execution finishes.
+  scope: Scope.Closeable
+  readonly instances: Set<WorkflowEngine.WorkflowInstance["Service"]>
+}
+
 const isSuspendedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) =>
   reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended"
 
+const isFailedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) =>
+  reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended" && reply.exit.value.failed === true
+
 const decodeDeferredWithExit = Schema.decodeSync(Schema.toCodecJson(Reply.WithExit.schema(DeferredRpc)))
 
 const ResumeRpc = Rpc.make("resume", {
@@ -875,7 +985,8 @@ const makeWorkflowEntity = (workflow: Workflow.Any) =>
         ...workflow.payloadSchema.fields,
         [payloadParentKey]: Schema.optional(Schema.Struct({
           workflowName: Schema.String,
-          executionId: Schema.String
+          executionId: Schema.String,
+          detached: Schema.optional(Schema.Boolean)
         }))
       },
       primaryKey: () => "",

package/src/cluster/Entity.ts

@@ -147,6 +147,7 @@ export interface Entity<
       readonly maxIdleTime?: Duration.Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -191,6 +192,7 @@ export interface Entity<
     options?: {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -287,6 +289,7 @@ const Proto = {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -335,6 +338,7 @@ const Proto = {
     options?: {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined

package/src/cluster/MessageStorage.ts

@@ -119,6 +119,13 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 
   /**
    * For locally sent messages, register a handler to process the replies.
+   *
+   * **Details**
+   *
+   * The handler must be registered as soon as the effect starts, before it
+   * first suspends. Callers fork it with `startImmediately` before saving or
+   * notifying the request, and rely on it to see every reply saved after that
+   * point. The effect completes when a `WithExit` reply arrives.
    */
   readonly registerReplyHandler: <R extends Rpc.Any>(
     message: Message.OutgoingRequest<R> | Message.IncomingRequest<R>
@@ -1086,8 +1093,8 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
           journal.push(envelope)
           return SaveResultEncoded.Success()
         }),
-      saveReply: (reply_) =>
-        Effect.sync(() => {
+      saveReply: (reply_) => {
+        const save = Effect.sync(() => {
           const reply = JSON.parse(JSON.stringify(reply_)) as Reply.Encoded
           const entry = requests.get(reply.requestId)
           if (!entry || replyIds.has(reply.id)) return
@@ -1098,7 +1105,18 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
           entry.replies.push(reply)
           replyIds.add(reply.id)
           replyLatch.openUnsafe()
-        }),
+        })
+        // Memory has no transaction isolation. Publish terminal replies at the
+        // outer commit boundary, before their queued handler notifications, so
+        // reply readers cannot deliver a success that may still be rolled back.
+        return TransactionNotifications.use((notifications) =>
+          reply_._tag === "WithExit" && notifications
+            ? Effect.sync(() => {
+              notifications.push(save)
+            })
+            : save
+        )
+      },
       clearReplies: (id, options) =>
         Effect.sync(() => {
           const entry = requests.get(String(id))