effect 4.0.4 → 4.0.6

package/src/cluster/Runners.ts

@@ -14,6 +14,7 @@
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
 import * as Exit from "../Exit.ts"
+import * as Fiber from "../Fiber.ts"
 import * as Latch from "../Latch.ts"
 import * as Layer from "../Layer.ts"
 import * as Option from "../Option.ts"
@@ -247,78 +248,149 @@ export const make: (
 
   type StorageRequestEntry = {
     readonly latch: Latch.Latch
-    doneLatch: Latch.Latch | undefined
     readonly messages: Set<Message.OutgoingRequest<any>>
+    readonly pump: Fiber.Fiber<void>
     replies: Array<Reply.Reply<any>>
   }
   const storageRequests = new Map<Snowflake.Snowflake, StorageRequestEntry>()
   const waitingStorageRequests = new Map<Snowflake.Snowflake, Message.OutgoingRequest<any>>()
-  const replyFromStorage = Effect.fnUntraced(
-    function*(message: Message.OutgoingRequest<any>) {
-      let entry = storageRequests.get(message.envelope.requestId)
-      if (entry) {
-        entry.messages.add(message)
-        entry.doneLatch ??= Latch.makeUnsafe(false)
-        return yield* entry.doneLatch.await
-      } else {
-        entry = {
-          latch: Latch.makeUnsafe(false),
-          doneLatch: undefined,
-          replies: [],
-          messages: new Set([message])
-        }
-        storageRequests.set(message.envelope.requestId, entry)
-      }
-
-      while (true) {
-        // wait for the storage loop to notify us
-        entry.latch.closeUnsafe()
-        waitingStorageRequests.set(message.envelope.requestId, message)
-        storageLatch.openUnsafe()
-        yield* entry.latch.await
 
-        // send the replies back
-        for (let i = 0; i < entry.replies.length; i++) {
-          const reply = entry.replies[i]
-          // we have reached the end
-          if (reply._tag === "WithExit") {
-            // A reply can immediately reset and resend the same request. Do not
-            // let that sender attach to this completed reply loop.
-            storageRequests.delete(message.envelope.requestId)
-            waitingStorageRequests.delete(message.envelope.requestId)
-            for (const message of entry.messages) {
-              yield* message.respond(reply)
-            }
-            entry.doneLatch?.openUnsafe()
-            return
-          }
+  // A reset request can register a replacement entry under the same id, which
+  // must survive the old entry's cleanup.
+  const removeStorageRequest = (requestId: Snowflake.Snowflake, entry: Omit<StorageRequestEntry, "pump">) => {
+    if (storageRequests.get(requestId) !== entry) return
+    storageRequests.delete(requestId)
+    waitingStorageRequests.delete(requestId)
+  }
 
-          entry.latch.closeUnsafe()
+  // The pump is owned by the shared entry rather than the first caller, so any
+  // waiter can be interrupted without stopping delivery to the others.
+  //
+  // `replyFromStorage` registers the request with the storage loop before the
+  // pump starts, so the pump only re-registers after a stream chunk is acked.
+  const pumpReplies = Effect.fnUntraced(function*(
+    requestId: Snowflake.Snowflake,
+    entry: Omit<StorageRequestEntry, "pump">
+  ) {
+    while (true) {
+      // wait for the storage loop to notify us
+      yield* entry.latch.await
+
+      // send the replies back
+      for (let i = 0; i < entry.replies.length; i++) {
+        const reply = entry.replies[i]
+        // we have reached the end
+        if (reply._tag === "WithExit") {
+          // A reply can immediately reset and resend the same request. Do not
+          // let that sender attach to this completed entry.
+          removeStorageRequest(requestId, entry)
           for (const message of entry.messages) {
             yield* message.respond(reply)
           }
-          // wait for ack
-          yield* entry.latch.await
+          return
         }
-        entry.replies = []
+
+        entry.latch.closeUnsafe()
+        for (const message of entry.messages) {
+          yield* message.respond(reply)
+        }
+        // wait for ack
+        yield* entry.latch.await
       }
-    },
-    (effect, message) =>
-      Effect.ensuring(
-        effect,
-        Effect.sync(() => {
-          const entry = storageRequests.get(message.envelope.requestId)
-          // An old waiter must not clean up a reset request's new reply loop.
-          if (!entry?.messages.has(message)) return
-          if (entry.messages.size > 1) {
-            entry.messages.delete(message)
-            return
+      entry.replies = []
+
+      // ask the storage loop for the next chunk
+      entry.latch.closeUnsafe()
+      const message = entry.messages.values().next().value
+      if (!message) return
+      waitingStorageRequests.set(requestId, message)
+      storageLatch.openUnsafe()
+    }
+  })
+
+  const replyFromStorage = (message: Message.OutgoingRequest<any>) =>
+    Effect.uninterruptibleMask((restore) =>
+      Effect.gen(function*() {
+        const requestId = message.envelope.requestId
+        let entry = storageRequests.get(requestId)
+        if (!entry) {
+          const state = {
+            latch: Latch.makeUnsafe(false),
+            messages: new Set<Message.OutgoingRequest<any>>(),
+            replies: [] as Array<Reply.Reply<any>>
           }
-          storageRequests.delete(message.envelope.requestId)
-          waitingStorageRequests.delete(message.envelope.requestId)
-        })
+          entry = Object.assign(state, { pump: yield* Effect.forkIn(pumpReplies(requestId, state), runnersScope) })
+          storageRequests.set(requestId, entry)
+          waitingStorageRequests.set(requestId, message)
+        }
+        const current = entry
+        current.messages.add(message)
+        // A waiter that joins an existing entry, such as a duplicate, reads
+        // now. A reply committed without a notification would otherwise wait
+        // for the next poll.
+        storageLatch.openUnsafe()
+        return yield* restore(Fiber.join(current.pump)).pipe(
+          Effect.ensuring(Effect.suspend(() => {
+            current.messages.delete(message)
+            if (current.messages.size === 0) {
+              removeStorageRequest(requestId, current)
+              return Fiber.interrupt(current.pump)
+            }
+            // The storage loop decodes replies using the waiting message.
+            if (waitingStorageRequests.get(requestId) === message) {
+              waitingStorageRequests.set(requestId, current.messages.values().next().value!)
+            }
+            return Effect.void
+          }))
+        )
+      })
+    )
+
+  // A request that may already be processing has to read its replies back
+  // from storage. A local reply handler wakes the storage loop, so streams do
+  // not have to wait for the next poll.
+  const replyFromStorageWithWake = (message: Message.OutgoingRequest<any>) =>
+    Effect.raceFirst(
+      replyFromStorage(message),
+      storage.registerReplyHandler(
+        new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open })
+      ).pipe(Effect.ignore, Effect.andThen(Effect.never))
+    )
+
+  // Stream replies stay on the storage loop, which delivers chunks in order
+  // and waits for their acks.
+  const canReplyLocally = (message: Message.OutgoingRequest<any>) => {
+    const rpc = message.rpc as any as Rpc.AnyWithProps
+    return storage !== MessageStorage.noop && Context.get(rpc.annotations, Persisted) &&
+      !RpcSchema.isStreamSchema(rpc.successSchema)
+  }
+
+  // A request can be processed as soon as it is saved, so its reply handler is
+  // registered first. `registerReplyHandler` registers as soon as it starts,
+  // so a fresh request gets its reply from the handler without reading it
+  // back from storage.
+  const notifyLocalRequest = Effect.fnUntraced(function*(
+    message: Message.OutgoingRequest<any>,
+    notifyEntity: (message: Message.Outgoing<any>) => Effect.Effect<void>
+  ) {
+    const handler = yield* Effect.forkScoped(storage.registerReplyHandler(message), { startImmediately: true })
+    return yield* notifyWith(message, (message, duplicate) => {
+      const outgoing = message as Message.OutgoingRequest<any>
+      // A duplicate is answered under its original request id.
+      if (duplicate) {
+        return Fiber.interrupt(handler).pipe(
+          Effect.andThen(notifyEntity(message)),
+          Effect.andThen(replyFromStorageWithWake(outgoing))
+        )
+      }
+      // The handler can be dropped, for example when its shard moves or a
+      // commit error hides the notification. The reply is then in storage.
+      return notifyEntity(message).pipe(
+        Effect.andThen(Fiber.join(handler)),
+        Effect.catchTag("EntityNotAssignedToRunner", () => replyFromStorage(outgoing))
       )
-  )
+    })
+  }, Effect.scoped)
 
   const storageLatch = Latch.makeUnsafe(false)
   if (storage !== MessageStorage.noop) {
@@ -415,22 +487,23 @@ export const make: (
       })
     },
     notifyLocal(options) {
-      return notifyWith(options.message, (message, duplicate) => {
+      const message = options.message
+      const notifyEntity = (message: Message.Outgoing<any>) =>
+        Effect.catchTag(
+          options.notify(Message.incomingLocalFromOutgoing(message)),
+          "EntityNotAssignedToRunner",
+          () => Effect.void
+        )
+      if (message._tag === "OutgoingRequest" && !options.discard && !options.storageOnly && canReplyLocally(message)) {
+        return notifyLocalRequest(message, notifyEntity)
+      }
+      return notifyWith(message, (message) => {
         if (options.discard || message._tag === "OutgoingEnvelope") {
-          return Effect.catchTag(
-            options.notify(Message.incomingLocalFromOutgoing(message)),
-            "EntityNotAssignedToRunner",
-            () => Effect.void
-          )
-        } else if (!duplicate && options.storageOnly !== true) {
-          return options.notify(Message.incomingLocalFromOutgoing(message)).pipe(
-            Effect.andThen(storage.registerReplyHandler(message)),
-            Effect.catchTag("EntityNotAssignedToRunner", () => replyFromStorage(message))
-          )
+          return notifyEntity(message)
         }
-        return options.notify(Message.incomingLocalFromOutgoing(message)).pipe(
-          Effect.catchTag("EntityNotAssignedToRunner", () => Effect.void),
-          Effect.andThen(replyFromStorage(message))
+        return Effect.andThen(
+          notifyEntity(message),
+          options.storageOnly ? replyFromStorage(message) : replyFromStorageWithWake(message)
         )
       })
     }

package/src/cluster/Sharding.ts

@@ -27,6 +27,7 @@ import * as HashRing from "../HashRing.ts"
 import * as InternalRecord from "../internal/record.ts"
 import * as Latch from "../Latch.ts"
 import * as Layer from "../Layer.ts"
+import * as Metric from "../Metric.ts"
 import * as MutableHashMap from "../MutableHashMap.ts"
 import * as MutableHashSet from "../MutableHashSet.ts"
 import * as MutableRef from "../MutableRef.ts"
@@ -141,6 +142,7 @@ export class Sharding extends Context.Service<Sharding, {
       readonly maxIdleTime?: Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -276,6 +278,17 @@ const make = Effect.gen(function*() {
 
   // the active shards are the ones that we have acquired the lock for
   const acquiredShards = MutableHashSet.empty<ShardId>()
+  // shards still held by this runner while their entities stop and their locks
+  // are released
+  const releasingShards = MutableHashSet.empty<ShardId>()
+  // The releasing gauge reports to the metric registry and attributes the
+  // layer was built with.
+  const updateReleasingShardsGauge = () =>
+    ClusterMetrics.shardsReleasing.updateUnsafe(BigInt(MutableHashSet.size(releasingShards)), services)
+  const updateShardGauges = () => {
+    ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
+    updateReleasingShardsGauge()
+  }
   const activeShardsLatch = yield* Latch.make(false)
   // open once shutdown starts handing every shard off
   const handoffLatch = Latch.makeUnsafe(false)
@@ -343,7 +356,9 @@ const make = Effect.gen(function*() {
   // This should be shutdown last, when all entities have been shutdown, to
   // allow them to move to another runner.
 
-  const releasingShards = MutableHashSet.empty<ShardId>()
+  // Keep startup fibers behind singleton initialization without moving resource
+  // acquisition (and therefore changing the shutdown finalizer order).
+  const singletonsReady = Latch.makeUnsafe(false)
   // Hands every held shard off on shutdown, set when this is a runner
   let handoff: Effect.Effect<void> = Effect.void
   // Shards whose entities must be force interrupted and locks fully released
@@ -352,6 +367,7 @@ const make = Effect.gen(function*() {
   const initialRunnerAddress = getRunnerAddress()
   if (initialRunnerAddress) {
     const selfAddress = initialRunnerAddress
+    updateReleasingShardsGauge()
     yield* Scope.addFinalizerExit(shardingScope, () => {
       // the locks expire over time, so if this fails we ignore it
       return Effect.ignore(runnerStorage.releaseAll(selfAddress))
@@ -405,6 +421,7 @@ const make = Effect.gen(function*() {
       for (const shardId of shardIds) {
         MutableHashSet.remove(releasingShards, shardId)
         MutableHashSet.remove(forceReleasingShards, shardId)
+        updateReleasingShardsGauge()
         // During shutdown, parked waiters are interrupted: this runner will
         // never observe the reply. During a rebalance the waiters are resumed
         // with `EntityNotAssignedToRunner`, and the client falls back to
@@ -419,22 +436,30 @@ const make = Effect.gen(function*() {
     const retryShardRelease =
       (annotations: { readonly fiber: string; readonly shardId?: ShardId }) =>
       <A, E, R>(effect: Effect.Effect<A, E, R>) =>
-        effect.pipe(
-          Effect.sandbox,
-          Effect.tapError((cause) =>
-            Effect.logDebug(`Could not release shards, retrying`, cause).pipe(
-              Effect.annotateLogs({
-                module: "effect/cluster/Sharding",
-                runner: selfAddress,
-                ...annotations
-              }),
-              // Effect.eventually retries immediately, so space failures to
-              // avoid hot-looping while storage is unavailable.
-              Effect.andThen(Effect.sleep(50))
-            )
-          ),
-          Effect.eventually
-        )
+        Effect.suspend(() => {
+          let lastWarning = -Infinity
+          return effect.pipe(
+            Effect.sandbox,
+            Effect.tapError((cause) =>
+              Effect.suspend(() => {
+                const now = clock.currentTimeMillisUnsafe()
+                if (now - lastWarning < 30_000) return Effect.void
+                lastWarning = now
+                return Effect.logWarning("Could not release shards, retrying", cause)
+              }).pipe(
+                Effect.annotateLogs({
+                  module: "effect/cluster/Sharding",
+                  runner: selfAddress,
+                  ...annotations
+                }),
+                // Effect.eventually retries immediately, so space failures to
+                // avoid hot-looping while storage is unavailable.
+                Effect.andThen(Effect.sleep(50))
+              )
+            ),
+            Effect.eventually
+          )
+        })
     const releaseShard = Effect.fnUntraced(
       function*(shardId: ShardId) {
         yield* runShardRelease(
@@ -446,6 +471,8 @@ const make = Effect.gen(function*() {
       (effect, shardId) =>
         effect.pipe(
           retryShardRelease({ fiber: "releaseShard", shardId }),
+          Effect.timed,
+          Effect.flatMap(([duration]) => Metric.update(ClusterMetrics.shardReleaseDuration, duration)),
           FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true })
         )
     )
@@ -454,14 +481,62 @@ const make = Effect.gen(function*() {
     // normal path while it is pending, otherwise the bulk release wipes a lock
     // the runner already considers acquired.
     const forcedShardReleasePending = () => forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) > 0
+    // Repeat while pending, but no more often than every 30 seconds
+    const forcedReleaseWarningInterval = Duration.max(
+      Duration.fromInputUnsafe(config.shardLockExpiration),
+      Duration.seconds(30)
+    )
+    // Names the shards still blocking reacquisition and the entity types and
+    // singletons still stopping on them
+    const warnForcedReleasePending = Effect.suspend(() => {
+      const entityTypes = Arr.empty<string>()
+      for (const [entityType, state] of entityManagers) {
+        for (const shardId of forceReleasingShards) {
+          if (state.manager.hasServersUnsafe(shardId)) {
+            entityTypes.push(entityType)
+            break
+          }
+        }
+      }
+      const singletonNames = new Set<string>()
+      for (const shardId of forceReleasingShards) {
+        for (const [address] of singletons.get(shardId) ?? []) {
+          if (FiberMap.hasUnsafe(singletonFibers, address)) singletonNames.add(address.name)
+        }
+      }
+      MutableHashMap.forEach(stoppingSingletons, (_, address) => {
+        if (MutableHashSet.has(forceReleasingShards, address.shardId)) singletonNames.add(address.name)
+      })
+      return Effect.logWarning(
+        "Forced shard release is still pending; reacquisition is blocked until release completes"
+      ).pipe(Effect.annotateLogs({
+        module: "effect/cluster/Sharding",
+        runner: selfAddress,
+        shardIds: Array.from(forceReleasingShards, String),
+        entityTypes,
+        singletonNames: [...singletonNames]
+      }))
+    })
     const releaseForcedShards = Effect.suspend(() => {
       if (forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) === 0) {
         return Effect.void
       }
       forcedShardReleaseRunning = true
       const shardIds = [...forceReleasingShards]
-      return runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
-        retryShardRelease({ fiber: "releaseForcedShards" }),
+      return Effect.gen(function*() {
+        // Observe independently: a timeout around release cannot interrupt
+        // uninterruptible work, and must not allow locks to be reacquired.
+        const monitor = yield* warnForcedReleasePending.pipe(
+          Effect.repeat(Schedule.spaced(forcedReleaseWarningInterval)),
+          Effect.delay(config.shardLockExpiration),
+          Effect.interruptible,
+          Effect.forkIn(releaseScope)
+        )
+        yield* runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
+          retryShardRelease({ fiber: "releaseForcedShards" }),
+          Effect.ensuring(Fiber.interrupt(monitor))
+        )
+      }).pipe(
         Effect.ensuring(Effect.sync(() => {
           forcedShardReleaseRunning = false
           activeShardsLatch.openUnsafe()
@@ -480,6 +555,7 @@ const make = Effect.gen(function*() {
     })
 
     yield* Effect.gen(function*() {
+      yield* singletonsReady.await
       activeShardsLatch.openUnsafe()
 
       while (true) {
@@ -494,6 +570,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.remove(acquiredShards, shardId)
           MutableHashSet.add(releasingShards, shardId)
         }
+        updateShardGauges()
 
         if (MutableHashSet.size(releasingShards) > 0 || MutableHashSet.size(forceReleasingShards) > 0) {
           yield* Effect.forkIn(syncSingletons, shardingScope)
@@ -555,13 +632,11 @@ const make = Effect.gen(function*() {
           }
           MutableHashSet.add(acquiredShards, shardId)
         }
+        updateShardGauges()
         if (acquired.length > 0) {
           yield* notifyShardLockWaiters
           yield* storageReadLatch.open
           yield* Effect.forkIn(syncSingletons, shardingScope)
-
-          // update metrics
-          ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
         }
         // the shutdown handoff releases the shards acquired here without waiting
         yield* Effect.raceFirst(Effect.sleep(1000), handoffLatch.await)
@@ -589,7 +664,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.add(releasingShards, shardId)
           MutableHashSet.add(forceReleasingShards, shardId)
         }
-        ClusterMetrics.shards.updateUnsafe(BigInt(0), Context.empty())
+        updateShardGauges()
         activeShardsLatch.openUnsafe()
 
         return Effect.gen(function*() {
@@ -632,6 +707,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.add(releasingShards, shardId)
         }
       }
+      updateShardGauges()
       if (MutableHashSet.size(releasingShards) > 0) {
         yield* activeShardsLatch.open
       }
@@ -655,6 +731,7 @@ const make = Effect.gen(function*() {
     yield* Effect.suspend(() => shardLocksHealthyLatch.isOpen() ? refreshShardLocks : probeShardLocks).pipe(
       Effect.repeat(Schedule.fixed(shardLockInterval)),
       Effect.forever,
+      (effect) => Effect.andThen(singletonsReady.await, effect),
       Effect.forkIn(shardingScope)
     )
 
@@ -941,20 +1018,18 @@ const make = Effect.gen(function*() {
           }
           removableNotifications.clear()
         }
-        if (MutableHashSet.size(resetAddresses) > 0) {
-          for (const address of resetAddresses) {
-            yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
-              Effect.annotateLogs({ address })
-            )
-            yield* Effect.forkIn(storage.resetAddress(address), shardingScope)
-          }
-          MutableHashSet.clear(resetAddresses)
+        for (const address of resetAddresses) {
+          yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
+            Effect.annotateLogs({ address })
+          )
         }
-        // Capture claims skipped because of the entity cap. They are reset in
-        // one storage operation after releasing the read lock.
-        let cappedAddressesToReset: Array<EntityAddress> | undefined
-        if (MutableHashSet.size(cappedAddresses) > 0) {
-          cappedAddressesToReset = Arr.fromIterable(cappedAddresses)
+        // Capture claims for unregistered entity types, and claims skipped
+        // because of the entity cap. They are reset in one storage operation
+        // after releasing the read lock, and before the next read starts.
+        let addressesToReset: Array<EntityAddress> | undefined
+        if (MutableHashSet.size(resetAddresses) > 0 || MutableHashSet.size(cappedAddresses) > 0) {
+          addressesToReset = [...resetAddresses, ...cappedAddresses]
+          MutableHashSet.clear(resetAddresses)
           MutableHashSet.clear(cappedAddresses)
         }
 
@@ -966,8 +1041,8 @@ const make = Effect.gen(function*() {
           resetRequestIds.length = 0
         }
 
-        if (cappedAddressesToReset !== undefined) {
-          yield* Effect.ignore(storage.resetAddresses(cappedAddressesToReset))
+        if (addressesToReset !== undefined) {
+          yield* Effect.ignore(storage.resetAddresses(addressesToReset))
         }
 
         // A full batch means more messages could be waiting; start the next
@@ -1796,6 +1871,8 @@ const make = Effect.gen(function*() {
     (stops) => Effect.all(stops, { discard: true })
   )
 
+  singletonsReady.openUnsafe()
+
   // --- Entities ---
 
   const reaper = yield* EntityReaper
@@ -1851,6 +1928,9 @@ const make = Effect.gen(function*() {
       yield* withStorageReadLock(Effect.sync(() => {
         entityRegistrationStartMillis ??= clock.currentTimeMillisUnsafe()
         entityManagers.set(entity.type, state)
+        // persisted messages read before this registration were reset, so
+        // read storage again instead of waiting for the next poll interval
+        storageReadLatch.openUnsafe()
         if (entityManagerLatches.has(entity.type)) {
           entityManagerLatches.get(entity.type)!.openUnsafe()
           entityManagerLatches.delete(entity.type)

package/src/cluster/ShardingConfig.ts

@@ -85,6 +85,8 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
   readonly shardLockExpiration: Duration.Input
   /**
    * Disable the use of advisory locks for shard locking.
+   *
+   * PGlite clients never use advisory locks.
    */
   readonly shardLockDisableAdvisory: boolean
   /**
@@ -97,6 +99,14 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
    * The default capacity of the mailbox for entities.
    */
   readonly entityMailboxCapacity: number | "unbounded"
+  /**
+   * Report the mailbox size of each resident entity with the
+   * `effect_cluster_entity_mailbox_size` gauge, unless overridden by the
+   * `mailboxMetrics` entity option.
+   *
+   * Defaults to `false`.
+   */
+  readonly entityMailboxMetrics: boolean
   /**
    * The maximum number of entities that can be resident on this runner at the
    * same time.
@@ -188,6 +198,7 @@ export const defaults: ShardingConfig["Service"] = {
   shardLockExpiration: Duration.seconds(35),
   shardLockDisableAdvisory: false,
   entityMailboxCapacity: 4096,
+  entityMailboxMetrics: false,
   maxResidentEntities: 10_000,
   unprocessedMessageBatchSize: 1024,
   entityMaxIdleTime: Duration.minutes(1),
@@ -307,6 +318,10 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.entityMailboxCapacity)
     // Config.withDescription("The default capacity of the mailbox for entities.")
   ),
+  entityMailboxMetrics: Config.Boolean("entityMailboxMetrics").pipe(
+    Config.withDefault(defaults.entityMailboxMetrics)
+    // Config.withDescription("Report the mailbox size of each resident entity.")
+  ),
   maxResidentEntities: Config.schema(
     Schema.Int.check(Schema.isGreaterThan(0)),
     "maxResidentEntities"