effect 4.0.5 → 4.0.6

package/src/cluster/Entity.ts

@@ -147,6 +147,7 @@ export interface Entity<
       readonly maxIdleTime?: Duration.Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -191,6 +192,7 @@ export interface Entity<
     options?: {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -287,6 +289,7 @@ const Proto = {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -335,6 +338,7 @@ const Proto = {
     options?: {
       readonly maxIdleTime?: Duration.Input | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined

package/src/cluster/MessageStorage.ts

@@ -119,6 +119,13 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 
   /**
    * For locally sent messages, register a handler to process the replies.
+   *
+   * **Details**
+   *
+   * The handler must be registered as soon as the effect starts, before it
+   * first suspends. Callers fork it with `startImmediately` before saving or
+   * notifying the request, and rely on it to see every reply saved after that
+   * point. The effect completes when a `WithExit` reply arrives.
    */
   readonly registerReplyHandler: <R extends Rpc.Any>(
     message: Message.OutgoingRequest<R> | Message.IncomingRequest<R>

package/src/cluster/Runners.ts

@@ -265,17 +265,15 @@ export const make: (
 
   // The pump is owned by the shared entry rather than the first caller, so any
   // waiter can be interrupted without stopping delivery to the others.
+  //
+  // `replyFromStorage` registers the request with the storage loop before the
+  // pump starts, so the pump only re-registers after a stream chunk is acked.
   const pumpReplies = Effect.fnUntraced(function*(
     requestId: Snowflake.Snowflake,
     entry: Omit<StorageRequestEntry, "pump">
   ) {
     while (true) {
       // wait for the storage loop to notify us
-      entry.latch.closeUnsafe()
-      const message = entry.messages.values().next().value
-      if (!message) return
-      waitingStorageRequests.set(requestId, message)
-      storageLatch.openUnsafe()
       yield* entry.latch.await
 
       // send the replies back
@@ -300,6 +298,13 @@ export const make: (
         yield* entry.latch.await
       }
       entry.replies = []
+
+      // ask the storage loop for the next chunk
+      entry.latch.closeUnsafe()
+      const message = entry.messages.values().next().value
+      if (!message) return
+      waitingStorageRequests.set(requestId, message)
+      storageLatch.openUnsafe()
     }
   })
 
@@ -316,9 +321,14 @@ export const make: (
           }
           entry = Object.assign(state, { pump: yield* Effect.forkIn(pumpReplies(requestId, state), runnersScope) })
           storageRequests.set(requestId, entry)
+          waitingStorageRequests.set(requestId, message)
         }
         const current = entry
         current.messages.add(message)
+        // A waiter that joins an existing entry, such as a duplicate, reads
+        // now. A reply committed without a notification would otherwise wait
+        // for the next poll.
+        storageLatch.openUnsafe()
         return yield* restore(Fiber.join(current.pump)).pipe(
           Effect.ensuring(Effect.suspend(() => {
             current.messages.delete(message)
@@ -336,6 +346,52 @@ export const make: (
       })
     )
 
+  // A request that may already be processing has to read its replies back
+  // from storage. A local reply handler wakes the storage loop, so streams do
+  // not have to wait for the next poll.
+  const replyFromStorageWithWake = (message: Message.OutgoingRequest<any>) =>
+    Effect.raceFirst(
+      replyFromStorage(message),
+      storage.registerReplyHandler(
+        new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open })
+      ).pipe(Effect.ignore, Effect.andThen(Effect.never))
+    )
+
+  // Stream replies stay on the storage loop, which delivers chunks in order
+  // and waits for their acks.
+  const canReplyLocally = (message: Message.OutgoingRequest<any>) => {
+    const rpc = message.rpc as any as Rpc.AnyWithProps
+    return storage !== MessageStorage.noop && Context.get(rpc.annotations, Persisted) &&
+      !RpcSchema.isStreamSchema(rpc.successSchema)
+  }
+
+  // A request can be processed as soon as it is saved, so its reply handler is
+  // registered first. `registerReplyHandler` registers as soon as it starts,
+  // so a fresh request gets its reply from the handler without reading it
+  // back from storage.
+  const notifyLocalRequest = Effect.fnUntraced(function*(
+    message: Message.OutgoingRequest<any>,
+    notifyEntity: (message: Message.Outgoing<any>) => Effect.Effect<void>
+  ) {
+    const handler = yield* Effect.forkScoped(storage.registerReplyHandler(message), { startImmediately: true })
+    return yield* notifyWith(message, (message, duplicate) => {
+      const outgoing = message as Message.OutgoingRequest<any>
+      // A duplicate is answered under its original request id.
+      if (duplicate) {
+        return Fiber.interrupt(handler).pipe(
+          Effect.andThen(notifyEntity(message)),
+          Effect.andThen(replyFromStorageWithWake(outgoing))
+        )
+      }
+      // The handler can be dropped, for example when its shard moves or a
+      // commit error hides the notification. The reply is then in storage.
+      return notifyEntity(message).pipe(
+        Effect.andThen(Fiber.join(handler)),
+        Effect.catchTag("EntityNotAssignedToRunner", () => replyFromStorage(outgoing))
+      )
+    })
+  }, Effect.scoped)
+
   const storageLatch = Latch.makeUnsafe(false)
   if (storage !== MessageStorage.noop) {
     yield* Effect.gen(function*() {
@@ -431,28 +487,23 @@ export const make: (
       })
     },
     notifyLocal(options) {
-      return notifyWith(options.message, (message) => {
+      const message = options.message
+      const notifyEntity = (message: Message.Outgoing<any>) =>
+        Effect.catchTag(
+          options.notify(Message.incomingLocalFromOutgoing(message)),
+          "EntityNotAssignedToRunner",
+          () => Effect.void
+        )
+      if (message._tag === "OutgoingRequest" && !options.discard && !options.storageOnly && canReplyLocally(message)) {
+        return notifyLocalRequest(message, notifyEntity)
+      }
+      return notifyWith(message, (message) => {
         if (options.discard || message._tag === "OutgoingEnvelope") {
-          return Effect.catchTag(
-            options.notify(Message.incomingLocalFromOutgoing(message)),
-            "EntityNotAssignedToRunner",
-            () => Effect.void
-          )
+          return notifyEntity(message)
         }
-        // The request can be processed as soon as it is saved, so replies are
-        // always read back from storage. The local reply handler only wakes the
-        // storage loop, so streams do not have to wait for the next poll.
-        const readReplies = options.storageOnly
-          ? replyFromStorage(message)
-          : Effect.raceFirst(
-            replyFromStorage(message),
-            storage.registerReplyHandler(
-              new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open })
-            ).pipe(Effect.ignore, Effect.andThen(Effect.never))
-          )
-        return options.notify(Message.incomingLocalFromOutgoing(message)).pipe(
-          Effect.catchTag("EntityNotAssignedToRunner", () => Effect.void),
-          Effect.andThen(readReplies)
+        return Effect.andThen(
+          notifyEntity(message),
+          options.storageOnly ? replyFromStorage(message) : replyFromStorageWithWake(message)
         )
       })
     }

package/src/cluster/Sharding.ts

@@ -27,6 +27,7 @@ import * as HashRing from "../HashRing.ts"
 import * as InternalRecord from "../internal/record.ts"
 import * as Latch from "../Latch.ts"
 import * as Layer from "../Layer.ts"
+import * as Metric from "../Metric.ts"
 import * as MutableHashMap from "../MutableHashMap.ts"
 import * as MutableHashSet from "../MutableHashSet.ts"
 import * as MutableRef from "../MutableRef.ts"
@@ -141,6 +142,7 @@ export class Sharding extends Context.Service<Sharding, {
       readonly maxIdleTime?: Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -276,6 +278,17 @@ const make = Effect.gen(function*() {
 
   // the active shards are the ones that we have acquired the lock for
   const acquiredShards = MutableHashSet.empty<ShardId>()
+  // shards still held by this runner while their entities stop and their locks
+  // are released
+  const releasingShards = MutableHashSet.empty<ShardId>()
+  // The releasing gauge reports to the metric registry and attributes the
+  // layer was built with.
+  const updateReleasingShardsGauge = () =>
+    ClusterMetrics.shardsReleasing.updateUnsafe(BigInt(MutableHashSet.size(releasingShards)), services)
+  const updateShardGauges = () => {
+    ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
+    updateReleasingShardsGauge()
+  }
   const activeShardsLatch = yield* Latch.make(false)
   // open once shutdown starts handing every shard off
   const handoffLatch = Latch.makeUnsafe(false)
@@ -343,7 +356,9 @@ const make = Effect.gen(function*() {
   // This should be shutdown last, when all entities have been shutdown, to
   // allow them to move to another runner.
 
-  const releasingShards = MutableHashSet.empty<ShardId>()
+  // Keep startup fibers behind singleton initialization without moving resource
+  // acquisition (and therefore changing the shutdown finalizer order).
+  const singletonsReady = Latch.makeUnsafe(false)
   // Hands every held shard off on shutdown, set when this is a runner
   let handoff: Effect.Effect<void> = Effect.void
   // Shards whose entities must be force interrupted and locks fully released
@@ -352,6 +367,7 @@ const make = Effect.gen(function*() {
   const initialRunnerAddress = getRunnerAddress()
   if (initialRunnerAddress) {
     const selfAddress = initialRunnerAddress
+    updateReleasingShardsGauge()
     yield* Scope.addFinalizerExit(shardingScope, () => {
       // the locks expire over time, so if this fails we ignore it
       return Effect.ignore(runnerStorage.releaseAll(selfAddress))
@@ -405,6 +421,7 @@ const make = Effect.gen(function*() {
       for (const shardId of shardIds) {
         MutableHashSet.remove(releasingShards, shardId)
         MutableHashSet.remove(forceReleasingShards, shardId)
+        updateReleasingShardsGauge()
         // During shutdown, parked waiters are interrupted: this runner will
         // never observe the reply. During a rebalance the waiters are resumed
         // with `EntityNotAssignedToRunner`, and the client falls back to
@@ -419,22 +436,30 @@ const make = Effect.gen(function*() {
     const retryShardRelease =
       (annotations: { readonly fiber: string; readonly shardId?: ShardId }) =>
       <A, E, R>(effect: Effect.Effect<A, E, R>) =>
-        effect.pipe(
-          Effect.sandbox,
-          Effect.tapError((cause) =>
-            Effect.logDebug(`Could not release shards, retrying`, cause).pipe(
-              Effect.annotateLogs({
-                module: "effect/cluster/Sharding",
-                runner: selfAddress,
-                ...annotations
-              }),
-              // Effect.eventually retries immediately, so space failures to
-              // avoid hot-looping while storage is unavailable.
-              Effect.andThen(Effect.sleep(50))
-            )
-          ),
-          Effect.eventually
-        )
+        Effect.suspend(() => {
+          let lastWarning = -Infinity
+          return effect.pipe(
+            Effect.sandbox,
+            Effect.tapError((cause) =>
+              Effect.suspend(() => {
+                const now = clock.currentTimeMillisUnsafe()
+                if (now - lastWarning < 30_000) return Effect.void
+                lastWarning = now
+                return Effect.logWarning("Could not release shards, retrying", cause)
+              }).pipe(
+                Effect.annotateLogs({
+                  module: "effect/cluster/Sharding",
+                  runner: selfAddress,
+                  ...annotations
+                }),
+                // Effect.eventually retries immediately, so space failures to
+                // avoid hot-looping while storage is unavailable.
+                Effect.andThen(Effect.sleep(50))
+              )
+            ),
+            Effect.eventually
+          )
+        })
     const releaseShard = Effect.fnUntraced(
       function*(shardId: ShardId) {
         yield* runShardRelease(
@@ -446,6 +471,8 @@ const make = Effect.gen(function*() {
       (effect, shardId) =>
         effect.pipe(
           retryShardRelease({ fiber: "releaseShard", shardId }),
+          Effect.timed,
+          Effect.flatMap(([duration]) => Metric.update(ClusterMetrics.shardReleaseDuration, duration)),
           FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true })
         )
     )
@@ -454,14 +481,62 @@ const make = Effect.gen(function*() {
     // normal path while it is pending, otherwise the bulk release wipes a lock
     // the runner already considers acquired.
     const forcedShardReleasePending = () => forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) > 0
+    // Repeat while pending, but no more often than every 30 seconds
+    const forcedReleaseWarningInterval = Duration.max(
+      Duration.fromInputUnsafe(config.shardLockExpiration),
+      Duration.seconds(30)
+    )
+    // Names the shards still blocking reacquisition and the entity types and
+    // singletons still stopping on them
+    const warnForcedReleasePending = Effect.suspend(() => {
+      const entityTypes = Arr.empty<string>()
+      for (const [entityType, state] of entityManagers) {
+        for (const shardId of forceReleasingShards) {
+          if (state.manager.hasServersUnsafe(shardId)) {
+            entityTypes.push(entityType)
+            break
+          }
+        }
+      }
+      const singletonNames = new Set<string>()
+      for (const shardId of forceReleasingShards) {
+        for (const [address] of singletons.get(shardId) ?? []) {
+          if (FiberMap.hasUnsafe(singletonFibers, address)) singletonNames.add(address.name)
+        }
+      }
+      MutableHashMap.forEach(stoppingSingletons, (_, address) => {
+        if (MutableHashSet.has(forceReleasingShards, address.shardId)) singletonNames.add(address.name)
+      })
+      return Effect.logWarning(
+        "Forced shard release is still pending; reacquisition is blocked until release completes"
+      ).pipe(Effect.annotateLogs({
+        module: "effect/cluster/Sharding",
+        runner: selfAddress,
+        shardIds: Array.from(forceReleasingShards, String),
+        entityTypes,
+        singletonNames: [...singletonNames]
+      }))
+    })
     const releaseForcedShards = Effect.suspend(() => {
       if (forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) === 0) {
         return Effect.void
       }
       forcedShardReleaseRunning = true
       const shardIds = [...forceReleasingShards]
-      return runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
-        retryShardRelease({ fiber: "releaseForcedShards" }),
+      return Effect.gen(function*() {
+        // Observe independently: a timeout around release cannot interrupt
+        // uninterruptible work, and must not allow locks to be reacquired.
+        const monitor = yield* warnForcedReleasePending.pipe(
+          Effect.repeat(Schedule.spaced(forcedReleaseWarningInterval)),
+          Effect.delay(config.shardLockExpiration),
+          Effect.interruptible,
+          Effect.forkIn(releaseScope)
+        )
+        yield* runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
+          retryShardRelease({ fiber: "releaseForcedShards" }),
+          Effect.ensuring(Fiber.interrupt(monitor))
+        )
+      }).pipe(
         Effect.ensuring(Effect.sync(() => {
           forcedShardReleaseRunning = false
           activeShardsLatch.openUnsafe()
@@ -480,6 +555,7 @@ const make = Effect.gen(function*() {
     })
 
     yield* Effect.gen(function*() {
+      yield* singletonsReady.await
       activeShardsLatch.openUnsafe()
 
       while (true) {
@@ -494,6 +570,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.remove(acquiredShards, shardId)
           MutableHashSet.add(releasingShards, shardId)
         }
+        updateShardGauges()
 
         if (MutableHashSet.size(releasingShards) > 0 || MutableHashSet.size(forceReleasingShards) > 0) {
           yield* Effect.forkIn(syncSingletons, shardingScope)
@@ -555,13 +632,11 @@ const make = Effect.gen(function*() {
           }
           MutableHashSet.add(acquiredShards, shardId)
         }
+        updateShardGauges()
         if (acquired.length > 0) {
           yield* notifyShardLockWaiters
           yield* storageReadLatch.open
           yield* Effect.forkIn(syncSingletons, shardingScope)
-
-          // update metrics
-          ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
         }
         // the shutdown handoff releases the shards acquired here without waiting
         yield* Effect.raceFirst(Effect.sleep(1000), handoffLatch.await)
@@ -589,7 +664,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.add(releasingShards, shardId)
           MutableHashSet.add(forceReleasingShards, shardId)
         }
-        ClusterMetrics.shards.updateUnsafe(BigInt(0), Context.empty())
+        updateShardGauges()
         activeShardsLatch.openUnsafe()
 
         return Effect.gen(function*() {
@@ -632,6 +707,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.add(releasingShards, shardId)
         }
       }
+      updateShardGauges()
       if (MutableHashSet.size(releasingShards) > 0) {
         yield* activeShardsLatch.open
       }
@@ -655,6 +731,7 @@ const make = Effect.gen(function*() {
     yield* Effect.suspend(() => shardLocksHealthyLatch.isOpen() ? refreshShardLocks : probeShardLocks).pipe(
       Effect.repeat(Schedule.fixed(shardLockInterval)),
       Effect.forever,
+      (effect) => Effect.andThen(singletonsReady.await, effect),
       Effect.forkIn(shardingScope)
     )
 
@@ -941,20 +1018,18 @@ const make = Effect.gen(function*() {
           }
           removableNotifications.clear()
         }
-        if (MutableHashSet.size(resetAddresses) > 0) {
-          for (const address of resetAddresses) {
-            yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
-              Effect.annotateLogs({ address })
-            )
-            yield* Effect.forkIn(storage.resetAddress(address), shardingScope)
-          }
-          MutableHashSet.clear(resetAddresses)
+        for (const address of resetAddresses) {
+          yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
+            Effect.annotateLogs({ address })
+          )
         }
-        // Capture claims skipped because of the entity cap. They are reset in
-        // one storage operation after releasing the read lock.
-        let cappedAddressesToReset: Array<EntityAddress> | undefined
-        if (MutableHashSet.size(cappedAddresses) > 0) {
-          cappedAddressesToReset = Arr.fromIterable(cappedAddresses)
+        // Capture claims for unregistered entity types, and claims skipped
+        // because of the entity cap. They are reset in one storage operation
+        // after releasing the read lock, and before the next read starts.
+        let addressesToReset: Array<EntityAddress> | undefined
+        if (MutableHashSet.size(resetAddresses) > 0 || MutableHashSet.size(cappedAddresses) > 0) {
+          addressesToReset = [...resetAddresses, ...cappedAddresses]
+          MutableHashSet.clear(resetAddresses)
           MutableHashSet.clear(cappedAddresses)
         }
 
@@ -966,8 +1041,8 @@ const make = Effect.gen(function*() {
           resetRequestIds.length = 0
         }
 
-        if (cappedAddressesToReset !== undefined) {
-          yield* Effect.ignore(storage.resetAddresses(cappedAddressesToReset))
+        if (addressesToReset !== undefined) {
+          yield* Effect.ignore(storage.resetAddresses(addressesToReset))
         }
 
         // A full batch means more messages could be waiting; start the next
@@ -1796,6 +1871,8 @@ const make = Effect.gen(function*() {
     (stops) => Effect.all(stops, { discard: true })
   )
 
+  singletonsReady.openUnsafe()
+
   // --- Entities ---
 
   const reaper = yield* EntityReaper
@@ -1851,6 +1928,9 @@ const make = Effect.gen(function*() {
       yield* withStorageReadLock(Effect.sync(() => {
         entityRegistrationStartMillis ??= clock.currentTimeMillisUnsafe()
         entityManagers.set(entity.type, state)
+        // persisted messages read before this registration were reset, so
+        // read storage again instead of waiting for the next poll interval
+        storageReadLatch.openUnsafe()
         if (entityManagerLatches.has(entity.type)) {
           entityManagerLatches.get(entity.type)!.openUnsafe()
           entityManagerLatches.delete(entity.type)

package/src/cluster/ShardingConfig.ts

@@ -85,6 +85,8 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
   readonly shardLockExpiration: Duration.Input
   /**
    * Disable the use of advisory locks for shard locking.
+   *
+   * PGlite clients never use advisory locks.
    */
   readonly shardLockDisableAdvisory: boolean
   /**
@@ -97,6 +99,14 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
    * The default capacity of the mailbox for entities.
    */
   readonly entityMailboxCapacity: number | "unbounded"
+  /**
+   * Report the mailbox size of each resident entity with the
+   * `effect_cluster_entity_mailbox_size` gauge, unless overridden by the
+   * `mailboxMetrics` entity option.
+   *
+   * Defaults to `false`.
+   */
+  readonly entityMailboxMetrics: boolean
   /**
    * The maximum number of entities that can be resident on this runner at the
    * same time.
@@ -188,6 +198,7 @@ export const defaults: ShardingConfig["Service"] = {
   shardLockExpiration: Duration.seconds(35),
   shardLockDisableAdvisory: false,
   entityMailboxCapacity: 4096,
+  entityMailboxMetrics: false,
   maxResidentEntities: 10_000,
   unprocessedMessageBatchSize: 1024,
   entityMaxIdleTime: Duration.minutes(1),
@@ -307,6 +318,10 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.entityMailboxCapacity)
     // Config.withDescription("The default capacity of the mailbox for entities.")
   ),
+  entityMailboxMetrics: Config.Boolean("entityMailboxMetrics").pipe(
+    Config.withDefault(defaults.entityMailboxMetrics)
+    // Config.withDescription("Report the mailbox size of each resident entity.")
+  ),
   maxResidentEntities: Config.schema(
     Schema.Int.check(Schema.isGreaterThan(0)),
     "maxResidentEntities"