effect 4.0.5 → 4.0.7

package/src/cluster/Sharding.ts

@@ -27,6 +27,7 @@ import * as HashRing from "../HashRing.ts"
 import * as InternalRecord from "../internal/record.ts"
 import * as Latch from "../Latch.ts"
 import * as Layer from "../Layer.ts"
+import * as Metric from "../Metric.ts"
 import * as MutableHashMap from "../MutableHashMap.ts"
 import * as MutableHashSet from "../MutableHashSet.ts"
 import * as MutableRef from "../MutableRef.ts"
@@ -141,6 +142,7 @@ export class Sharding extends Context.Service<Sharding, {
       readonly maxIdleTime?: Input | undefined
       readonly concurrency?: number | "unbounded" | undefined
       readonly mailboxCapacity?: number | "unbounded" | undefined
+      readonly mailboxMetrics?: boolean | undefined
       readonly disableFatalDefects?: boolean | undefined
       readonly defectRetryPolicy?: Schedule.Schedule<any, unknown> | undefined
       readonly spanAttributes?: Record<string, string> | undefined
@@ -276,7 +278,36 @@ const make = Effect.gen(function*() {
 
   // the active shards are the ones that we have acquired the lock for
   const acquiredShards = MutableHashSet.empty<ShardId>()
+  // shards still held by this runner while their entities stop and their locks
+  // are released
+  const releasingShards = MutableHashSet.empty<ShardId>()
+  // The releasing gauge reports to the metric registry and attributes the
+  // layer was built with.
+  const updateReleasingShardsGauge = () =>
+    ClusterMetrics.shardsReleasing.updateUnsafe(BigInt(MutableHashSet.size(releasingShards)), services)
+  const updateShardGauges = () => {
+    ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
+    updateReleasingShardsGauge()
+  }
   const activeShardsLatch = yield* Latch.make(false)
+
+  // Applies the result of a lock refresh: refreshed shards it did not return,
+  // and returned shards no longer assigned to this runner, move from
+  // `acquiredShards` to `releasingShards`. Shards acquired during the refresh
+  // are absent from its response, so only the refreshed ones are considered.
+  const releaseLostShards = (refreshed: ReadonlyArray<ShardId>, acquired: ReadonlyArray<ShardId>) => {
+    for (const shardId of refreshed) {
+      if (
+        MutableHashSet.has(acquiredShards, shardId) &&
+        (!acquired.includes(shardId) || !MutableHashSet.has(selfShards, shardId))
+      ) {
+        MutableHashSet.remove(acquiredShards, shardId)
+        MutableHashSet.add(releasingShards, shardId)
+      }
+    }
+    updateShardGauges()
+    if (MutableHashSet.size(releasingShards) > 0) activeShardsLatch.openUnsafe()
+  }
   // open once shutdown starts handing every shard off
   const handoffLatch = Latch.makeUnsafe(false)
   // open once the handoff has released every shard lock
@@ -343,7 +374,9 @@ const make = Effect.gen(function*() {
   // This should be shutdown last, when all entities have been shutdown, to
   // allow them to move to another runner.
 
-  const releasingShards = MutableHashSet.empty<ShardId>()
+  // Keep startup fibers behind singleton initialization without moving resource
+  // acquisition (and therefore changing the shutdown finalizer order).
+  const singletonsReady = Latch.makeUnsafe(false)
   // Hands every held shard off on shutdown, set when this is a runner
   let handoff: Effect.Effect<void> = Effect.void
   // Shards whose entities must be force interrupted and locks fully released
@@ -352,6 +385,7 @@ const make = Effect.gen(function*() {
   const initialRunnerAddress = getRunnerAddress()
   if (initialRunnerAddress) {
     const selfAddress = initialRunnerAddress
+    updateReleasingShardsGauge()
     yield* Scope.addFinalizerExit(shardingScope, () => {
       // the locks expire over time, so if this fails we ignore it
       return Effect.ignore(runnerStorage.releaseAll(selfAddress))
@@ -405,6 +439,7 @@ const make = Effect.gen(function*() {
       for (const shardId of shardIds) {
         MutableHashSet.remove(releasingShards, shardId)
         MutableHashSet.remove(forceReleasingShards, shardId)
+        updateReleasingShardsGauge()
         // During shutdown, parked waiters are interrupted: this runner will
         // never observe the reply. During a rebalance the waiters are resumed
         // with `EntityNotAssignedToRunner`, and the client falls back to
@@ -419,22 +454,30 @@ const make = Effect.gen(function*() {
     const retryShardRelease =
       (annotations: { readonly fiber: string; readonly shardId?: ShardId }) =>
       <A, E, R>(effect: Effect.Effect<A, E, R>) =>
-        effect.pipe(
-          Effect.sandbox,
-          Effect.tapError((cause) =>
-            Effect.logDebug(`Could not release shards, retrying`, cause).pipe(
-              Effect.annotateLogs({
-                module: "effect/cluster/Sharding",
-                runner: selfAddress,
-                ...annotations
-              }),
-              // Effect.eventually retries immediately, so space failures to
-              // avoid hot-looping while storage is unavailable.
-              Effect.andThen(Effect.sleep(50))
-            )
-          ),
-          Effect.eventually
-        )
+        Effect.suspend(() => {
+          let lastWarning = -Infinity
+          return effect.pipe(
+            Effect.sandbox,
+            Effect.tapError((cause) =>
+              Effect.suspend(() => {
+                const now = clock.currentTimeMillisUnsafe()
+                if (now - lastWarning < 30_000) return Effect.void
+                lastWarning = now
+                return Effect.logWarning("Could not release shards, retrying", cause)
+              }).pipe(
+                Effect.annotateLogs({
+                  module: "effect/cluster/Sharding",
+                  runner: selfAddress,
+                  ...annotations
+                }),
+                // Effect.eventually retries immediately, so space failures to
+                // avoid hot-looping while storage is unavailable.
+                Effect.andThen(Effect.sleep(50))
+              )
+            ),
+            Effect.eventually
+          )
+        })
     const releaseShard = Effect.fnUntraced(
       function*(shardId: ShardId) {
         yield* runShardRelease(
@@ -446,6 +489,8 @@ const make = Effect.gen(function*() {
       (effect, shardId) =>
         effect.pipe(
           retryShardRelease({ fiber: "releaseShard", shardId }),
+          Effect.timed,
+          Effect.flatMap(([duration]) => Metric.update(ClusterMetrics.shardReleaseDuration, duration)),
           FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true })
         )
     )
@@ -454,14 +499,62 @@ const make = Effect.gen(function*() {
     // normal path while it is pending, otherwise the bulk release wipes a lock
     // the runner already considers acquired.
     const forcedShardReleasePending = () => forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) > 0
+    // Repeat while pending, but no more often than every 30 seconds
+    const forcedReleaseWarningInterval = Duration.max(
+      Duration.fromInputUnsafe(config.shardLockExpiration),
+      Duration.seconds(30)
+    )
+    // Names the shards still blocking reacquisition and the entity types and
+    // singletons still stopping on them
+    const warnForcedReleasePending = Effect.suspend(() => {
+      const entityTypes = Arr.empty<string>()
+      for (const [entityType, state] of entityManagers) {
+        for (const shardId of forceReleasingShards) {
+          if (state.manager.hasServersUnsafe(shardId)) {
+            entityTypes.push(entityType)
+            break
+          }
+        }
+      }
+      const singletonNames = new Set<string>()
+      for (const shardId of forceReleasingShards) {
+        for (const [address] of singletons.get(shardId) ?? []) {
+          if (FiberMap.hasUnsafe(singletonFibers, address)) singletonNames.add(address.name)
+        }
+      }
+      MutableHashMap.forEach(stoppingSingletons, (_, address) => {
+        if (MutableHashSet.has(forceReleasingShards, address.shardId)) singletonNames.add(address.name)
+      })
+      return Effect.logWarning(
+        "Forced shard release is still pending; reacquisition is blocked until release completes"
+      ).pipe(Effect.annotateLogs({
+        module: "effect/cluster/Sharding",
+        runner: selfAddress,
+        shardIds: Array.from(forceReleasingShards, String),
+        entityTypes,
+        singletonNames: [...singletonNames]
+      }))
+    })
     const releaseForcedShards = Effect.suspend(() => {
       if (forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) === 0) {
         return Effect.void
       }
       forcedShardReleaseRunning = true
       const shardIds = [...forceReleasingShards]
-      return runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
-        retryShardRelease({ fiber: "releaseForcedShards" }),
+      return Effect.gen(function*() {
+        // Observe independently: a timeout around release cannot interrupt
+        // uninterruptible work, and must not allow locks to be reacquired.
+        const monitor = yield* warnForcedReleasePending.pipe(
+          Effect.repeat(Schedule.spaced(forcedReleaseWarningInterval)),
+          Effect.delay(config.shardLockExpiration),
+          Effect.interruptible,
+          Effect.forkIn(releaseScope)
+        )
+        yield* runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe(
+          retryShardRelease({ fiber: "releaseForcedShards" }),
+          Effect.ensuring(Fiber.interrupt(monitor))
+        )
+      }).pipe(
         Effect.ensuring(Effect.sync(() => {
           forcedShardReleaseRunning = false
           activeShardsLatch.openUnsafe()
@@ -480,6 +573,7 @@ const make = Effect.gen(function*() {
     })
 
     yield* Effect.gen(function*() {
+      yield* singletonsReady.await
       activeShardsLatch.openUnsafe()
 
       while (true) {
@@ -494,6 +588,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.remove(acquiredShards, shardId)
           MutableHashSet.add(releasingShards, shardId)
         }
+        updateShardGauges()
 
         if (MutableHashSet.size(releasingShards) > 0 || MutableHashSet.size(forceReleasingShards) > 0) {
           yield* Effect.forkIn(syncSingletons, shardingScope)
@@ -555,13 +650,11 @@ const make = Effect.gen(function*() {
           }
           MutableHashSet.add(acquiredShards, shardId)
         }
+        updateShardGauges()
         if (acquired.length > 0) {
           yield* notifyShardLockWaiters
           yield* storageReadLatch.open
           yield* Effect.forkIn(syncSingletons, shardingScope)
-
-          // update metrics
-          ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
         }
         // the shutdown handoff releases the shards acquired here without waiting
         yield* Effect.raceFirst(Effect.sleep(1000), handoffLatch.await)
@@ -589,7 +682,7 @@ const make = Effect.gen(function*() {
           MutableHashSet.add(releasingShards, shardId)
           MutableHashSet.add(forceReleasingShards, shardId)
         }
-        ClusterMetrics.shards.updateUnsafe(BigInt(0), Context.empty())
+        updateShardGauges()
         activeShardsLatch.openUnsafe()
 
         return Effect.gen(function*() {
@@ -615,26 +708,12 @@ const make = Effect.gen(function*() {
       return Effect.logInfo("Shard lock storage has recovered")
     })
 
-    const refreshShardLocks = Effect.gen(function*() {
+    const refreshShardLocks = Effect.suspend(() => {
       const refreshed = [...acquiredShards, ...releasingShards]
-      const acquired = yield* runnerStorage.refresh(selfAddress, refreshed)
-      // Shards acquired during the request are absent from its response.
-      for (const shardId of refreshed) {
-        if (MutableHashSet.has(acquiredShards, shardId) && !acquired.includes(shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      for (let i = 0; i < acquired.length; i++) {
-        const shardId = acquired[i]
-        if (!MutableHashSet.has(selfShards, shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      if (MutableHashSet.size(releasingShards) > 0) {
-        yield* activeShardsLatch.open
-      }
+      return Effect.map(
+        runnerStorage.refresh(selfAddress, refreshed),
+        (acquired) => releaseLostShards(refreshed, acquired)
+      )
     }).pipe(
       Effect.retry({
         times: 5,
@@ -655,6 +734,7 @@ const make = Effect.gen(function*() {
     yield* Effect.suspend(() => shardLocksHealthyLatch.isOpen() ? refreshShardLocks : probeShardLocks).pipe(
       Effect.repeat(Schedule.fixed(shardLockInterval)),
       Effect.forever,
+      (effect) => Effect.andThen(singletonsReady.await, effect),
       Effect.forkIn(shardingScope)
     )
 
@@ -941,20 +1021,18 @@ const make = Effect.gen(function*() {
           }
           removableNotifications.clear()
         }
-        if (MutableHashSet.size(resetAddresses) > 0) {
-          for (const address of resetAddresses) {
-            yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
-              Effect.annotateLogs({ address })
-            )
-            yield* Effect.forkIn(storage.resetAddress(address), shardingScope)
-          }
-          MutableHashSet.clear(resetAddresses)
+        for (const address of resetAddresses) {
+          yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe(
+            Effect.annotateLogs({ address })
+          )
         }
-        // Capture claims skipped because of the entity cap. They are reset in
-        // one storage operation after releasing the read lock.
-        let cappedAddressesToReset: Array<EntityAddress> | undefined
-        if (MutableHashSet.size(cappedAddresses) > 0) {
-          cappedAddressesToReset = Arr.fromIterable(cappedAddresses)
+        // Capture claims for unregistered entity types, and claims skipped
+        // because of the entity cap. They are reset in one storage operation
+        // after releasing the read lock, and before the next read starts.
+        let addressesToReset: Array<EntityAddress> | undefined
+        if (MutableHashSet.size(resetAddresses) > 0 || MutableHashSet.size(cappedAddresses) > 0) {
+          addressesToReset = [...resetAddresses, ...cappedAddresses]
+          MutableHashSet.clear(resetAddresses)
           MutableHashSet.clear(cappedAddresses)
         }
 
@@ -966,8 +1044,8 @@ const make = Effect.gen(function*() {
           resetRequestIds.length = 0
         }
 
-        if (cappedAddressesToReset !== undefined) {
-          yield* Effect.ignore(storage.resetAddresses(cappedAddressesToReset))
+        if (addressesToReset !== undefined) {
+          yield* Effect.ignore(storage.resetAddresses(addressesToReset))
         }
 
         // A full batch means more messages could be waiting; start the next
@@ -1466,6 +1544,27 @@ const make = Effect.gen(function*() {
   }
   const clientRequests = new Map<Snowflake.Snowflake, ClientRequestEntry>()
 
+  // During shutdown, abandon local callers parked on persisted requests that a
+  // retiring entity never started. They are served by the next owner. Callers
+  // on a runner that keeps running are left waiting, so they stay durable.
+  const abandonUndelivered = (
+    address: EntityAddress,
+    isDelivered: (requestId: Snowflake.Snowflake) => boolean
+  ): Effect.Effect<void> =>
+    Effect.suspend(() => {
+      if (!MutableRef.get(isShutdown)) return Effect.void
+      const abandon: Array<Effect.Effect<void>> = []
+      for (const [requestId, entry] of clientRequests) {
+        if (
+          !Context.get(entry.message.annotations, Persisted) ||
+          !Equal.equals(entry.message.envelope.address, address) ||
+          isDelivered(requestId)
+        ) continue
+        abandon.push(storage.unregisterReplyHandler(requestId, { interrupt: true }))
+      }
+      return Effect.all(abandon, { discard: true })
+    })
+
   const clients: ResourceMap<
     Entity<any, any>,
     (entityId: string) => RpcClient.RpcClient<
@@ -1796,11 +1895,49 @@ const make = Effect.gen(function*() {
     (stops) => Effect.all(stops, { discard: true })
   )
 
+  singletonsReady.openUnsafe()
+
   // --- Entities ---
 
   const reaper = yield* EntityReaper
   const entityManagerLatches = new Map<string, Latch.Latch>()
 
+  // This runner's view of its shards lags lock storage by up to one refresh,
+  // or longer while the process is paused. So the final reply of a persisted
+  // request is stored only once a refresh of its shard's lock confirms it.
+  // Anything short of that, including a refresh that failed, timed out, was
+  // interrupted, or never started because shutdown closed the scope, leaves
+  // the reply for the shard's next owner. Concurrent writers on a shard share
+  // one refresh, which outlives an interrupted writer. Entities draining from
+  // a released shard still hold its lock.
+  const pendingLockChecks = new Map<ShardId, Fiber.Fiber<boolean>>()
+  const confirmShardLock = (runnerAddress: RunnerAddress, shardId: ShardId): Effect.Effect<boolean> =>
+    Effect.suspend(() => {
+      if (!MutableHashSet.has(acquiredShards, shardId) && !MutableHashSet.has(releasingShards, shardId)) {
+        return Effect.succeed(false)
+      }
+      const pending = pendingLockChecks.get(shardId)
+      const check = pending ? Effect.succeed(pending) : runnerStorage.refresh(runnerAddress, [shardId]).pipe(
+        Effect.map((acquired) => {
+          releaseLostShards([shardId], acquired)
+          return acquired.includes(shardId)
+        }),
+        Effect.timeoutOption(shardLockInterval),
+        Effect.map(Option.getOrElse(() => false)),
+        Effect.catch(() => Effect.succeed(false)),
+        Effect.forkIn(shardingScope, { startImmediately: true }),
+        Effect.map((fiber) => {
+          pendingLockChecks.set(shardId, fiber)
+          fiber.addObserver(() => pendingLockChecks.delete(shardId))
+          return fiber
+        })
+      )
+      return Effect.flatMap(
+        check,
+        (fiber) => Effect.map(Fiber.await(fiber), (exit) => exit._tag === "Success" && exit.value)
+      )
+    })
+
   const registerEntity: Sharding["Service"]["registerEntity"] = Effect.fnUntraced(
     function*(entity, build, options) {
       const runnerAddress = getRunnerAddress()
@@ -1818,7 +1955,9 @@ const make = Effect.gen(function*() {
         storage,
         runnerAddress,
         residency,
-        sharding
+        sharding,
+        abandonUndelivered,
+        isShardOwned: (shardId) => confirmShardLock(runnerAddress, shardId)
       }).pipe(
         Effect.provideContext(services.pipe(
           // Registration overrides construction fallbacks, except for runner-owned services.
@@ -1851,6 +1990,9 @@ const make = Effect.gen(function*() {
       yield* withStorageReadLock(Effect.sync(() => {
         entityRegistrationStartMillis ??= clock.currentTimeMillisUnsafe()
         entityManagers.set(entity.type, state)
+        // persisted messages read before this registration were reset, so
+        // read storage again instead of waiting for the next poll interval
+        storageReadLatch.openUnsafe()
         if (entityManagerLatches.has(entity.type)) {
           entityManagerLatches.get(entity.type)!.openUnsafe()
           entityManagerLatches.delete(entity.type)

package/src/cluster/ShardingConfig.ts

@@ -81,10 +81,17 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
   readonly shardLockRefreshInterval: Duration.Input
   /**
    * Shard lock expiration duration.
+   *
+   * With PostgreSQL advisory locks (PostgreSQL 14+), this also bounds how long
+   * a runner that stops refreshing, for example because its process is frozen,
+   * keeps its shards: the server ends its idle lock session after this
+   * duration, releasing the locks.
    */
   readonly shardLockExpiration: Duration.Input
   /**
    * Disable the use of advisory locks for shard locking.
+   *
+   * PGlite clients never use advisory locks.
    */
   readonly shardLockDisableAdvisory: boolean
   /**
@@ -97,6 +104,14 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
    * The default capacity of the mailbox for entities.
    */
   readonly entityMailboxCapacity: number | "unbounded"
+  /**
+   * Report the mailbox size of each resident entity with the
+   * `effect_cluster_entity_mailbox_size` gauge, unless overridden by the
+   * `mailboxMetrics` entity option.
+   *
+   * Defaults to `false`.
+   */
+  readonly entityMailboxMetrics: boolean
   /**
    * The maximum number of entities that can be resident on this runner at the
    * same time.
@@ -188,6 +203,7 @@ export const defaults: ShardingConfig["Service"] = {
   shardLockExpiration: Duration.seconds(35),
   shardLockDisableAdvisory: false,
   entityMailboxCapacity: 4096,
+  entityMailboxMetrics: false,
   maxResidentEntities: 10_000,
   unprocessedMessageBatchSize: 1024,
   entityMaxIdleTime: Duration.minutes(1),
@@ -307,6 +323,10 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.entityMailboxCapacity)
     // Config.withDescription("The default capacity of the mailbox for entities.")
   ),
+  entityMailboxMetrics: Config.Boolean("entityMailboxMetrics").pipe(
+    Config.withDefault(defaults.entityMailboxMetrics)
+    // Config.withDescription("Report the mailbox size of each resident entity.")
+  ),
   maxResidentEntities: Config.schema(
     Schema.Int.check(Schema.isGreaterThan(0)),
     "maxResidentEntities"

package/src/cluster/Snowflake.ts

@@ -257,18 +257,14 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
         machineId = newMachineId
       }),
     nextUnsafe() {
-      let now = Math.floor(clock.currentTimeMillisUnsafe())
+      const now = Math.floor(clock.currentTimeMillisUnsafe())
 
-      // account for clock drift, only allow time to move forward
-      if (now < sequenceAt) {
-        now = sequenceAt
-      } else if (now > sequenceAt) {
+      if (now > sequenceAt) {
         // reset sequence if we're in a new millisecond
         sequence = 0
         sequenceAt = now
       } else if (sequence >= 4096) {
-        // if we've hit the max sequence for this millisecond, go to the next
-        // millisecond
+        // sequence exhausted and the clock hasn't advanced, so borrow the next millisecond
         sequenceAt++
         sequence = 0
       }