effect 4.0.6 → 4.1.0

package/src/cluster/ClusterWorkflowEngine.ts

@@ -7,6 +7,7 @@
  * @stability unstable
  * @since 4.0.0
  */
+import * as Cause from "../Cause.ts"
 import * as Context from "../Context.ts"
 import * as DateTime from "../DateTime.ts"
 import * as Deferred from "../Deferred.ts"
@@ -539,9 +540,26 @@ export const make = Effect.gen(function*() {
                       !instance.abandoned &&
                       !(suspendOnFailure && exit._tag === "Failure")
                     ) {
-                      return parent && !parent.detached
+                      const resumeParent = parent && !parent.detached
                         ? ensureSuccess(sendResumeParent(parent, executionId))
                         : Effect.void
+                      if (exit._tag === "Success" || !Cause.hasInterruptsOnly(exit.cause)) {
+                        return resumeParent
+                      }
+                      // `interruptUnsafe` cuts the run off without a result and
+                      // its replay applies the interrupt. Retain the scope like a
+                      // suspended run, so `closeRuns` supersedes it and only the
+                      // final run's compensations observe the interrupt.
+                      return engine.deferredResult(InterruptSignal).pipe(
+                        Effect.flatMap((maybeExit) => {
+                          if (Option.isNone(maybeExit)) {
+                            return resumeParent
+                          }
+                          instance.suspended = true
+                          return Effect.void
+                        }),
+                        Effect.orDie
+                      )
                     }
                     return engine.deferredResult(InterruptSignal).pipe(
                       Effect.flatMap((maybeExit) => {
@@ -797,25 +815,28 @@ export const make = Effect.gen(function*() {
           if (exit) {
             return Option.some(exit)
           }
-          return yield* requestReply({
+          const requestId = yield* requestIdFor({
             workflow: instance.workflow,
             entityType: `Workflow/${instance.workflow._tag}`,
             executionId: instance.executionId,
             tag: "deferred",
             id: deferred.name
-          }).pipe(
-            Effect.map((reply) => {
-              if (Option.isNone(reply)) {
-                return Option.none<Exit.Exit<unknown, unknown>>()
-              }
-              const decoded = decodeDeferredWithExit(reply.value as any)
-              return Option.some(
-                decoded.exit._tag === "Success"
-                  ? decoded.exit.value
-                  : decoded.exit
-              )
-            })
-          )
+          })
+          if (Option.isNone(requestId)) {
+            return Option.none<Exit.Exit<unknown, unknown>>()
+          }
+          const reply = yield* replyForRequestId(requestId.value)
+          if (Option.isSome(reply)) {
+            return Option.some(deferredReplyExit(reply.value))
+          }
+          // The first completion saved under the deferred name wins, so a
+          // request the owner has not processed yet already holds the result.
+          const [request] = yield* storage.unprocessedMessagesById([requestId.value])
+          if (request?.envelope._tag === "Request") {
+            return Option.some(decodeDeferredPayload(request.envelope.payload).exit)
+          }
+          // It was processed after the first read, so its reply is stored now.
+          return Option.map(yield* replyForRequestId(requestId.value), deferredReplyExit)
         })),
         Effect.retry({
           while: (e) => e._tag === "PersistenceError",
@@ -896,10 +917,14 @@ const retryPolicy = Schedule.min([
   Schedule.spaced("1 minute")
 ])
 
+// A cluster abandonment means this owner can no longer complete the effect, so
+// retrying it would only wait on a retired run. Re-raise it as an annotated
+// interrupt, so the activation can unwind and its shard can be released.
 const ensureSuccess = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
   effect.pipe(
     Effect.sandbox,
-    Effect.retry(retryPolicy),
+    Effect.retry({ schedule: retryPolicy, while: (cause) => !ClusterAbandon.isCause(cause) }),
+    Effect.catchIf(ClusterAbandon.isCause, () => ClusterAbandon.interrupt),
     Effect.orDie
   )
 
@@ -959,6 +984,11 @@ const isFailedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>)
   reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended" && reply.exit.value.failed === true
 
 const decodeDeferredWithExit = Schema.decodeSync(Schema.toCodecJson(Reply.WithExit.schema(DeferredRpc)))
+const decodeDeferredPayload = Schema.decodeUnknownSync(Schema.toCodecJson(DeferredRpc.payloadSchema))
+const deferredReplyExit = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>): Exit.Exit<unknown, unknown> => {
+  const decoded = decodeDeferredWithExit(reply as any)
+  return decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit
+}
 
 const ResumeRpc = Rpc.make("resume", {
   // Older persisted resume envelopes have an empty payload and use the empty key.

package/src/cluster/MessageStorage.ts

@@ -133,8 +133,15 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 
   /**
    * Unregister the reply handler for the specified message.
+   *
+   * By default the waiters are resumed with `EntityNotAssignedToRunner`. With
+   * `interrupt: true` they are interrupted with a cluster-abandon annotation
+   * instead, for when the request will not be served by this runner.
    */
-  readonly unregisterReplyHandler: (requestId: Snowflake.Snowflake) => Effect.Effect<void>
+  readonly unregisterReplyHandler: (
+    requestId: Snowflake.Snowflake,
+    options?: { readonly interrupt?: boolean | undefined }
+  ) => Effect.Effect<void>
 
   /**
    * Unregister the reply handlers for the specified ShardId.
@@ -569,6 +576,14 @@ export const make = (
     }
     const replyHandlers = new Map<Snowflake.Snowflake, Array<ReplyHandler>>()
     const replyHandlersShard = new Map<string, Set<ReplyHandler>>()
+    const resumeUnregistered = (handler: ReplyHandler, options?: { readonly interrupt?: boolean | undefined }) =>
+      handler.resume(
+        options?.interrupt ? ClusterAbandon.interrupt : Effect.fail(
+          new EntityNotAssignedToRunner({
+            address: handler.message.envelope.address
+          })
+        )
+      )
     return MessageStorage.of({
       ...storage,
       // Nested transactions defer notifications to the outermost commit.
@@ -614,7 +629,7 @@ export const make = (
           })
         })
       },
-      unregisterReplyHandler: (requestId) =>
+      unregisterReplyHandler: (requestId, options) =>
         Effect.sync(() => {
           const handlers = replyHandlers.get(requestId)
           if (!handlers) return
@@ -622,11 +637,7 @@ export const make = (
           for (let i = 0; i < handlers.length; i++) {
             const handler = handlers[i]
             handler.shardSet.delete(handler)
-            handler.resume(Effect.fail(
-              new EntityNotAssignedToRunner({
-                address: handler.message.envelope.address
-              })
-            ))
+            resumeUnregistered(handler, options)
           }
         }),
       unregisterShardReplyHandlers: (shardId, options) =>
@@ -637,13 +648,7 @@ export const make = (
           replyHandlersShard.delete(id)
           shardSet.forEach((handler) => {
             replyHandlers.delete(handler.message.envelope.requestId)
-            handler.resume(
-              options?.interrupt ? ClusterAbandon.interrupt : Effect.fail(
-                new EntityNotAssignedToRunner({
-                  address: handler.message.envelope.address
-                })
-              )
-            )
+            resumeUnregistered(handler, options)
           })
         }),
       saveReply(reply) {

package/src/cluster/Sharding.ts

@@ -19,7 +19,7 @@ import type { Input } from "../Duration.ts"
 import * as Duration from "../Duration.ts"
 import * as Effect from "../Effect.ts"
 import * as Equal from "../Equal.ts"
-import type * as Exit from "../Exit.ts"
+import * as Exit from "../Exit.ts"
 import * as Fiber from "../Fiber.ts"
 import * as FiberMap from "../FiberMap.ts"
 import { constant, flow } from "../Function.ts"
@@ -281,6 +281,9 @@ const make = Effect.gen(function*() {
   // shards still held by this runner while their entities stop and their locks
   // are released
   const releasingShards = MutableHashSet.empty<ShardId>()
+  // runners that joined after the first assignment, kept out of the hash rings
+  // until the healthy set settles (see `runnerJoinDebounce`)
+  const pendingRunners = MutableHashSet.empty<Runner>()
   // The releasing gauge reports to the metric registry and attributes the
   // layer was built with.
   const updateReleasingShardsGauge = () =>
@@ -290,6 +293,24 @@ const make = Effect.gen(function*() {
     updateReleasingShardsGauge()
   }
   const activeShardsLatch = yield* Latch.make(false)
+
+  // Applies the result of a lock refresh: refreshed shards it did not return,
+  // and returned shards no longer assigned to this runner, move from
+  // `acquiredShards` to `releasingShards`. Shards acquired during the refresh
+  // are absent from its response, so only the refreshed ones are considered.
+  const releaseLostShards = (refreshed: ReadonlyArray<ShardId>, acquired: ReadonlyArray<ShardId>) => {
+    for (const shardId of refreshed) {
+      if (
+        MutableHashSet.has(acquiredShards, shardId) &&
+        (!acquired.includes(shardId) || !MutableHashSet.has(selfShards, shardId))
+      ) {
+        MutableHashSet.remove(acquiredShards, shardId)
+        MutableHashSet.add(releasingShards, shardId)
+      }
+    }
+    updateShardGauges()
+    if (MutableHashSet.size(releasingShards) > 0) activeShardsLatch.openUnsafe()
+  }
   // open once shutdown starts handing every shard off
   const handoffLatch = Latch.makeUnsafe(false)
   // open once the handoff has released every shard lock
@@ -593,6 +614,13 @@ const make = Effect.gen(function*() {
           continue
         }
 
+        // While joins are pending the assignments are about to change, so take
+        // no new shards; otherwise a shard released for a joining runner could
+        // be taken here and handed off again. Publishing reopens the latch.
+        if (MutableHashSet.size(pendingRunners) > 0) {
+          continue
+        }
+
         // if a shard has been assigned to this runner, we acquire it
         const unacquiredShards = MutableHashSet.empty<ShardId>()
         for (const shardId of selfShards) {
@@ -617,13 +645,16 @@ const make = Effect.gen(function*() {
           Effect.ignore,
           Effect.timeoutOption(shardLockInterval)
         )
-        // A forced release can start while `acquire` is in flight, so re-check
-        // it here as well as before acquiring.
+        // A forced release or a pending join can begin while `acquire` is in
+        // flight, so re-check both. Shards locked meanwhile are released
+        // without running anything on them.
         const forcedReleasePending = forcedShardReleasePending()
+        const joinsPending = MutableHashSet.size(pendingRunners) > 0
         for (const shardId of acquired) {
           if (
             !shardLocksHealthyLatch.isOpen() ||
             forcedReleasePending ||
+            joinsPending ||
             MutableHashSet.has(releasingShards, shardId) ||
             !MutableHashSet.has(selfShards, shardId)
           ) {
@@ -690,27 +721,12 @@ const make = Effect.gen(function*() {
       return Effect.logInfo("Shard lock storage has recovered")
     })
 
-    const refreshShardLocks = Effect.gen(function*() {
+    const refreshShardLocks = Effect.suspend(() => {
       const refreshed = [...acquiredShards, ...releasingShards]
-      const acquired = yield* runnerStorage.refresh(selfAddress, refreshed)
-      // Shards acquired during the request are absent from its response.
-      for (const shardId of refreshed) {
-        if (MutableHashSet.has(acquiredShards, shardId) && !acquired.includes(shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      for (let i = 0; i < acquired.length; i++) {
-        const shardId = acquired[i]
-        if (!MutableHashSet.has(selfShards, shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      updateShardGauges()
-      if (MutableHashSet.size(releasingShards) > 0) {
-        yield* activeShardsLatch.open
-      }
+      return Effect.map(
+        runnerStorage.refresh(selfAddress, refreshed),
+        (acquired) => releaseLostShards(refreshed, acquired)
+      )
     }).pipe(
       Effect.retry({
         times: 5,
@@ -1412,10 +1428,33 @@ const make = Effect.gen(function*() {
     ClusterMetrics.runnersHealthy.updateUnsafe(BigInt(1), Context.empty())
   }
 
+  // Kept outside the loop so a restart after a storage failure does not drop
+  // this state and publish pending joins early.
+  const hashRings = new Map<string, HashRing.HashRing<RunnerAddress>>()
+  let nextRunners = MutableHashMap.empty<Runner, boolean>()
+  const healthyRunners = MutableHashSet.empty<Runner>()
+  const joinDebounce = Duration.toMillis(Duration.fromInputUnsafe(config.runnerJoinDebounce))
+  // when the first pending join and the latest pending change were observed
+  let pendingSince = 0
+  let pendingChangedAt = 0
+  // the initial assignment never waits, even if this runner ends up with no shards
+  let assignmentsPublished = false
+  // stays set until the assignments are actually recomputed
+  let changed = false
+
+  const addToRings = (runner: Runner) => {
+    for (let i = 0; i < runner.groups.length; i++) {
+      const group = runner.groups[i]
+      let ring = hashRings.get(group)
+      if (!ring) {
+        ring = HashRing.make()
+        hashRings.set(group, ring)
+      }
+      HashRing.add(ring, runner.address, { weight: runner.weight })
+    }
+  }
+
   yield* Effect.gen(function*() {
-    const hashRings = new Map<string, HashRing.HashRing<RunnerAddress>>()
-    let nextRunners = MutableHashMap.empty<Runner, boolean>()
-    const healthyRunners = MutableHashSet.empty<Runner>()
     const withTimeout = Effect.timeout(Duration.seconds(5))
 
     while (true) {
@@ -1430,7 +1469,7 @@ const make = Effect.gen(function*() {
       }
 
       const runners = yield* withTimeout(runnerStorage.getRunners)
-      let changed = false
+      const now = clock.currentTimeMillisUnsafe()
       for (let i = 0; i < runners.length; i++) {
         const [runner, healthy] = runners[i]
         MutableHashMap.set(nextRunners, runner, healthy)
@@ -1442,31 +1481,55 @@ const make = Effect.gen(function*() {
           }
           continue
         }
-        changed = true
         MutableHashSet.add(healthyRunners, runner)
         MutableHashMap.remove(allRunners, runner)
-        for (let j = 0; j < runner.groups.length; j++) {
-          const group = runner.groups[j]
-          let ring = hashRings.get(group)
-          if (!ring) {
-            ring = HashRing.make()
-            hashRings.set(group, ring)
+        if (joinDebounce > 0 && assignmentsPublished) {
+          if (MutableHashSet.size(pendingRunners) === 0) {
+            pendingSince = now
           }
-          HashRing.add(ring, runner.address, { weight: runner.weight })
+          pendingChangedAt = now
+          MutableHashSet.add(pendingRunners, runner)
+          continue
         }
+        changed = true
+        addToRings(runner)
       }
 
       // Remove runners that are no longer present or healthy
       MutableHashMap.forEach(allRunners, (_, runner) => {
-        changed = true
         MutableHashMap.remove(allRunners, runner)
         MutableHashSet.remove(healthyRunners, runner)
         runFork(runnersService.onRunnerUnavailable(runner.address))
+        // A pending runner owns no shards, so its departure only restarts the
+        // quiet period instead of publishing the other joins mid-burst.
+        if (MutableHashSet.has(pendingRunners, runner)) {
+          MutableHashSet.remove(pendingRunners, runner)
+          pendingChangedAt = now
+          if (MutableHashSet.size(pendingRunners) === 0) {
+            // let the acquisition loop take the shards it was holding off on
+            activeShardsLatch.openUnsafe()
+          }
+          return
+        }
+        changed = true
         for (let i = 0; i < runner.groups.length; i++) {
           HashRing.remove(hashRings.get(runner.groups[i])!, runner.address)
         }
       })
 
+      // Publish pending joins once the healthy set has settled, the cap is
+      // reached, or the assignments are recomputed anyway.
+      if (
+        MutableHashSet.size(pendingRunners) > 0 &&
+        (changed || now - pendingChangedAt >= joinDebounce || now - pendingSince >= joinDebounce * 4)
+      ) {
+        changed = true
+        for (const runner of pendingRunners) {
+          addToRings(runner)
+        }
+        MutableHashSet.clear(pendingRunners)
+      }
+
       // swap allRunners and nextRunners
       const prevRunners = allRunners
       allRunners = nextRunners
@@ -1480,6 +1543,8 @@ const make = Effect.gen(function*() {
 
       // Recompute shard assignments if the set of healthy runners has changed.
       if (changed) {
+        changed = false
+        assignmentsPublished = true
         MutableHashSet.clear(selfShards)
         hashRings.forEach((ring, group) => {
           const newAssignments = HashRing.getShards(ring, config.shardsPerGroup)
@@ -1541,6 +1606,27 @@ const make = Effect.gen(function*() {
   }
   const clientRequests = new Map<Snowflake.Snowflake, ClientRequestEntry>()
 
+  // During shutdown, abandon local callers parked on persisted requests that a
+  // retiring entity never started. They are served by the next owner. Callers
+  // on a runner that keeps running are left waiting, so they stay durable.
+  const abandonUndelivered = (
+    address: EntityAddress,
+    isDelivered: (requestId: Snowflake.Snowflake) => boolean
+  ): Effect.Effect<void> =>
+    Effect.suspend(() => {
+      if (!MutableRef.get(isShutdown)) return Effect.void
+      const abandon: Array<Effect.Effect<void>> = []
+      for (const [requestId, entry] of clientRequests) {
+        if (
+          !Context.get(entry.message.annotations, Persisted) ||
+          !Equal.equals(entry.message.envelope.address, address) ||
+          isDelivered(requestId)
+        ) continue
+        abandon.push(storage.unregisterReplyHandler(requestId, { interrupt: true }))
+      }
+      return Effect.all(abandon, { discard: true })
+    })
+
   const clients: ResourceMap<
     Entity<any, any>,
     (entityId: string) => RpcClient.RpcClient<
@@ -1595,18 +1681,23 @@ const make = Effect.gen(function*() {
                   envelope as any
                 )
               })
-              if (!options.discard) {
-                const entry: ClientRequestEntry = {
-                  rpc: rpc as any,
-                  context: fiber.context,
-                  message
-                }
-                clientRequests.set(id, entry)
-                respond = makeClientRespond(entry, client.write)
-              } else {
+              if (options.message.discard === true) {
                 respond = clientRespondDiscard
+                // a discarded call is acknowledged once its message is sent
+                const requestId = options.message.id
+                return Effect.flatMap(
+                  sendOutgoing(message, true),
+                  () => client.write({ _tag: "Exit", clientId: 0, requestId, exit: Exit.void as any })
+                )
               }
-              return sendOutgoing(message, options.discard)
+              const entry: ClientRequestEntry = {
+                rpc: rpc as any,
+                context: fiber.context,
+                message
+              }
+              clientRequests.set(id, entry)
+              respond = makeClientRespond(entry, client.write)
+              return sendOutgoing(message, false)
             }
             case "Ack": {
               const requestId = Snowflake.Snowflake(options.message.requestId)
@@ -1878,6 +1969,42 @@ const make = Effect.gen(function*() {
   const reaper = yield* EntityReaper
   const entityManagerLatches = new Map<string, Latch.Latch>()
 
+  // This runner's view of its shards lags lock storage by up to one refresh,
+  // or longer while the process is paused. So the final reply of a persisted
+  // request is stored only once a refresh of its shard's lock confirms it.
+  // Anything short of that, including a refresh that failed, timed out, was
+  // interrupted, or never started because shutdown closed the scope, leaves
+  // the reply for the shard's next owner. Concurrent writers on a shard share
+  // one refresh, which outlives an interrupted writer. Entities draining from
+  // a released shard still hold its lock.
+  const pendingLockChecks = new Map<ShardId, Fiber.Fiber<boolean>>()
+  const confirmShardLock = (runnerAddress: RunnerAddress, shardId: ShardId): Effect.Effect<boolean> =>
+    Effect.suspend(() => {
+      if (!MutableHashSet.has(acquiredShards, shardId) && !MutableHashSet.has(releasingShards, shardId)) {
+        return Effect.succeed(false)
+      }
+      const pending = pendingLockChecks.get(shardId)
+      const check = pending ? Effect.succeed(pending) : runnerStorage.refresh(runnerAddress, [shardId]).pipe(
+        Effect.map((acquired) => {
+          releaseLostShards([shardId], acquired)
+          return acquired.includes(shardId)
+        }),
+        Effect.timeoutOption(shardLockInterval),
+        Effect.map(Option.getOrElse(() => false)),
+        Effect.catch(() => Effect.succeed(false)),
+        Effect.forkIn(shardingScope, { startImmediately: true }),
+        Effect.map((fiber) => {
+          pendingLockChecks.set(shardId, fiber)
+          fiber.addObserver(() => pendingLockChecks.delete(shardId))
+          return fiber
+        })
+      )
+      return Effect.flatMap(
+        check,
+        (fiber) => Effect.map(Fiber.await(fiber), (exit) => exit._tag === "Success" && exit.value)
+      )
+    })
+
   const registerEntity: Sharding["Service"]["registerEntity"] = Effect.fnUntraced(
     function*(entity, build, options) {
       const runnerAddress = getRunnerAddress()
@@ -1895,7 +2022,9 @@ const make = Effect.gen(function*() {
         storage,
         runnerAddress,
         residency,
-        sharding
+        sharding,
+        abandonUndelivered,
+        isShardOwned: (shardId) => confirmShardLock(runnerAddress, shardId)
       }).pipe(
         Effect.provideContext(services.pipe(
           // Registration overrides construction fallbacks, except for runner-owned services.

package/src/cluster/ShardingConfig.ts

@@ -81,6 +81,13 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
   readonly shardLockRefreshInterval: Duration.Input
   /**
    * Shard lock expiration duration.
+   *
+   * With PostgreSQL (14+) or MySQL advisory locks, this also bounds how long a
+   * runner that stops refreshing, for example because its process is frozen,
+   * keeps its shards: the server ends its idle lock session after this
+   * duration, releasing the locks. On MySQL it is rounded up to whole
+   * seconds. Vitess ignores the MySQL session timeout, so behind Vitess the
+   * locks stay held until the runner's connection closes.
    */
   readonly shardLockExpiration: Duration.Input
   /**
@@ -160,6 +167,37 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
    * assignments.
    */
   readonly refreshAssignmentsInterval: Duration.Input
+  /**
+   * How long the healthy runners must stay unchanged before runners that
+   * joined are added to shard assignments. A burst of joins, such as a
+   * scale-up or a deployment with a large surge, is then published as one
+   * assignment and each shard is handed off once instead of once per join.
+   * In a rolling update that replaces one runner at a time, each departure is
+   * still published immediately, so shards still move at every step.
+   *
+   * Only joins wait. The initial assignment, departures and unhealthy runners
+   * are published immediately, together with any pending joins. A pending
+   * runner that leaves has no shards to move, so it only restarts the quiet
+   * period.
+   *
+   * While joins are pending, shards keep running where they are, the new
+   * runners stay idle and no runner takes shards it does not already hold.
+   * Runners publish on their own polls, so a shard released by one runner can
+   * stay unlocked until its new owner publishes, usually within two
+   * `refreshAssignmentsInterval`s. A runner that does not observe the joins,
+   * for example because it cannot read membership, can still take shards from
+   * its outdated assignment and hand them off again, as with `0`.
+   *
+   * Pending joins are published at most four times this duration after the
+   * first one, even if runners keep joining. The quiet period is checked once
+   * per `refreshAssignmentsInterval`, so the worst case delay is four times
+   * this duration plus one interval, and values shorter than the interval
+   * behave like a single quiet poll.
+   *
+   * Defaults to `0`, which publishes every change immediately. Leave it at
+   * `0` for scale-ups driven by load, where idle new runners are costly.
+   */
+  readonly runnerJoinDebounce: Duration.Input
   /**
    * The interval to retry a send if EntityNotAssignedToRunner is returned.
    */
@@ -208,6 +246,7 @@ export const defaults: ShardingConfig["Service"] = {
   entityReplyPollInterval: Duration.millis(200),
   sendRetryInterval: Duration.millis(100),
   refreshAssignmentsInterval: Duration.seconds(3),
+  runnerJoinDebounce: Duration.zero,
   runnerHealthCheckInterval: Duration.minutes(1),
   simulateRemoteSerialization: true
 }
@@ -366,6 +405,10 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.refreshAssignmentsInterval)
     // Config.withDescription("The interval at which to refresh shard assignments.")
   ),
+  runnerJoinDebounce: Config.Duration("runnerJoinDebounce").pipe(
+    Config.withDefault(defaults.runnerJoinDebounce)
+    // Config.withDescription("How long the healthy runners must stay unchanged before joining runners are assigned shards.")
+  ),
   runnerHealthCheckInterval: Config.Duration("runnerHealthCheckInterval").pipe(
     Config.withDefault(defaults.runnerHealthCheckInterval)
     // Config.withDescription("The interval at which to check for unhealthy runners and report them.")

package/src/cluster/SingleRunner.ts

@@ -13,7 +13,7 @@
  */
 import * as Layer from "effect/Layer"
 import type { ConfigError } from "../Config.ts"
-import type * as Crypto from "../Crypto.ts"
+import type * as Digest from "../crypto/Digest.ts"
 import type * as SqlClient from "../sql/SqlClient.ts"
 import type * as MessageStorage from "./MessageStorage.ts"
 import * as RunnerHealth from "./RunnerHealth.ts"
@@ -44,7 +44,7 @@ import * as SqlRunnerStorage from "./SqlRunnerStorage.ts"
  * **Gotchas**
  *
  * - Even when `runnerStorage` is `"memory"`, message storage remains
- *   SQL-backed, so callers must still provide `SqlClient` and `Crypto.Crypto`
+ *   SQL-backed, so callers must still provide `SqlClient` and `Digest.Digest`
  *   (used to hash over-length message deduplication keys).
  * - Runner communication and runner health are no-op services, so this layer is
  *   for single-process use rather than multi-runner coordination.
@@ -66,7 +66,7 @@ export const layer = (options?: {
   | Runners.Runners
   | MessageStorage.MessageStorage,
   ConfigError,
-  SqlClient.SqlClient | Crypto.Crypto
+  SqlClient.SqlClient | Digest.Digest
 > =>
   Sharding.layer.pipe(
     Layer.provideMerge(Runners.layerNoop),

package/src/cluster/Snowflake.ts

@@ -257,18 +257,14 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
         machineId = newMachineId
       }),
     nextUnsafe() {
-      let now = Math.floor(clock.currentTimeMillisUnsafe())
+      const now = Math.floor(clock.currentTimeMillisUnsafe())
 
-      // account for clock drift, only allow time to move forward
-      if (now < sequenceAt) {
-        now = sequenceAt
-      } else if (now > sequenceAt) {
+      if (now > sequenceAt) {
         // reset sequence if we're in a new millisecond
         sequence = 0
         sequenceAt = now
       } else if (sequence >= 4096) {
-        // if we've hit the max sequence for this millisecond, go to the next
-        // millisecond
+        // sequence exhausted and the clock hasn't advanced, so borrow the next millisecond
         sequenceAt++
         sequence = 0
       }