effect 4.0.0-rc.112 → 4.0.0-rc.113

package/src/unstable/cluster/MessageStorage.ts

@@ -28,6 +28,7 @@ import { EntityNotAssignedToRunner, MalformedMessage, type PersistenceError } fr
 import * as DeliverAt from "./DeliverAt.ts"
 import type { EntityAddress } from "./EntityAddress.ts"
 import * as Envelope from "./Envelope.ts"
+import * as ClusterAbandon from "./internal/clusterAbandon.ts"
 import * as Message from "./Message.ts"
 import * as Reply from "./Reply.ts"
 import * as ShardId from "./ShardId.ts"
@@ -121,8 +122,16 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 
   /**
    * Unregister the reply handlers for the specified ShardId.
+   *
+   * By default the waiters are resumed with `EntityNotAssignedToRunner`, so
+   * they can re-route the wait. With `interrupt: true` the waiters are
+   * interrupted with a cluster-abandon annotation instead, for when the runner
+   * is shutting down and the reply can never be observed here.
    */
-  readonly unregisterShardReplyHandlers: (shardId: ShardId.ShardId) => Effect.Effect<void>
+  readonly unregisterShardReplyHandlers: (
+    shardId: ShardId.ShardId,
+    options?: { readonly interrupt?: boolean | undefined }
+  ) => Effect.Effect<void>
 
   /**
    * Retrieves the unprocessed messages for the specified shards.
@@ -478,10 +487,23 @@ export type EncodedRepliesOptions<A> = {
  * @since 4.0.0
  */
 export const make = (
-  storage: Omit<
-    MessageStorage["Service"],
-    "registerReplyHandler" | "unregisterReplyHandler" | "unregisterShardReplyHandlers"
-  >
+  storage:
+    & Omit<
+      MessageStorage["Service"],
+      "saveReply" | "registerReplyHandler" | "unregisterReplyHandler" | "unregisterShardReplyHandlers"
+    >
+    & {
+      /**
+       * Save the provided `Reply`, returning the reply as it was persisted.
+       *
+       * Implementations that persist a different representation, such as a
+       * defect reply when the original cannot be encoded, return that fallback
+       * so reply handlers deliver exactly what storage recorded.
+       */
+      readonly saveReply: <R extends Rpc.Any>(
+        reply: Reply.ReplyWithContext<R>
+      ) => Effect.Effect<Reply.ReplyWithContext<R>, PersistenceError | MalformedMessage>
+    }
 ): Effect.Effect<MessageStorage["Service"]> =>
   Effect.sync(() => {
     type ReplyHandler = {
@@ -538,7 +560,7 @@ export const make = (
             ))
           }
         }),
-      unregisterShardReplyHandlers: (shardId) =>
+      unregisterShardReplyHandlers: (shardId, options) =>
         Effect.sync(() => {
           const id = shardId.toString()
           const shardSet = replyHandlersShard.get(id)
@@ -546,20 +568,22 @@ export const make = (
           replyHandlersShard.delete(id)
           shardSet.forEach((handler) => {
             replyHandlers.delete(handler.message.envelope.requestId)
-            handler.resume(Effect.fail(
-              new EntityNotAssignedToRunner({
-                address: handler.message.envelope.address
-              })
-            ))
+            handler.resume(
+              options?.interrupt ? ClusterAbandon.interrupt : Effect.fail(
+                new EntityNotAssignedToRunner({
+                  address: handler.message.envelope.address
+                })
+              )
+            )
           })
         }),
       saveReply(reply) {
         const requestId = reply.reply.requestId
-        return Effect.flatMap(storage.saveReply(reply), () => {
+        return Effect.flatMap(storage.saveReply(reply), (persisted) => {
           const handlers = replyHandlers.get(requestId)
           if (!handlers) {
             return Effect.void
-          } else if (reply.reply._tag === "WithExit") {
+          } else if (persisted.reply._tag === "WithExit") {
             replyHandlers.delete(requestId)
             for (let i = 0; i < handlers.length; i++) {
               const handler = handlers[i]
@@ -568,8 +592,8 @@ export const make = (
             }
           }
           return handlers.length === 1
-            ? handlers[0].respond(reply)
-            : Effect.forEach(handlers, (handler) => handler.respond(reply))
+            ? handlers[0].respond(persisted)
+            : Effect.forEach(handlers, (handler) => handler.respond(persisted))
         })
       }
     })
@@ -634,7 +658,22 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
         ),
         Effect.asVoid
       ),
-    saveReply: (reply) => Effect.flatMap(Reply.serializeOrDefect(reply, codecForJson), encoded.saveReply),
+    saveReply: (reply) =>
+      Reply.serialize(reply, codecForJson).pipe(
+        Effect.map((encodedReply) => ({ encodedReply, persisted: reply })),
+        Effect.catchTag("MalformedMessage", (error) => {
+          const persisted = Reply.ReplyWithContext.fromDefect({
+            id: reply.reply.id,
+            requestId: reply.reply.requestId,
+            defect: error
+          })
+          return Effect.map(
+            Effect.orDie(Reply.serialize(persisted, codecForJson)),
+            (encodedReply) => ({ encodedReply, persisted })
+          )
+        }),
+        Effect.flatMap(({ encodedReply, persisted }) => Effect.as(encoded.saveReply(encodedReply), persisted))
+      ),
     clearReplies: encoded.clearReplies,
     repliesFor: Effect.fnUntraced(function*(messages) {
       const requestIds = Arr.empty<string>()
@@ -658,7 +697,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
       return encoded.requestIdForPrimaryKey(primaryKey)
     },
     unprocessedMessages(shardIds, options) {
-      const storage = this as MessageStorage["Service"]
+      const storage = this as unknown as MessageStorage["Service"]
       const shards = Array.from(shardIds, (id) => id.toString())
       if (!Arr.isArrayNonEmpty(shards)) return Effect.succeed([])
       if (options?.addresses !== undefined && options.addresses.length === 0) return Effect.succeed([])
@@ -668,7 +707,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
       )
     },
     unprocessedMessagesById(messageIds) {
-      const storage = this as MessageStorage["Service"]
+      const storage = this as unknown as MessageStorage["Service"]
       const ids = Array.from(messageIds)
       if (!Arr.isArrayNonEmpty(ids)) return Effect.succeed([])
       return Effect.flatMap(
@@ -799,7 +838,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
 export const noop: MessageStorage["Service"] = Effect.runSync(make({
   saveRequest: () => Effect.succeed(SaveResult.Success()),
   saveEnvelope: () => Effect.void,
-  saveReply: () => Effect.void,
+  saveReply: (reply) => Effect.succeed(reply),
   clearReplies: () => Effect.void,
   repliesFor: () => Effect.succeed([]),
   repliesForUnfiltered: () => Effect.succeed([]),
@@ -1066,12 +1105,14 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
             const envelope = journal[i]
             const sameAddress = address.entityType === envelope.address.entityType &&
               address.entityId === envelope.address.entityId
-            if (!sameAddress || envelope._tag !== "Request") {
+            if (!sameAddress) {
               continue
             }
             unprocessed.delete(envelope)
             lastRead.delete(envelope)
-            requests.delete(envelope.requestId)
+            if (envelope._tag === "Request") {
+              requests.delete(envelope.requestId)
+            }
             journal.splice(i, 1)
           }
         }),

package/src/unstable/cluster/Reply.ts

@@ -407,8 +407,8 @@ export const Reply = <R extends Rpc.Any>(
 ): Schema.Codec<
   WithExit<R> | Chunk<R>,
   Encoded,
-  Rpc.ServicesServer<R>,
-  Rpc.ServicesClient<R>
+  Rpc.ServicesClient<R>,
+  Rpc.ServicesServer<R>
 > => {
   let schemaCache = schemaCaches.get(codecFor)
   if (schemaCache === undefined) {

package/src/unstable/cluster/Sharding.ts

@@ -45,16 +45,17 @@ import { AlreadyProcessingMessage, EntityNotAssignedToRunner } from "./ClusterEr
 import * as ClusterMetrics from "./ClusterMetrics.ts"
 import { Persisted } from "./ClusterSchema.ts"
 import * as ClusterSchema from "./ClusterSchema.ts"
-import type { CurrentAddress, CurrentRunnerAddress, Entity, HandlersFrom } from "./Entity.ts"
+import { CurrentAddress, type CurrentRunnerAddress, type Entity, type HandlersFrom } from "./Entity.ts"
 import type { EntityAddress } from "./EntityAddress.ts"
 import { make as makeEntityAddress } from "./EntityAddress.ts"
 import type { EntityId } from "./EntityId.ts"
 import { make as makeEntityId } from "./EntityId.ts"
 import * as Envelope from "./Envelope.ts"
+import * as ClusterAbandon from "./internal/clusterAbandon.ts"
 import * as EntityManager from "./internal/entityManager.ts"
 import { EntityReaper } from "./internal/entityReaper.ts"
 import { hashString } from "./internal/hash.ts"
-import { internalInterruptors } from "./internal/interruptors.ts"
+import * as ActiveTeardown from "./internal/interruptors.ts"
 import { ResourceMap } from "./internal/resourceMap.ts"
 import { effectiveInterval } from "./internal/shardLock.ts"
 import * as Message from "./Message.ts"
@@ -317,7 +318,13 @@ const make = Effect.gen(function*() {
       for (const shardId of shardIds) {
         MutableHashSet.remove(releasingShards, shardId)
         MutableHashSet.remove(forceReleasingShards, shardId)
-        yield* storage.unregisterShardReplyHandlers(shardId)
+        // During shutdown, parked waiters are interrupted: this runner will
+        // never observe the reply. During a rebalance the waiters are resumed
+        // with `EntityNotAssignedToRunner`, and the client falls back to
+        // waiting for the reply via storage while the entity moves.
+        yield* storage.unregisterShardReplyHandlers(shardId, {
+          interrupt: MutableRef.get(isShutdown)
+        })
       }
     })
     const retryShardRelease =
@@ -552,7 +559,7 @@ const make = Effect.gen(function*() {
     const probeShardLocks = runnerStorage.refresh(selfAddress, []).pipe(
       Effect.timeout(shardLockInterval),
       Effect.andThen(markShardLocksHealthy),
-      Effect.catchCause(() => Effect.void)
+      Effect.catchCause((cause) => Effect.logWarning("Shard lock storage is still unhealthy", cause))
     )
 
     // Refresh shard locks at the lease-safe interval, or probe storage while
@@ -1113,7 +1120,13 @@ const make = Effect.gen(function*() {
       return Effect.catchTag(persist, "MalformedMessage", Effect.die).pipe(
         Effect.andThen(
           shouldFail
-            ? Effect.fail(error)
+            // The message is durable, so a transient routing state is never an
+            // error for the caller. The runner is tearing down and can no
+            // longer observe the reply, so interrupt the caller instead: the
+            // request will be served under the next owner.
+            ? message._tag === "OutgoingRequest"
+              ? ClusterAbandon.interrupt
+              : Effect.fail(error)
             : Effect.logWarning("Persisting outgoing message abandoned during shutdown", message.envelope.address)
         )
       )
@@ -1328,6 +1341,13 @@ const make = Effect.gen(function*() {
     never
   > = yield* ResourceMap.make(
     Effect.fnUntraced(function*(entity: Entity<string, any>) {
+      const clientScope = yield* Effect.scope
+      yield* Scope.addFinalizer(
+        clientScope,
+        Effect.sync(() => {
+          ActiveTeardown.releaseEntityType(entity.type)
+        })
+      )
       const client = yield* RpcClient.makeNoSerialization(entity.protocol, {
         spanPrefix: `${entity.type}.client`,
         disableTracing: !Context.get(entity.protocol.annotations, ClusterSchema.ClientTracingEnabled),
@@ -1406,8 +1426,9 @@ const make = Effect.gen(function*() {
               }
               // for durable messages, we ignore interrupts on shutdown or as a
               // result of a shard being resassigned
+              const caller = Context.getOption(entry.context, CurrentAddress).valueOrUndefined
               const isTransientInterrupt = MutableRef.get(isShutdown) ||
-                options.message.interruptors.some((id) => internalInterruptors.has(id))
+                (caller !== undefined && ActiveTeardown.isActive(caller))
               if (isTransientInterrupt && Context.get(entry.message.annotations, Persisted)) {
                 return Effect.void
               }
@@ -1430,10 +1451,9 @@ const make = Effect.gen(function*() {
       })
 
       yield* Scope.addFinalizer(
-        yield* Effect.scope,
-        Effect.withFiber((fiber) => {
-          internalInterruptors.add(fiber.id)
-          return Effect.void
+        clientScope,
+        Effect.sync(() => {
+          ActiveTeardown.acquireEntityType(entity.type)
         })
       )
 
@@ -1567,8 +1587,10 @@ const make = Effect.gen(function*() {
         const shouldBeRunning = MutableHashSet.has(acquiredShards, shardId)
         if (running && !shouldBeRunning) {
           yield* Effect.logDebug("Stopping singleton", address)
-          internalInterruptors.add(yield* Effect.fiberId)
-          yield* FiberMap.remove(singletonFibers, address)
+          yield* ActiveTeardown.aroundShard(
+            address.shardId,
+            FiberMap.remove(singletonFibers, address)
+          )
         } else if (!running && shouldBeRunning) {
           yield* Effect.logDebug("Starting singleton", address)
           yield* FiberMap.run(singletonFibers, address, run)
@@ -1591,6 +1613,7 @@ const make = Effect.gen(function*() {
       const runnerAddress = getRunnerAddress()
       if (!runnerAddress || entityManagers.has(entity.type)) return
       const scope = yield* Effect.scope
+      const registrationContext = yield* Effect.context<never>()
       yield* Scope.addFinalizer(
         scope,
         Effect.sync(() => {
@@ -1605,6 +1628,10 @@ const make = Effect.gen(function*() {
         sharding
       }).pipe(
         Effect.provideContext(services.pipe(
+          // Registration overrides construction fallbacks, except for runner-owned services.
+          Context.merge(registrationContext),
+          Context.add(ShardingConfig, config),
+          Context.add(Clock, clock),
           Context.add(EntityReaper, reaper),
           Context.add(Scope.Scope, scope),
           Context.add(Snowflake.Generator, snowflakeGen)
@@ -1617,12 +1644,12 @@ const make = Effect.gen(function*() {
       }
       yield* Scope.addFinalizer(
         scope,
-        Effect.withFiber((fiber) => {
+        Effect.suspend(() => {
           state.status = "closing"
-          internalInterruptors.add(fiber.id)
-          // if preemptive shutdown is enabled, we start shutting down Sharding
-          // too
-          return config.preemptiveShutdown ? shutdown() : Effect.void
+          return ActiveTeardown.aroundEntityType(
+            entity.type,
+            config.preemptiveShutdown ? shutdown() : Effect.void
+          )
         })
       )
 
@@ -1704,9 +1731,7 @@ const make = Effect.gen(function*() {
       )
     }
 
-    internalInterruptors.add(yield* Effect.fiberId)
     if (isShutdown.current) return
-
     MutableRef.set(isShutdown, true)
     if (selfRunner) {
       yield* Effect.ignore(runnerStorage.unregister(selfRunner.address))

package/src/unstable/cluster/ShardingConfig.ts

@@ -248,56 +248,56 @@ export const layerDefaults: Layer.Layer<ShardingConfig> = layer()
  */
 export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
   runnerAddress: Config.all({
-    host: Config.string("host").pipe(
+    host: Config.String("host").pipe(
       Config.withDefault(defaultRunnerAddress.host)
       // Config.withDescription("The hostname or IP address of the runner.")
     ),
-    port: Config.int("port").pipe(
+    port: Config.Int("port").pipe(
       Config.withDefault(defaultRunnerAddress.port)
       // Config.withDescription("The port used for inter-runner communication.")
     )
   }).pipe(Config.map((options) => RunnerAddress.make(options)), Config.option),
   runnerListenAddress: Config.all({
-    host: Config.string("listenHost"),
+    host: Config.String("listenHost"),
     // Config.withDescription("The host to listen on.")
-    port: Config.int("listenPort").pipe(
+    port: Config.Int("listenPort").pipe(
       Config.withDefault(defaultRunnerAddress.port)
       // Config.withDescription("The port to listen on.")
     )
   }).pipe(Config.map((options) => RunnerAddress.make(options)), Config.option),
-  runnerShardWeight: Config.int("runnerShardWeight").pipe(
+  runnerShardWeight: Config.Int("runnerShardWeight").pipe(
     Config.withDefault(defaults.runnerShardWeight)
     // Config.withDescription("A number that determines how many shards this runner will be assigned relative to other runners.")
   ),
-  availableShardGroups: Config.schema(Config.Array(Schema.String), "availableShardGroups").pipe(
+  availableShardGroups: Config.Array(Schema.String, "availableShardGroups").pipe(
     Config.withDefault(["default"])
     // Config.withDescription("The shard groups available across all runners.")
   ),
-  assignedShardGroups: Config.schema(Config.Array(Schema.String), "shardGroups").pipe(
+  assignedShardGroups: Config.Array(Schema.String, "shardGroups").pipe(
     Config.withDefault(["default"])
     // Config.withDescription("The shard groups that are assigned to this runner.")
   ),
-  shardsPerGroup: Config.int("shardsPerGroup").pipe(
+  shardsPerGroup: Config.Int("shardsPerGroup").pipe(
     Config.withDefault(defaults.shardsPerGroup)
     // Config.withDescription("The number of shards to allocate per shard group.")
   ),
-  shardLockRefreshInterval: Config.duration("shardLockRefreshInterval").pipe(
+  shardLockRefreshInterval: Config.Duration("shardLockRefreshInterval").pipe(
     Config.withDefault(defaults.shardLockRefreshInterval)
     // Config.withDescription("Shard lock refresh interval.")
   ),
-  shardLockExpiration: Config.duration("shardLockExpiration").pipe(
+  shardLockExpiration: Config.Duration("shardLockExpiration").pipe(
     Config.withDefault(defaults.shardLockExpiration)
     // Config.withDescription("Shard lock expiration duration.")
   ),
-  shardLockDisableAdvisory: Config.boolean("shardLockDisableAdvisory").pipe(
+  shardLockDisableAdvisory: Config.Boolean("shardLockDisableAdvisory").pipe(
     Config.withDefault(defaults.shardLockDisableAdvisory)
     // Config.withDescription("Disable the use of advisory locks for shard locking.")
   ),
-  preemptiveShutdown: Config.boolean("preemptiveShutdown").pipe(
+  preemptiveShutdown: Config.Boolean("preemptiveShutdown").pipe(
     Config.withDefault(defaults.preemptiveShutdown)
     // Config.withDescription("Start shutting down as soon as an Entity has started shutting down.")
   ),
-  entityMailboxCapacity: Config.int("entityMailboxCapacity").pipe(
+  entityMailboxCapacity: Config.Int("entityMailboxCapacity").pipe(
     Config.withDefault(defaults.entityMailboxCapacity)
     // Config.withDescription("The default capacity of the mailbox for entities.")
   ),
@@ -315,42 +315,42 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.unprocessedMessageBatchSize)
     // Config.withDescription("The maximum number of unprocessed messages read from storage in a single poll.")
   ),
-  entityMaxIdleTime: Config.duration("entityMaxIdleTime").pipe(
+  entityMaxIdleTime: Config.Duration("entityMaxIdleTime").pipe(
     Config.withDefault(defaults.entityMaxIdleTime)
     // Config.withDescription(
     //   "The maximum duration of inactivity (i.e. without receiving a message) after which an entity will be interrupted."
     // )
   ),
-  entityRegistrationTimeout: Config.duration("entityRegistrationTimeout").pipe(
+  entityRegistrationTimeout: Config.Duration("entityRegistrationTimeout").pipe(
     Config.withDefault(defaults.entityRegistrationTimeout)
     // Config.withDescription("If an entity does not register itself within this time after a message is sent to it, the message will be marked as failed.")
   ),
-  entityTerminationTimeout: Config.duration("entityTerminationTimeout").pipe(
+  entityTerminationTimeout: Config.Duration("entityTerminationTimeout").pipe(
     Config.withDefault(defaults.entityTerminationTimeout)
     // Config.withDescription("The maximum duration of time to wait for an entity to terminate.")
   ),
-  entityMessagePollInterval: Config.duration("entityMessagePollInterval").pipe(
+  entityMessagePollInterval: Config.Duration("entityMessagePollInterval").pipe(
     Config.withDefault(defaults.entityMessagePollInterval)
     // Config.withDescription("The interval at which to poll for unprocessed messages from storage.")
   ),
-  entityReplyPollInterval: Config.duration("entityReplyPollInterval").pipe(
+  entityReplyPollInterval: Config.Duration("entityReplyPollInterval").pipe(
     Config.withDefault(defaults.entityReplyPollInterval)
     // Config.withDescription("The interval at which to poll for client replies from storage.")
   ),
-  sendRetryInterval: Config.duration("sendRetryInterval").pipe(
+  sendRetryInterval: Config.Duration("sendRetryInterval").pipe(
     Config.withDefault(defaults.sendRetryInterval)
     // Config.withDescription("The interval to retry a send if EntityNotManagedByRunner is returned.")
   ),
-  refreshAssignmentsInterval: Config.duration("refreshAssignmentsInterval").pipe(
+  refreshAssignmentsInterval: Config.Duration("refreshAssignmentsInterval").pipe(
     Config.withDefault(defaults.refreshAssignmentsInterval)
     // Config.withDescription("The interval at which to refresh shard assignments.")
   ),
-  runnerHealthCheckInterval: Config.duration("runnerHealthCheckInterval").pipe(
+  runnerHealthCheckInterval: Config.Duration("runnerHealthCheckInterval").pipe(
     Config.withDefault(defaults.runnerHealthCheckInterval)
     // Config.withDescription("The interval at which to check for unhealthy runners and report them.")
   ),
-  // unhealthyRunnerReportInterval: Config.duration("unhealthyRunnerReportInterval").pipe(
-  simulateRemoteSerialization: Config.boolean("simulateRemoteSerialization").pipe(
+  // unhealthyRunnerReportInterval: Config.Duration("unhealthyRunnerReportInterval").pipe(
+  simulateRemoteSerialization: Config.Boolean("simulateRemoteSerialization").pipe(
     Config.withDefault(defaults.simulateRemoteSerialization)
     // Config.withDescription("Simulate serialization and deserialization to remote runners for local entities.")
   )

package/src/unstable/cluster/SocketRunner.ts

@@ -11,6 +11,7 @@
  */
 import * as Effect from "../../Effect.ts"
 import * as Layer from "../../Layer.ts"
+import * as NetAddress from "../net/NetAddress.ts"
 import type * as RpcSerialization from "../rpc/RpcSerialization.ts"
 import * as RpcServer from "../rpc/RpcServer.ts"
 import { SocketServer } from "../socket/SocketServer.ts"
@@ -25,9 +26,9 @@ import type { ShardingConfig } from "./ShardingConfig.ts"
 const withLogAddress = <A, E, R>(layer: Layer.Layer<A, E, R>): Layer.Layer<A, E, R | SocketServer> =>
   Layer.effectDiscard(Effect.gen(function*() {
     const server = yield* SocketServer
-    const address = server.address._tag === "UnixAddress"
+    const address = server.address._tag === "UnixPathAddress"
       ? server.address.path
-      : `${server.address.hostname}:${server.address.port}`
+      : NetAddress.formatInet(server.address)
     yield* Effect.annotateLogs(Effect.logInfo(`Listening on: ${address}`), {
       package: "@effect/cluster",
       service: "Runner"
@@ -46,8 +47,8 @@ const withLogAddress = <A, E, R>(layer: Layer.Layer<A, E, R>): Layer.Layer<A, E,
  *
  * **Details**
  *
- * It logs the bound `SocketServer.address` when the layer starts, formatting TCP
- * addresses as `hostname:port` and Unix socket addresses as their filesystem
+ * It logs the bound `SocketServer.address` when the layer starts, formatting internet
+ * addresses as numeric socket addresses and Unix socket addresses as their filesystem
  * path.
  *
  * **Gotchas**

package/src/unstable/cluster/SqlMessageStorage.ts

@@ -666,7 +666,7 @@ export const makeEncoded: (options?: {
     unprocessedMessagesById(ids, now) {
       const idArr = Array.from(ids, (id) => String(id))
       return sql<MessageRow & ReplyJoinRow>`
-        SELECT m.*, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence
+        SELECT m.*, r.id as reply_reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence
         FROM ${messagesTableSql} m
         LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id
         WHERE m.id IN (${sql.literal(idArr.join(","))})
@@ -964,13 +964,17 @@ const migrations = (options?: {
             ON ${messagesTableSql} (request_id);
           `.unprepared.pipe(Effect.ignore),
         pg: () =>
-          sql`
-            CREATE INDEX IF NOT EXISTS ${sql(shardLookupIndex)}
-            ON ${messagesTableSql} (shard_id, processed, last_read, deliver_at);
-
-            CREATE INDEX IF NOT EXISTS ${sql(requestIdLookupIndex)}
-            ON ${messagesTableSql} (request_id);
-          `.pipe(
+          Effect.gen(function*() {
+            yield* sql`
+              CREATE INDEX IF NOT EXISTS ${sql(shardLookupIndex)}
+              ON ${messagesTableSql} (shard_id, processed, last_read, deliver_at)
+            `
+            yield* sql`
+              CREATE INDEX IF NOT EXISTS ${sql(requestIdLookupIndex)}
+              ON ${messagesTableSql} (request_id)
+            `
+          }).pipe(
+            sql.withTransaction,
             Effect.tapDefect((error) =>
               Effect.annotateLogs(Effect.logDebug("Failed to create indexes", error), {
                 package: "@effect/cluster",

package/src/unstable/cluster/SqlRunnerStorage.ts

@@ -427,7 +427,7 @@ export const make = Effect.fnUntraced(function*(options: {
         )
         for (let i = 0; i < takenLocks.length; i++) {
           const lockNum = takenLocks[i][0] as number
-          const shardId = lockNumbersReverse.get(lockNum)
+          const shardId = toAcquire.get(lockNum)
           if (shardId === undefined) continue
           acquiredShardIds.push(shardId)
           toAcquire.delete(lockNum)
@@ -530,7 +530,6 @@ export const make = Effect.fnUntraced(function*(options: {
   })
 
   const lockNumbers = new Map<string, number>()
-  const lockNumbersReverse = new Map<number, string>()
   for (let i = 0; i < availableShardGroups.length; i++) {
     const group = availableShardGroups[i]
     const base = (i + 1) * 1000000
@@ -538,7 +537,6 @@ export const make = Effect.fnUntraced(function*(options: {
       const shardId = ShardId.make(group, shard).toString()
       const lockNum = base + shard
       lockNumbers.set(shardId, lockNum)
-      lockNumbersReverse.set(lockNum, shardId)
     }
   }
 
@@ -737,18 +735,28 @@ export const make = Effect.fnUntraced(function*(options: {
         withTracerDisabled
       ),
 
-    refresh: (address, shardIds) =>
-      withLockOperationDeadline(
-        sql`UPDATE ${runnersTableSql} SET last_heartbeat = ${sqlNow} WHERE address = ${address}`.pipe(
+    refresh: (address, shardIds) => {
+      const heartbeat = sql`UPDATE ${runnersTableSql} SET last_heartbeat = ${sqlNow} WHERE address = ${address}`
+      // An empty refresh is the liveness probe used while lock storage is
+      // unhealthy. Run it on the shared pool, so a wedged reserved lock
+      // connection cannot block recovery.
+      if (shardIds.length === 0) {
+        return withDeadline(heartbeat).pipe(
+          Effect.as([]),
+          PersistenceError.refail,
+          withTracerDisabled
+        )
+      }
+      return withLockOperationDeadline(
+        heartbeat.pipe(
           execWithLockConn,
-          shardIds.length > 0 ?
-            Effect.andThen(refreshShards(address, shardIds)) :
-            Effect.as([])
+          Effect.andThen(refreshShards(address, shardIds))
         )
       ).pipe(
         PersistenceError.refail,
         withTracerDisabled
-      ),
+      )
+    },
 
     release: (address, shardId) =>
       withLockOperationDeadline(releaseShard(address, shardId)).pipe(

package/src/unstable/cluster/internal/clusterAbandon.ts

@@ -0,0 +1,14 @@
+import type * as Cause from "../../../Cause.ts"
+import * as Effect from "../../../Effect.ts"
+import * as Fiber from "../../../Fiber.ts"
+import { Abandon } from "../ClusterSchema.ts"
+
+export const interrupt: Effect.Effect<never> = Effect.interruptible(
+  Effect.callback<never>(() => {
+    const fiber = Fiber.getCurrent()!
+    fiber.interruptUnsafe(fiber.id, Abandon.annotation)
+  })
+)
+
+export const isCause = (cause: Cause.Cause<unknown>): boolean =>
+  cause.reasons.some((reason) => reason._tag === "Interrupt" && reason.annotations.has(Abandon.key))

package/src/unstable/cluster/internal/entityManager.ts

@@ -37,7 +37,7 @@ import type { Sharding } from "../Sharding.ts"
 import { ShardingConfig } from "../ShardingConfig.ts"
 import * as Snowflake from "../Snowflake.ts"
 import { EntityReaper } from "./entityReaper.ts"
-import { internalInterruptors } from "./interruptors.ts"
+import { acquireEntity, releaseEntity } from "./interruptors.ts"
 import { ResourceMap } from "./resourceMap.ts"
 import { ResourceRef } from "./resourceRef.ts"
 
@@ -168,6 +168,13 @@ export const make = Effect.fnUntraced(function*<
       force: Latch.makeUnsafe()
     }
 
+    yield* Scope.addFinalizer(
+      scope,
+      Effect.sync(() => {
+        releaseEntity(address)
+      })
+    )
+
     // on shutdown, reset the storage for the entity
     yield* Scope.addFinalizerExit(
       scope,
@@ -356,7 +363,8 @@ export const make = Effect.fnUntraced(function*<
         }
 
         return server.write
-      })
+      }),
+      address
     )
 
     function onDefect(cause: Cause.Cause<never>): Effect.Effect<void> {
@@ -405,9 +413,9 @@ export const make = Effect.fnUntraced(function*<
     // If the termination timeout is reached, let the server clean itself up
     yield* Scope.addFinalizer(
       scope,
-      Effect.withFiber((fiber) => {
+      Effect.suspend(() => {
         activeServers.delete(address.entityId)
-        internalInterruptors.add(fiber.id)
+        acquireEntity(address)
         return Effect.raceFirst(
           state.write(0, { _tag: "Eof" }).pipe(
             Effect.andThen(endLatch.await),