effect 4.0.3 → 4.0.5

package/src/cluster/ClusterWorkflowEngine.ts

@@ -282,7 +282,7 @@ export const make = Effect.gen(function*() {
     yield* storage.clearAddress(clockAddress)
   })
 
-  const resume = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string) {
+  const resume = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string, mode: WakeMode) {
     const maybeReply = yield* requestReply({
       workflow,
       entityType: `Workflow/${workflow._tag}`,
@@ -291,12 +291,11 @@ export const make = Effect.gen(function*() {
       id: ""
     })
 
-    const maybeSuspended = Option.filter(
-      maybeReply,
-      (reply) => reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended"
-    )
+    const maybeSuspended = Option.filter(maybeReply, isSuspendedReply)
 
     if (Option.isNone(maybeSuspended)) return
+    // Replies without `failed` retain automatic wakes for compatibility.
+    if (mode === "automatic" && isFailedReply(maybeSuspended.value)) return
     if (
       !(yield* sharding.reset(
         Snowflake.Snowflake(maybeSuspended.value.requestId),
@@ -308,6 +307,15 @@ export const make = Effect.gen(function*() {
     yield* sharding.pollStorage
   })
 
+  // Activations load a persisted interrupt when they start, so the per-call
+  // check reads only memory.
+  const interruptPending = (instance: WorkflowEngine.WorkflowInstance["Service"]) =>
+    Effect.map(Effect.serviceOption(CurrentActivationScope), (activation) => {
+      const pending = Option.isSome(activation) ? pendingByActivation.get(activation.value) : undefined
+      return pending?.workflowName === instance.workflow._tag && pending.executionId === instance.executionId &&
+        pending.results.has(InterruptSignal.name)
+    })
+
   const sendResumeParent = Effect.fnUntraced(function*(options: {
     readonly workflowName: string
     readonly executionId: string
@@ -328,10 +336,14 @@ export const make = Effect.gen(function*() {
     yield* sharding.reset(requestId.value)
   }, Effect.scoped)
 
-  // A child or deferred may complete while a run is still unwinding. Subscribe
-  // to the run's reply before reading storage so the wake cannot fall between
-  // the run's last read and its suspended reply.
-  const waitForRunReply = Effect.fnUntraced(function*(workflow: Workflow.Any, request: Entity.Request<any>) {
+  // Subscribe to the run's next reply before reading storage, so a reply
+  // cannot fall between the read and the subscription. Returns `true` when the
+  // stored reply already settles the wait, `false` when a new reply arrived.
+  const awaitRunReply = Effect.fnUntraced(function*(
+    workflow: Workflow.Any,
+    request: Envelope.Request<any>,
+    isSettled: (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) => boolean
+  ) {
     const waiter = yield* storage.registerReplyHandler(
       new Message.OutgoingRequest<any>({
         envelope: request,
@@ -343,12 +355,45 @@ export const make = Effect.gen(function*() {
       })
     ).pipe(Effect.forkScoped({ startImmediately: true }))
     const reply = yield* replyForRequestId(request.requestId)
-    if (Option.isSome(reply)) return
-    yield* Fiber.join(waiter).pipe(
+    if (Option.isSome(reply) && isSettled(reply.value)) return true
+    yield* Fiber.join(waiter)
+    return false
+  }, Effect.scoped)
+
+  // A child or deferred may complete while a run is still unwinding. Wait for
+  // the run's reply so the wake cannot fall between the run's last read and
+  // its suspended reply.
+  const waitForRunReply = (workflow: Workflow.Any, request: Entity.Request<any>) =>
+    awaitRunReply(workflow, request, () => true).pipe(
+      Effect.asVoid,
       Effect.catchTag("EntityNotAssignedToRunner", () => ClusterAbandon.interrupt),
       Effect.catchCause((cause) => ClusterAbandon.isCause(cause) ? ClusterAbandon.interrupt : Effect.failCause(cause))
     )
-  }, Effect.scoped)
+
+  // Wake an external caller once its run publishes a reply other than
+  // `Suspended`. Replies are only observed on the runner that saves them, so
+  // callers on other runners, and any failure here, leave delivery to the
+  // caller's retry schedule.
+  const awaitComplete = (workflow: Workflow.Any, executionId: string) =>
+    Effect.gen(function*() {
+      const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId })
+      if (!sharding.hasShardId(address.shardId)) return yield* Effect.never
+      const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" })
+      if (Option.isNone(requestId)) return yield* Effect.never
+      const request = Envelope.makeRequest<any>({
+        requestId: requestId.value,
+        address,
+        tag: "run",
+        payload: {},
+        headers: Headers.empty
+      })
+      // A replay that suspends again re-arms the wait, so it does not spend a
+      // step of the caller's retry schedule.
+      let settled = false
+      while (!settled) {
+        settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply))
+      }
+    }).pipe(Effect.catchCause(() => Effect.never))
 
   const interrupt = Effect.fnUntraced(
     function*(workflow: Workflow.Any, executionId: string) {
@@ -367,7 +412,7 @@ export const make = Effect.gen(function*() {
 
       const nonSuspendedReply = Option.filter(
         reply,
-        (reply) => reply.exit._tag !== "Success" || reply.exit.value._tag !== "Suspended"
+        (reply) => !isSuspendedReply(reply)
       )
       if (Option.isSome(nonSuspendedReply)) {
         return Option.none()
@@ -402,7 +447,25 @@ export const make = Effect.gen(function*() {
             const activation = yield* CurrentActivationScope
             let pending = pendingByActivation.get(activation)
             if (!pending) {
-              pending = { workflowName: workflow._tag, executionId, results: new Map() }
+              const results = new Map<string, Exit.Exit<unknown, unknown>>()
+              // A new owner must see an interrupt persisted before it took over.
+              // Publish the cache only once loaded, so a retried build repeats
+              // a failed lookup.
+              const interrupt = yield* Effect.orDie(requestReply({
+                workflow,
+                entityType: `Workflow/${workflow._tag}`,
+                executionId,
+                tag: "deferred",
+                id: InterruptSignal.name
+              }))
+              if (Option.isSome(interrupt)) {
+                const decoded = decodeDeferredWithExit(interrupt.value as any)
+                results.set(
+                  InterruptSignal.name,
+                  (decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit) as Exit.Exit<unknown, unknown>
+                )
+              }
+              pending = { workflowName: workflow._tag, executionId, results }
               pendingByActivation.set(activation, pending)
             }
             // Retain awaited names through reply persistence, including suspension.
@@ -410,25 +473,30 @@ export const make = Effect.gen(function*() {
             let currentRun: {
               readonly request: Entity.Request<any>
               readonly instance: WorkflowEngine.WorkflowInstance["Service"]
+              // Activity awaits must wake the run after suspension without preempting it.
+              readonly activityAwaits: Map<string, number>
             } | undefined
             // Concurrent wakes share one wait for the current run to publish its reply.
+            // Explicit wakes cannot be dropped: operator resume bypasses the gate;
+            // InterruptSignal enters it only on a fresh entity, with no run to wait for.
             const resumeGate = Semaphore.makeUnsafe(1)
-            const resumeCurrentRun = Effect.suspend(() =>
-              resumeGate.withPermitsIfAvailable(1)(
-                currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
+            const resumeCurrentRun = (mode: WakeMode) =>
+              Effect.suspend(() =>
+                resumeGate.withPermitsIfAvailable(1)(
+                  currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
+                )
+              ).pipe(
+                // Release the gate before reset can start another run, so later
+                // completions can wake that replay.
+                Effect.flatMap((waited) => Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void),
+                ensureSuccess
               )
-            ).pipe(
-              // Release the gate before reset can start another run, so later
-              // completions can wake that replay.
-              Effect.flatMap((waited) => Option.isSome(waited) ? resume(workflow, executionId) : Effect.void),
-              ensureSuccess
-            )
             return {
               run: (request: Entity.Request<any>) => {
                 const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId)
-                currentRun = { request, instance }
+                currentRun = { request, instance, activityAwaits: new Map() }
                 const parent = (request.payload as any)[payloadParentKey] as
-                  | { workflowName: string; executionId: string }
+                  | { workflowName: string; executionId: string; detached?: boolean }
                   | undefined
                 return execute(workflow.payloadSchema.make(request.payload) as object, executionId).pipe(
                   Effect.onExit((exit) => {
@@ -445,7 +513,9 @@ export const make = Effect.gen(function*() {
                       !instance.abandoned &&
                       !(suspendOnFailure && exit._tag === "Failure")
                     ) {
-                      return parent ? ensureSuccess(sendResumeParent(parent, executionId)) : Effect.void
+                      return parent && !parent.detached
+                        ? ensureSuccess(sendResumeParent(parent, executionId))
+                        : Effect.void
                     }
                     return engine.deferredResult(InterruptSignal).pipe(
                       Effect.flatMap((maybeExit) => {
@@ -472,7 +542,9 @@ export const make = Effect.gen(function*() {
               activity(request: Entity.Request<any>) {
                 const payload = request.payload as { name: string; attempt: number }
                 const activityId = `${executionId}/${payload.name}`
-                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId)
+                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId, {
+                  awaitedDeferreds: currentRun?.activityAwaits
+                })
                 interruptedActivities.delete(activityId)
                 const entry = activities.get(activityId)
                 if (!entry) {
@@ -512,18 +584,24 @@ export const make = Effect.gen(function*() {
               deferred: (request: Entity.Request<any>) => {
                 const payload = request.payload as any
                 pending.results.set(payload.name, payload.exit)
+                const mode: WakeMode = payload.name === InterruptSignal.name ? "explicit" : "automatic"
                 // Await registers before reading the cache. If this run has not
                 // registered yet, a later await will see the result just stored.
                 // Check when the wake runs and skip preemption too: interrupting
                 // a later await would require waiting for its reply before reset.
                 // A run completing its own deferred also reads it from the cache
-                // when it awaits later, so that completion never wakes the run.
+                // when it awaits later, unless an activity is already parked on it.
                 const wake = Effect.suspend(() =>
                   currentRun && !currentRun.instance.awaitedDeferreds.has(payload.name)
-                    ? currentRun.instance.completedDeferreds.has(payload.name)
+                    ? currentRun.activityAwaits.has(payload.name)
+                      // Hold an activity wake until the run has published its reply.
+                      ? resumeCurrentRun(mode)
+                      : currentRun.instance.completedDeferreds.has(payload.name)
                       ? Effect.void
-                      : ensureSuccess(resume(workflow, executionId))
-                    : deferredState.deferredDone(executionId, payload.name).pipe(Effect.andThen(resumeCurrentRun))
+                      : ensureSuccess(resume(workflow, executionId, mode))
+                    : deferredState.deferredDone(executionId, payload.name).pipe(
+                      Effect.andThen(resumeCurrentRun(mode))
+                    )
                 )
                 // An asynchronous reply releases the RPC concurrency permit while
                 // the wake waits; the pending request still occupies mailbox capacity.
@@ -538,7 +616,7 @@ export const make = Effect.gen(function*() {
                 )
               },
 
-              resume: () => resumeCurrentRun.pipe(Rpc.wrap({ fork: true, uninterruptible: true }))
+              resume: () => resumeCurrentRun("automatic").pipe(Rpc.wrap({ fork: true, uninterruptible: true }))
             }
           }),
           // Alongside the run, let deferred handlers briefly acquire a permit to
@@ -555,7 +633,7 @@ export const make = Effect.gen(function*() {
         ).pipe(Effect.andThen(registration))
       }),
 
-    execute: (workflow, { discard, executionId, parent, payload }) => {
+    execute: (workflow, { detached, discard, executionId, parent, payload }) => {
       ensureEntity(workflow)
       return RcMap.get(clients, workflow._tag).pipe(
         Effect.flatMap((make) =>
@@ -563,7 +641,10 @@ export const make = Effect.gen(function*() {
             parent ?
               {
                 ...payload,
-                [payloadParentKey]: { workflowName: parent.workflow._tag, executionId: parent.executionId }
+                // A detached child keeps its parent for observability only.
+                [payloadParentKey]: detached
+                  ? { workflowName: parent.workflow._tag, executionId: parent.executionId, detached: true }
+                  : { workflowName: parent.workflow._tag, executionId: parent.executionId }
               } :
               payload,
             { discard }
@@ -615,7 +696,9 @@ export const make = Effect.gen(function*() {
       ))
     }),
 
-    resume: (workflow, executionId) => ensureSuccess(resume(workflow, executionId)),
+    resume: (workflow, executionId) => ensureSuccess(resume(workflow, executionId, "explicit")),
+
+    awaitComplete,
 
     activityExecute: Effect.fnUntraced(
       function*(activity, attempt) {
@@ -623,6 +706,21 @@ export const make = Effect.gen(function*() {
         const instance = Context.get(services, WorkflowEngine.WorkflowInstance)
         yield* Effect.annotateCurrentSpan("executionId", instance.executionId)
         const activityId = `${instance.executionId}/${activity.name}`
+        // A pending interrupt replays only to rebuild compensations, so it must
+        // not run new activities. Finalizers run once `interrupted` is set.
+        if (!instance.interrupted && (yield* interruptPending(instance))) {
+          const reply = yield* Effect.orDie(requestReply({
+            workflow: instance.workflow,
+            entityType: `Workflow/${instance.workflow._tag}`,
+            executionId: instance.executionId,
+            tag: "activity",
+            id: activityPrimaryKey(activity.name, attempt)
+          }))
+          // Suspend so `onExit` applies the interrupt to the run.
+          if (Option.isNone(reply) || isSuspendedReply(reply.value)) {
+            return new Workflow.Suspended({})
+          }
+        }
         const client = (yield* RcMap.get(clientsPartial, instance.workflow._tag))(instance.executionId)
         while (true) {
           if (!activities.has(activityId)) {
@@ -809,6 +907,14 @@ const DeferredRpc = Rpc.make("deferred", {
   .annotate(ClusterSchema.Persisted, true)
   .annotate(ClusterSchema.Uninterruptible, true)
 
+type WakeMode = "explicit" | "automatic"
+
+const isSuspendedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) =>
+  reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended"
+
+const isFailedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) =>
+  reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended" && reply.exit.value.failed === true
+
 const decodeDeferredWithExit = Schema.decodeSync(Schema.toCodecJson(Reply.WithExit.schema(DeferredRpc)))
 
 const ResumeRpc = Rpc.make("resume", {
@@ -836,7 +942,8 @@ const makeWorkflowEntity = (workflow: Workflow.Any) =>
         ...workflow.payloadSchema.fields,
         [payloadParentKey]: Schema.optional(Schema.Struct({
           workflowName: Schema.String,
-          executionId: Schema.String
+          executionId: Schema.String,
+          detached: Schema.optional(Schema.Boolean)
         }))
       },
       primaryKey: () => "",

package/src/cluster/MessageStorage.ts

@@ -1086,8 +1086,8 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
           journal.push(envelope)
           return SaveResultEncoded.Success()
         }),
-      saveReply: (reply_) =>
-        Effect.sync(() => {
+      saveReply: (reply_) => {
+        const save = Effect.sync(() => {
           const reply = JSON.parse(JSON.stringify(reply_)) as Reply.Encoded
           const entry = requests.get(reply.requestId)
           if (!entry || replyIds.has(reply.id)) return
@@ -1098,7 +1098,18 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
           entry.replies.push(reply)
           replyIds.add(reply.id)
           replyLatch.openUnsafe()
-        }),
+        })
+        // Memory has no transaction isolation. Publish terminal replies at the
+        // outer commit boundary, before their queued handler notifications, so
+        // reply readers cannot deliver a success that may still be rolled back.
+        return TransactionNotifications.use((notifications) =>
+          reply_._tag === "WithExit" && notifications
+            ? Effect.sync(() => {
+              notifications.push(save)
+            })
+            : save
+        )
+      },
       clearReplies: (id, options) =>
         Effect.sync(() => {
           const entry = requests.get(String(id))

package/src/cluster/Runners.ts

@@ -14,6 +14,7 @@
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
 import * as Exit from "../Exit.ts"
+import * as Fiber from "../Fiber.ts"
 import * as Latch from "../Latch.ts"
 import * as Layer from "../Layer.ts"
 import * as Option from "../Option.ts"
@@ -247,78 +248,93 @@ export const make: (
 
   type StorageRequestEntry = {
     readonly latch: Latch.Latch
-    doneLatch: Latch.Latch | undefined
     readonly messages: Set<Message.OutgoingRequest<any>>
+    readonly pump: Fiber.Fiber<void>
     replies: Array<Reply.Reply<any>>
   }
   const storageRequests = new Map<Snowflake.Snowflake, StorageRequestEntry>()
   const waitingStorageRequests = new Map<Snowflake.Snowflake, Message.OutgoingRequest<any>>()
-  const replyFromStorage = Effect.fnUntraced(
-    function*(message: Message.OutgoingRequest<any>) {
-      let entry = storageRequests.get(message.envelope.requestId)
-      if (entry) {
-        entry.messages.add(message)
-        entry.doneLatch ??= Latch.makeUnsafe(false)
-        return yield* entry.doneLatch.await
-      } else {
-        entry = {
-          latch: Latch.makeUnsafe(false),
-          doneLatch: undefined,
-          replies: [],
-          messages: new Set([message])
-        }
-        storageRequests.set(message.envelope.requestId, entry)
-      }
 
-      while (true) {
-        // wait for the storage loop to notify us
-        entry.latch.closeUnsafe()
-        waitingStorageRequests.set(message.envelope.requestId, message)
-        storageLatch.openUnsafe()
-        yield* entry.latch.await
-
-        // send the replies back
-        for (let i = 0; i < entry.replies.length; i++) {
-          const reply = entry.replies[i]
-          // we have reached the end
-          if (reply._tag === "WithExit") {
-            // A reply can immediately reset and resend the same request. Do not
-            // let that sender attach to this completed reply loop.
-            storageRequests.delete(message.envelope.requestId)
-            waitingStorageRequests.delete(message.envelope.requestId)
-            for (const message of entry.messages) {
-              yield* message.respond(reply)
-            }
-            entry.doneLatch?.openUnsafe()
-            return
-          }
+  // A reset request can register a replacement entry under the same id, which
+  // must survive the old entry's cleanup.
+  const removeStorageRequest = (requestId: Snowflake.Snowflake, entry: Omit<StorageRequestEntry, "pump">) => {
+    if (storageRequests.get(requestId) !== entry) return
+    storageRequests.delete(requestId)
+    waitingStorageRequests.delete(requestId)
+  }
 
-          entry.latch.closeUnsafe()
+  // The pump is owned by the shared entry rather than the first caller, so any
+  // waiter can be interrupted without stopping delivery to the others.
+  const pumpReplies = Effect.fnUntraced(function*(
+    requestId: Snowflake.Snowflake,
+    entry: Omit<StorageRequestEntry, "pump">
+  ) {
+    while (true) {
+      // wait for the storage loop to notify us
+      entry.latch.closeUnsafe()
+      const message = entry.messages.values().next().value
+      if (!message) return
+      waitingStorageRequests.set(requestId, message)
+      storageLatch.openUnsafe()
+      yield* entry.latch.await
+
+      // send the replies back
+      for (let i = 0; i < entry.replies.length; i++) {
+        const reply = entry.replies[i]
+        // we have reached the end
+        if (reply._tag === "WithExit") {
+          // A reply can immediately reset and resend the same request. Do not
+          // let that sender attach to this completed entry.
+          removeStorageRequest(requestId, entry)
           for (const message of entry.messages) {
             yield* message.respond(reply)
           }
-          // wait for ack
-          yield* entry.latch.await
+          return
         }
-        entry.replies = []
+
+        entry.latch.closeUnsafe()
+        for (const message of entry.messages) {
+          yield* message.respond(reply)
+        }
+        // wait for ack
+        yield* entry.latch.await
       }
-    },
-    (effect, message) =>
-      Effect.ensuring(
-        effect,
-        Effect.sync(() => {
-          const entry = storageRequests.get(message.envelope.requestId)
-          // An old waiter must not clean up a reset request's new reply loop.
-          if (!entry?.messages.has(message)) return
-          if (entry.messages.size > 1) {
-            entry.messages.delete(message)
-            return
+      entry.replies = []
+    }
+  })
+
+  const replyFromStorage = (message: Message.OutgoingRequest<any>) =>
+    Effect.uninterruptibleMask((restore) =>
+      Effect.gen(function*() {
+        const requestId = message.envelope.requestId
+        let entry = storageRequests.get(requestId)
+        if (!entry) {
+          const state = {
+            latch: Latch.makeUnsafe(false),
+            messages: new Set<Message.OutgoingRequest<any>>(),
+            replies: [] as Array<Reply.Reply<any>>
           }
-          storageRequests.delete(message.envelope.requestId)
-          waitingStorageRequests.delete(message.envelope.requestId)
-        })
-      )
-  )
+          entry = Object.assign(state, { pump: yield* Effect.forkIn(pumpReplies(requestId, state), runnersScope) })
+          storageRequests.set(requestId, entry)
+        }
+        const current = entry
+        current.messages.add(message)
+        return yield* restore(Fiber.join(current.pump)).pipe(
+          Effect.ensuring(Effect.suspend(() => {
+            current.messages.delete(message)
+            if (current.messages.size === 0) {
+              removeStorageRequest(requestId, current)
+              return Fiber.interrupt(current.pump)
+            }
+            // The storage loop decodes replies using the waiting message.
+            if (waitingStorageRequests.get(requestId) === message) {
+              waitingStorageRequests.set(requestId, current.messages.values().next().value!)
+            }
+            return Effect.void
+          }))
+        )
+      })
+    )
 
   const storageLatch = Latch.makeUnsafe(false)
   if (storage !== MessageStorage.noop) {
@@ -415,22 +431,28 @@ export const make: (
       })
     },
     notifyLocal(options) {
-      return notifyWith(options.message, (message, duplicate) => {
+      return notifyWith(options.message, (message) => {
         if (options.discard || message._tag === "OutgoingEnvelope") {
           return Effect.catchTag(
             options.notify(Message.incomingLocalFromOutgoing(message)),
             "EntityNotAssignedToRunner",
             () => Effect.void
           )
-        } else if (!duplicate && options.storageOnly !== true) {
-          return options.notify(Message.incomingLocalFromOutgoing(message)).pipe(
-            Effect.andThen(storage.registerReplyHandler(message)),
-            Effect.catchTag("EntityNotAssignedToRunner", () => replyFromStorage(message))
-          )
         }
+        // The request can be processed as soon as it is saved, so replies are
+        // always read back from storage. The local reply handler only wakes the
+        // storage loop, so streams do not have to wait for the next poll.
+        const readReplies = options.storageOnly
+          ? replyFromStorage(message)
+          : Effect.raceFirst(
+            replyFromStorage(message),
+            storage.registerReplyHandler(
+              new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open })
+            ).pipe(Effect.ignore, Effect.andThen(Effect.never))
+          )
         return options.notify(Message.incomingLocalFromOutgoing(message)).pipe(
           Effect.catchTag("EntityNotAssignedToRunner", () => Effect.void),
-          Effect.andThen(replyFromStorage(message))
+          Effect.andThen(readReplies)
         )
       })
     }

package/src/cluster/Sharding.ts

@@ -692,6 +692,12 @@ const make = Effect.gen(function*() {
   const storageReadLatch = yield* Latch.make(true)
   const openStorageReadLatch = constant(Effect.asVoid(storageReadLatch.open))
 
+  type PendingNotification = {
+    resume: (_: Effect.Effect<void, EntityNotAssignedToRunner>) => void
+    readonly message: Message.IncomingRequest<any>
+  }
+  const pendingNotifications = new Map<Snowflake.Snowflake, PendingNotification>()
+
   const storageReadLock = Semaphore.makeUnsafe(1)
   const withStorageReadLock = storageReadLock.withPermits(1)
 
@@ -725,7 +731,7 @@ const make = Effect.gen(function*() {
   if (storageEnabled && initialRunnerAddress) {
     const selfAddress = initialRunnerAddress
 
-    yield* Effect.gen(function*() {
+    const storageReadLoop = Effect.gen(function*() {
       yield* Effect.logDebug("Starting")
       yield* Effect.addFinalizer(() => Effect.logDebug("Shutting down"))
 
@@ -982,8 +988,7 @@ const make = Effect.gen(function*() {
         module: "effect/cluster/Sharding",
         fiber: "Storage read loop",
         runner: selfAddress
-      }),
-      Effect.forkIn(shardingScope)
+      })
     )
 
     // open the storage latch every poll interval
@@ -1110,6 +1115,9 @@ const make = Effect.gen(function*() {
         ),
       Effect.forkIn(shardingScope)
     )
+
+    // fork after the state the loop reads has been initialized
+    yield* Effect.forkIn(storageReadLoop, shardingScope)
   }
 
   // --- Sending messages ---
@@ -1150,11 +1158,6 @@ const make = Effect.gen(function*() {
     return isRequest ? Effect.andThen(awaitShardLock(address.shardId), send) : send
   }
 
-  type PendingNotification = {
-    resume: (_: Effect.Effect<void, EntityNotAssignedToRunner>) => void
-    readonly message: Message.IncomingRequest<any>
-  }
-  const pendingNotifications = new Map<Snowflake.Snowflake, PendingNotification>()
   const notifyLocal = <M extends Message.Outgoing<any> | Message.Incoming<any>>(
     message: M,
     discard: boolean,
@@ -1172,7 +1175,7 @@ const make = Effect.gen(function*() {
       const state = entityManagers.get(address.entityType)
       if (!state) {
         return Effect.flatMap(waitForEntityManager(address.entityType), loop)
-      } else if (state.status === "closed" || !isEntityOnLocalShards(address)) {
+      } else if (state.status === "closed") {
         return Effect.fail(new EntityNotAssignedToRunner({ address }))
       }
 

package/src/cluster/internal/entityManager.ts

@@ -250,7 +250,7 @@ export const make = Effect.fnUntraced(function*<
         ) as Effect.Effect<Context.Context<Rpc.ToHandler<Rpcs>>>)
 
         const server = yield* RpcServer.makeNoSerialization(entity.protocol, {
-          spanPrefix: `${entity.type}(${address.entityId})`,
+          spanPrefix: entity.type,
           spanAttributes: {
             ...options.spanAttributes,
             "entity.type": entity.type,