effect 4.0.5 → 4.0.6

package/src/cli/internal/parser.ts

@@ -125,6 +125,8 @@ type FlagRegistry = {
   readonly params: ReadonlyArray<FlagParam>
   /** Maps canonical names and aliases → flag param */
   readonly index: Map<string, FlagParam>
+  /** Maps negated names of boolean flags → flag param */
+  readonly negations: Map<string, FlagParam>
 }
 
 /**
@@ -229,12 +231,22 @@ const makeCursor = (tokens: ReadonlyArray<Token>): TokenCursor => {
 /* Flag Registry                                                              */
 /* ========================================================================== */
 
+/**
+ * Long name that sets a boolean flag to `false`, or `undefined` when the flag
+ * has no negated form.
+ */
+const negatedName = (param: FlagParam): string | undefined =>
+  !Primitive.isBoolean(param.primitiveType) || param.negation === false
+    ? undefined
+    : param.negation ?? `no-${param.name}`
+
 /**
  * Creates a registry for O(1) flag lookup by name or alias.
  * @internal
  */
 export const createFlagRegistry = (params: ReadonlyArray<FlagParam>): FlagRegistry => {
   const index = new Map<string, FlagParam>()
+  const negations = new Map<string, FlagParam>()
 
   for (const param of params) {
     if (index.has(param.name)) {
@@ -250,9 +262,19 @@ export const createFlagRegistry = (params: ReadonlyArray<FlagParam>): FlagRegist
       }
       index.set(alias, param)
     }
+
+    const negation = negatedName(param)
+    if (negation !== undefined) {
+      if (negations.has(negation)) {
+        throw new Error(
+          `Duplicate negation "${negation}" in command definition (conflicts with "${negations.get(negation)!.name}")`
+        )
+      }
+      negations.set(negation, param)
+    }
   }
 
-  return { params, index }
+  return { params, index, negations }
 }
 
 const buildSubcommandIndex = (
@@ -362,14 +384,11 @@ const resolveFlag = (
     }
   }
 
-  if (token._tag === "LongOption" && token.name.startsWith("no-")) {
-    const canonicalName = token.name.slice(3)
-    const param = registry.index.get(canonicalName)
-    if (param && param.name === canonicalName && Primitive.isBoolean(param.primitiveType)) {
-      return {
-        param,
-        negated: true
-      }
+  const negated = token._tag === "LongOption" ? registry.negations.get(token.name) : undefined
+  if (negated) {
+    return {
+      param: negated,
+      negated: true
     }
   }
 
@@ -740,8 +759,9 @@ const createUnrecognizedFlagError = (
     // that was intentionally kept out of --help.
     if (p.hidden) continue
     validNames.push(p.name)
-    if (Primitive.isBoolean(p.primitiveType)) {
-      validNames.push(`no-${p.name}`)
+    const negation = negatedName(p)
+    if (negation !== undefined) {
+      validNames.push(negation)
     }
     for (const alias of p.aliases) {
       validNames.push(alias)

package/src/cluster/ClusterCron.ts

@@ -7,9 +7,15 @@
  * for distributed maintenance work where the job should be owned by the cluster
  * rather than by every runner independently.
  *
+ * With `calculateNextRunFromPrevious`, every run is processed by a single
+ * entity per cron job, so runs never overlap and are caught up in order. The
+ * scheduled time of the current run is available through
+ * `CurrentScheduledTime`.
+ *
  * @stability unstable
  * @since 4.0.0
  */
+import * as Context from "../Context.ts"
 import * as Cron from "../Cron.ts"
 import * as DateTime from "../DateTime.ts"
 import * as Duration from "../Duration.ts"
@@ -29,6 +35,23 @@ import * as Entity from "./Entity.ts"
 import type { Sharding } from "./Sharding.ts"
 import * as Singleton from "./Singleton.ts"
 
+/**
+ * Service tag for the scheduled time of the cron run currently executing.
+ *
+ * **When to use**
+ *
+ * Use to read the time the current run was scheduled for, inside the
+ * `execute` effect of `make`.
+ *
+ * @stability unstable
+ * @category services
+ * @since 4.0.0
+ */
+export class CurrentScheduledTime extends Context.Service<
+  CurrentScheduledTime,
+  DateTime.Utc
+>()("effect/cluster/ClusterCron/CurrentScheduledTime") {}
+
 /**
  * Creates a layer that runs a cron job through the cluster sharding system.
  *
@@ -60,6 +83,11 @@ export const make = <E, R>(options: {
    *
    * Defaults to `false`, meaning the next run will be calculated from the
    * current time.
+   *
+   * When enabled, all runs are processed one at a time by a single entity, so
+   * missed runs are caught up in order without overlapping. A run that fails
+   * is retried in place and blocks later runs until it succeeds or is older
+   * than `skipIfOlderThan`.
    */
   readonly calculateNextRunFromPrevious?: boolean | undefined
 
@@ -76,26 +104,42 @@ export const make = <E, R>(options: {
    * Defaults to "1 day".
    */
   readonly skipIfOlderThan?: Duration.Input | undefined
-}): Layer.Layer<never, never, Sharding | Exclude<R, Scope>> => {
+}): Layer.Layer<never, never, Sharding | Exclude<Exclude<R, CurrentScheduledTime>, Scope>> => {
+  const fromPrevious = options.calculateNextRunFromPrevious === true
+  const nextRun = (after: DateTime.Utc) => DateTime.fromDateUnsafe(Cron.next(options.cron, after))
+
   const CronEntity = Entity.make(`ClusterCron/${options.name}`, [
     Rpc.make("run", {
       payload: CronPayload
     })
       .annotate(Persisted, true)
-      .annotate(Uninterruptible, true)
+      .annotate(Uninterruptible, true),
+    Rpc.make("start").annotate(Persisted, true)
   ])
     .annotate(ClusterSchema.ShardGroup, () => options.shardGroup ?? "default")
     .annotate(ClusterSchema.ClientTracingEnabled, false)
 
+  // with `calculateNextRunFromPrevious`, every run shares the executor address
+  const send = (dateTime: DateTime.Utc) =>
+    Effect.flatMap(CronEntity.client, (makeClient) =>
+      fromPrevious
+        ? makeClient(executorId).run(new CronPayload({ dateTime, executor: true }), { discard: true })
+        : makeClient(DateTime.formatIso(dateTime)).run(new CronPayload({ dateTime }), { discard: true }))
+
+  const schedule = (dateTime: DateTime.Utc) => send(dateTime).pipe(withRetry("Failed to schedule next run, retrying"))
+
   const InitialRun = Singleton.make(
     `ClusterCron/${options.name}`,
-    Effect.gen(function*() {
-      const now = yield* DateTime.now
-      const next = DateTime.fromDateUnsafe(Cron.next(options.cron, now))
-      const entityId = options.calculateNextRunFromPrevious ? "initial" : DateTime.formatIso(next)
-      const client = (yield* CronEntity.client)(entityId)
-      yield* client.run({ dateTime: next }, { discard: true })
-    }),
+    fromPrevious
+      // the executor schedules the next run if its chain has been lost.
+      // `start` has no primary key, so each retry is a new message
+      ? Effect.flatMap(CronEntity.client, (makeClient) => makeClient(executorId).start()).pipe(
+        withRetry("Failed to start cron job, retrying")
+      )
+      : Effect.flatMap(
+        DateTime.now,
+        (now) => send(nextRun(now)).pipe(withRetry("Failed to schedule initial run, retrying"))
+      ),
     { shardGroup: options.shardGroup }
   )
 
@@ -104,61 +148,76 @@ export const make = <E, R>(options: {
     Option.getOrElse(() => Duration.days(1))
   )
 
-  const effect = Effect.fnUntraced(function*(dateTime: DateTime.Utc) {
+  const execute = Effect.fnUntraced(function*(dateTime: DateTime.Utc) {
     const now = yield* DateTime.now
     if (DateTime.isLessThan(dateTime, DateTime.subtractDuration(now, skipIfOlderThan))) {
       return
     }
-    return yield* options.execute
+    return yield* Effect.provideService(options.execute, CurrentScheduledTime, dateTime)
   }, Effect.orDie)
 
   const EntityLayer = CronEntity.toLayer(Effect.gen(function*() {
-    const makeClient = yield* CronEntity.client
+    const address = yield* Entity.CurrentAddress
+    // runs are processed one at a time, so a run processed before `start`
+    // means the chain is alive
+    let processedRun = false
     return {
-      run: (request) =>
-        Effect.onExitPrimitive(
-          effect(request.payload.dateTime),
-          Effect.fnUntraced(function*(exit) {
-            if (Exit.isFailure(exit)) {
-              yield* Effect.logWarning(exit.cause)
-            }
-            const now = yield* DateTime.now
-            const next = DateTime.fromDateUnsafe(Cron.next(
-              options.cron,
-              options.calculateNextRunFromPrevious ? request.payload.dateTime : now
-            ))
-            const client = makeClient(DateTime.formatIso(next))
-            return yield* client.run({ dateTime: next }, { discard: true }).pipe(
-              Effect.tapCause((cause) => Effect.logWarning("Failed to schedule next run, retrying", cause)),
-              Effect.sandbox,
-              Effect.retry(retryPolicy),
-              Effect.orDie
-            )
-          }),
+      run: (request) => {
+        const dateTime = request.payload.dateTime
+        if (fromPrevious && address.entityId !== executorId) {
+          // a run scheduled by an earlier release joins the executor's chain
+          return schedule(dateTime)
+        }
+        processedRun = true
+        return Effect.onExitPrimitive(
+          execute(dateTime),
+          (exit) =>
+            Effect.andThen(
+              Exit.isFailure(exit) ? Effect.logWarning(exit.cause) : Effect.void,
+              fromPrevious ? schedule(nextRun(dateTime)) : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now)))
+            ),
           true
         ).pipe(
           Effect.annotateLogs({
             module: "effect/cluster/ClusterCron",
             name: options.name,
-            dateTime: request.payload.dateTime
+            dateTime
           })
         )
+      },
+      start: () =>
+        processedRun
+          ? Effect.void
+          // deduplicated against the pending run if the chain is alive
+          : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now)))
     }
   }))
 
   return Layer.merge(InitialRun, EntityLayer)
 }
 
+const executorId = "executor"
+
 const retryPolicy = Schedule.min([
   Schedule.exponential(200, 1.5),
   Schedule.spaced("1 minute")
 ])
 
+const withRetry = (message: string) => <A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, never, R> =>
+  self.pipe(
+    Effect.tapCause((cause) => Effect.logWarning(message, cause)),
+    Effect.sandbox,
+    Effect.retry(retryPolicy),
+    Effect.orDie
+  )
+
 class CronPayload extends Schema.Class<CronPayload>("effect/cluster/ClusterCron/CronPayload")({
-  dateTime: Schema.DateTimeUtc
+  dateTime: Schema.DateTimeUtc,
+  // set for runs sent to the executor, which share one entity address
+  executor: Schema.optional(Schema.Boolean)
 }) {
   [PrimaryKey.symbol]() {
-    return ""
+    return this.executor ? DateTime.formatIso(this.dateTime) : ""
   }
   [DeliverAt.symbol]() {
     return this.dateTime

package/src/cluster/ClusterMetrics.ts

@@ -1,10 +1,11 @@
 /**
  * Standard metric definitions used by the unstable cluster runtime. The gauges
- * track runner-local entities, singleton processes, registered runners,
- * healthy runners, and acquired shards.
+ * track runner-local entities, entity mailbox sizes, singleton processes,
+ * registered runners, healthy runners, acquired shards, and shards being
+ * released. The timer tracks shard release duration.
  *
  * This module only defines the metric handles. The runtime components that
- * manage entities, runners, singletons, and sharding update these gauges while
+ * manage entities, runners, singletons, and sharding update these metrics while
  * the cluster is running.
  *
  * @stability unstable
@@ -23,13 +24,13 @@ import * as Metric from "../Metric.ts"
  *
  * **Details**
  *
- * Bigint gauge named `effect_cluster_entities`, updated with the entity type as
- * a metric tag.
+ * Bigint gauge named `effect_cluster_entities`, tagged with the entity type.
+ * Each entity manager registers a series that reads its active entity count
+ * when metrics are snapshotted, and removes it when the manager shuts down.
  *
  * **Gotchas**
  *
- * This gauge is runner-local and sampled by the entity manager loop. Aggregate
- * across runners and expect up to roughly one polling interval of lag.
+ * This gauge is runner-local. Aggregate across runners.
  *
  * @see {@link singletons} for singleton process counts on the current runner
  * @see {@link shards} for shard ownership on the current runner
@@ -40,6 +41,86 @@ import * as Metric from "../Metric.ts"
  */
 export const entities = Metric.gauge("effect_cluster_entities", { bigint: true })
 
+/**
+ * Represents a gauge tracking the mailbox size of each entity resident on the
+ * current runner.
+ *
+ * **When to use**
+ *
+ * Use to find entities whose mailboxes back up, before sends start failing
+ * with `MailboxFull`.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size`, with one series per
+ * resident entity tagged with `type`, `id` and `shard`. The value is the
+ * number the mailbox capacity check compares against: requests admitted to the
+ * entity, whether queued or in flight, excluding internal keep-alive requests.
+ *
+ * The value is read when metrics are snapshotted, so it adds no work per
+ * message. Each series is removed when its entity is released.
+ *
+ * **Gotchas**
+ *
+ * Off by default. Enable it with `ShardingConfig.entityMailboxMetrics` or the
+ * `mailboxMetrics` option when registering an entity. One series per resident
+ * entity can mean many series on busy runners.
+ *
+ * An entity keeps reporting while it drains in-flight requests during
+ * shutdown. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entities} for the number of resident entities per type
+ * @see {@link entityMailboxSizeMax} for the always-on per-type maximum
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSize = Metric.gauge("effect_cluster_entity_mailbox_size", { bigint: true })
+
+/**
+ * Represents a gauge tracking the largest mailbox size among the entities of
+ * each type resident on the current runner.
+ *
+ * **When to use**
+ *
+ * Use to alert on entity types whose mailboxes back up, before sends start
+ * failing with `MailboxFull`, without paying for a series per entity.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_entity_mailbox_size_max`, with one series
+ * per registered entity type tagged with `type`. The value is the largest
+ * mailbox size that {@link entityMailboxSize} would report for the type,
+ * including entities draining in-flight requests during shutdown, or `0` when
+ * the type has no resident entities.
+ *
+ * The value is computed when metrics are snapshotted, by visiting each
+ * resident entity of the type, so it adds no work per message. Entities
+ * draining during shutdown keep their slot until they finish, so the visit is
+ * bounded by `ShardingConfig.maxResidentEntities` across all types, unless it
+ * is `"unbounded"`. The series is removed when the entity type shuts down.
+ *
+ * **Gotchas**
+ *
+ * Always on. `Metric.update` and `Metric.modify` on this gauge do nothing for
+ * series the runtime registered.
+ *
+ * Like the other runner gauges it has no runner attribute: runners sharing a
+ * registry without distinct metric attributes overwrite each other's series.
+ *
+ * @see {@link entityMailboxSize} for the opt-in per-entity mailbox sizes
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.6
+ */
+export const entityMailboxSizeMax = Metric.gauge("effect_cluster_entity_mailbox_size_max", { bigint: true })
+
 /**
  * Creates a gauge tracking the number of singleton processes currently running on the
  * current runner.
@@ -116,3 +197,64 @@ export const runnersHealthy = Metric.gauge("effect_cluster_runners_healthy", { b
  * @since 4.0.0
  */
 export const shards = Metric.gauge("effect_cluster_shards", { bigint: true })
+
+/**
+ * Represents a gauge tracking the number of shards the current runner is still
+ * releasing.
+ *
+ * **When to use**
+ *
+ * Use to detect shard releases that stall during rebalancing, shutdown handoff
+ * or lock storage failures.
+ *
+ * **Details**
+ *
+ * Bigint gauge named `effect_cluster_shards_releasing`, updated with the number
+ * of distinct shards being released. A shard counts from the moment it is no
+ * longer acquired until its entities and singletons have stopped and its lock
+ * is released, including waits for lock storage to recover and release
+ * retries.
+ *
+ * **Gotchas**
+ *
+ * Positive values are normal while shards move; alert only when the value
+ * stays positive. It does not reflect lock ownership, and reacquisition can
+ * stay blocked briefly after it returns to zero, while a forced release
+ * finishes its bookkeeping.
+ *
+ * Like the other runner gauges it assumes one runner per process: runners
+ * sharing a registry without distinct metric attributes overwrite each other's
+ * value.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.1.0
+ */
+export const shardsReleasing = Metric.gauge("effect_cluster_shards_releasing", { bigint: true })
+
+/**
+ * Represents a timer tracking how long the current runner takes to release a
+ * shard after reassignment, lock loss or shutdown handoff.
+ *
+ * **When to use**
+ *
+ * Use to observe shard handoff latency during rebalancing, shutdown, or lock
+ * storage failures.
+ *
+ * **Details**
+ *
+ * Millisecond timer named `effect_cluster_shard_release`. Each sample covers
+ * entity teardown, retries and any wait for shard lock storage to recover.
+ *
+ * **Gotchas**
+ *
+ * Only completed releases are recorded. Interrupted releases and forced
+ * releases after lock storage becomes unhealthy are not.
+ *
+ * @stability unstable
+ * @category metrics
+ * @since 4.0.0
+ */
+export const shardReleaseDuration = Metric.timer("effect_cluster_shard_release", {
+  description: "Time taken to release a shard, including entity teardown, retries and lock storage recovery"
+})

package/src/cluster/ClusterWorkflowEngine.ts

@@ -24,7 +24,7 @@ import * as Rpc from "../rpc/Rpc.ts"
 import { ClientAbort } from "../rpc/RpcSchema.ts"
 import * as Schedule from "../Schedule.ts"
 import * as Schema from "../Schema.ts"
-import type * as Scope from "../Scope.ts"
+import * as Scope from "../Scope.ts"
 import * as Semaphore from "../Semaphore.ts"
 import * as Activity from "../workflow/Activity.ts"
 import * as DurableClock from "../workflow/DurableClock.ts"
@@ -135,11 +135,21 @@ export const make = Effect.gen(function*() {
   // Rebuilds share an activation; overlapping activations have separate results.
   // Weak keys do not retain scopes, but contexts captured by clients, activities,
   // or finalizers can keep a closed scope (and its results) reachable until later.
-  const pendingByActivation = new WeakMap<Scope.Scope, {
-    readonly workflowName: string
-    readonly executionId: string
-    readonly results: Map<string, Exit.Exit<unknown, unknown>>
-  }>()
+  const pendingByActivation = new WeakMap<Scope.Scope, ActivationState>()
+  // Closes the scopes retained by suspended runs. The latest run already closed
+  // its own scope with its real exit, so earlier finalizers observe success and
+  // skip superseded compensations. Abandonment releases owner-local resources
+  // without running durable finalizers, so the run can replay elsewhere.
+  const closeRuns = (state: ActivationState, abandoned: boolean) =>
+    Effect.suspend(() => {
+      const scope = state.scope
+      if (abandoned) {
+        for (const instance of state.instances) instance.abandoned = true
+      }
+      state.instances.clear()
+      state.scope = Scope.makeUnsafe()
+      return Scope.close(scope, Exit.void)
+    })
   const clients = yield* RcMap.make({
     lookup: Effect.fnUntraced(function*(workflowName: string) {
       const entity = entities.get(workflowName)
@@ -374,26 +384,25 @@ export const make = Effect.gen(function*() {
   // `Suspended`. Replies are only observed on the runner that saves them, so
   // callers on other runners, and any failure here, leave delivery to the
   // caller's retry schedule.
-  const awaitComplete = (workflow: Workflow.Any, executionId: string) =>
-    Effect.gen(function*() {
-      const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId })
-      if (!sharding.hasShardId(address.shardId)) return yield* Effect.never
-      const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" })
-      if (Option.isNone(requestId)) return yield* Effect.never
-      const request = Envelope.makeRequest<any>({
-        requestId: requestId.value,
-        address,
-        tag: "run",
-        payload: {},
-        headers: Headers.empty
-      })
-      // A replay that suspends again re-arms the wait, so it does not spend a
-      // step of the caller's retry schedule.
-      let settled = false
-      while (!settled) {
-        settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply))
-      }
-    }).pipe(Effect.catchCause(() => Effect.never))
+  const awaitComplete = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string) {
+    const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId })
+    if (!sharding.hasShardId(address.shardId)) return yield* Effect.never
+    const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" })
+    if (Option.isNone(requestId)) return yield* Effect.never
+    const request = Envelope.makeRequest<any>({
+      requestId: requestId.value,
+      address,
+      tag: "run",
+      payload: {},
+      headers: Headers.empty
+    })
+    // A replay that suspends again re-arms the wait, so it does not spend a
+    // step of the caller's retry schedule.
+    let settled = false
+    while (!settled) {
+      settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply))
+    }
+  }, Effect.catchCause(() => Effect.never))
 
   const interrupt = Effect.fnUntraced(
     function*(workflow: Workflow.Any, executionId: string) {
@@ -465,7 +474,14 @@ export const make = Effect.gen(function*() {
                   (decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit) as Exit.Exit<unknown, unknown>
                 )
               }
-              pending = { workflowName: workflow._tag, executionId, results }
+              pending = {
+                workflowName: workflow._tag,
+                executionId,
+                results,
+                instances: new Set(),
+                scope: Scope.makeUnsafe()
+              }
+              yield* Scope.addFinalizer(activation, closeRuns(pending, true))
               pendingByActivation.set(activation, pending)
             }
             // Retain awaited names through reply persistence, including suspension.
@@ -476,24 +492,34 @@ export const make = Effect.gen(function*() {
               // Activity awaits must wake the run after suspension without preempting it.
               readonly activityAwaits: Map<string, number>
             } | undefined
-            // Concurrent wakes share one wait for the current run to publish its reply.
-            // Explicit wakes cannot be dropped: operator resume bypasses the gate;
-            // InterruptSignal enters it only on a fresh entity, with no run to wait for.
+            // Concurrent automatic wakes share one wait for the current run to
+            // publish its reply. Explicit wakes cannot be dropped, because an
+            // automatic wake skips the reset of a failure suspension, so they
+            // wait outside the gate. Operator resume does not use the gate.
             const resumeGate = Semaphore.makeUnsafe(1)
+            const waitForCurrentRunReply = Effect.suspend(() =>
+              currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
+            )
             const resumeCurrentRun = (mode: WakeMode) =>
-              Effect.suspend(() =>
-                resumeGate.withPermitsIfAvailable(1)(
-                  currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void
-                )
-              ).pipe(
-                // Release the gate before reset can start another run, so later
-                // completions can wake that replay.
-                Effect.flatMap((waited) => Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void),
-                ensureSuccess
+              ensureSuccess(
+                mode === "explicit"
+                  ? Effect.andThen(waitForCurrentRunReply, resume(workflow, executionId, mode))
+                  : resumeGate.withPermitsIfAvailable(1)(waitForCurrentRunReply).pipe(
+                    // Release the gate before reset can start another run, so later
+                    // completions can wake that replay.
+                    Effect.flatMap((waited) =>
+                      Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void
+                    )
+                  )
               )
             return {
               run: (request: Entity.Request<any>) => {
-                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId)
+                // Each run gets a fresh scope under the activation's parent, so
+                // replays rebuild compensations while suspended runs keep theirs.
+                const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId, {
+                  scope: Scope.forkUnsafe(pending.scope)
+                })
+                pending.instances.add(instance)
                 currentRun = { request, instance, activityAwaits: new Map() }
                 const parent = (request.payload as any)[payloadParentKey] as
                   | { workflowName: string; executionId: string; detached?: boolean }
@@ -533,6 +559,11 @@ export const make = Effect.gen(function*() {
                     )
                   }),
                   Workflow.intoResult,
+                  Effect.onExit((exit) =>
+                    Exit.isSuccess(exit) && exit.value._tag === "Suspended"
+                      ? Effect.void
+                      : closeRuns(pending, instance.abandoned)
+                  ),
                   (effect) => deferredState.trackRun(instance, effect),
                   // Request middleware and local callers may carry another entity's context.
                   Effect.provideService(CurrentActivationScope, activation)
@@ -591,8 +622,10 @@ export const make = Effect.gen(function*() {
                 // a later await would require waiting for its reply before reset.
                 // A run completing its own deferred also reads it from the cache
                 // when it awaits later, unless an activity is already parked on it.
+                // A run only reads InterruptSignal while suspending, so an interrupt
+                // always waits for the current run's reply before it resets the run.
                 const wake = Effect.suspend(() =>
-                  currentRun && !currentRun.instance.awaitedDeferreds.has(payload.name)
+                  currentRun && mode === "automatic" && !currentRun.instance.awaitedDeferreds.has(payload.name)
                     ? currentRun.activityAwaits.has(payload.name)
                       // Hold an activity wake until the run has published its reply.
                       ? resumeCurrentRun(mode)
@@ -909,6 +942,16 @@ const DeferredRpc = Rpc.make("deferred", {
 
 type WakeMode = "explicit" | "automatic"
 
+interface ActivationState {
+  readonly workflowName: string
+  readonly executionId: string
+  readonly results: Map<string, Exit.Exit<unknown, unknown>>
+  // Every run forks its scope from here, so runs that suspended keep their
+  // resources and durable finalizers until the execution finishes.
+  scope: Scope.Closeable
+  readonly instances: Set<WorkflowEngine.WorkflowInstance["Service"]>
+}
+
 const isSuspendedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>) =>
   reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended"