@effect/cluster 0.38.9 → 0.38.11

package/src/ClusterWorkflowEngine.ts

@@ -141,7 +141,15 @@ export const make = Effect.gen(function*() {
       times: 3,
       schedule: Schedule.exponential(250)
     }),
-    Effect.orDie
+    Effect.orDie,
+    (effect, { activity, attempt, executionId }) =>
+      Effect.withSpan(effect, "WorkflowEngine.resetActivityAttempt", {
+        attributes: {
+          name: activity.name,
+          executionId,
+          attempt
+        }
+      })
   )
 
   const clearClock = Effect.fnUntraced(function*(options: {
@@ -289,40 +297,59 @@ export const make = Effect.gen(function*() {
         times: 3,
         schedule: Schedule.exponential(250)
       }),
-      Effect.orDie
+      Effect.orDie,
+      (effect, workflow, executionId) =>
+        Effect.withSpan(effect, "WorkflowEngine.interrupt", {
+          attributes: {
+            name: workflow.name,
+            executionId
+          }
+        })
     ),
 
-    activityExecute: Effect.fnUntraced(function*({ activity, attempt }) {
-      const context = yield* Effect.context<WorkflowInstance>()
-      const instance = Context.get(context, WorkflowInstance)
-      const activityId = `${instance.executionId}/${activity.name}`
-      activities.set(activityId, { activity, context })
-      const latch = activityLatches.get(activityId)
-      if (latch) {
-        yield* latch.release
-        activityLatches.delete(activityId)
-      }
-      const client = (yield* RcMap.get(clients, instance.workflow.name))(instance.executionId)
-      while (true) {
-        const result = yield* Effect.orDie(client.activity({ name: activity.name, attempt }))
-        // If the activity has suspended and did not execute, we need to resume
-        // it by resetting the attempt and re-executing.
-        if (result._tag === "Suspended" && activities.has(activityId)) {
-          yield* resetActivityAttempt({
-            workflow: instance.workflow,
-            executionId: instance.executionId,
-            activity,
-            attempt
-          })
-          continue
+    activityExecute: Effect.fnUntraced(
+      function*({ activity, attempt }) {
+        const context = yield* Effect.context<WorkflowInstance>()
+        const instance = Context.get(context, WorkflowInstance)
+        yield* Effect.annotateCurrentSpan("executionId", instance.executionId)
+        const activityId = `${instance.executionId}/${activity.name}`
+        activities.set(activityId, { activity, context })
+        const latch = activityLatches.get(activityId)
+        if (latch) {
+          yield* latch.release
+          activityLatches.delete(activityId)
         }
-        activities.delete(activityId)
-        return result
-      }
-    }, Effect.scoped),
+        const client = (yield* RcMap.get(clients, instance.workflow.name))(instance.executionId)
+        while (true) {
+          const result = yield* Effect.orDie(client.activity({ name: activity.name, attempt }))
+          // If the activity has suspended and did not execute, we need to resume
+          // it by resetting the attempt and re-executing.
+          if (result._tag === "Suspended" && activities.has(activityId)) {
+            yield* resetActivityAttempt({
+              workflow: instance.workflow,
+              executionId: instance.executionId,
+              activity,
+              attempt
+            })
+            continue
+          }
+          activities.delete(activityId)
+          return result
+        }
+      },
+      Effect.scoped,
+      (effect, { activity, attempt }) =>
+        Effect.withSpan(effect, "WorkflowEngine.activityExecute", {
+          attributes: {
+            name: activity.name,
+            attempt
+          }
+        })
+    ),
 
     deferredResult: (deferred) =>
       WorkflowInstance.pipe(
+        Effect.tap((instance) => Effect.annotateCurrentSpan("executionId", instance.executionId)),
         Effect.flatMap((instance) =>
           requestReply({
             workflow: instance.workflow,
@@ -338,18 +365,33 @@ export const make = Effect.gen(function*() {
           times: 3,
           schedule: Schedule.exponential(250)
         }),
-        Effect.orDie
+        Effect.orDie,
+        Effect.withSpan("WorkflowEngine.deferredResult", {
+          attributes: {
+            name: deferred.name
+          }
+        })
       ),
 
-    deferredDone: Effect.fnUntraced(function*({ deferred, executionId, exit, workflowName }) {
-      const client = yield* RcMap.get(clients, workflowName)
-      return yield* Effect.orDie(
-        client(executionId).deferred({
-          name: deferred.name,
-          exit
-        }, { discard: true })
-      )
-    }, Effect.scoped),
+    deferredDone: Effect.fnUntraced(
+      function*({ deferred, executionId, exit, workflowName }) {
+        const client = yield* RcMap.get(clients, workflowName)
+        return yield* Effect.orDie(
+          client(executionId).deferred({
+            name: deferred.name,
+            exit
+          }, { discard: true })
+        )
+      },
+      Effect.scoped,
+      (effect, { deferred, executionId }) =>
+        Effect.withSpan(effect, "WorkflowEngine.deferredDone", {
+          attributes: {
+            name: deferred.name,
+            executionId
+          }
+        })
+    ),
 
     scheduleClock(options) {
       const client = clockClient(options.executionId)

package/src/ShardManager.ts

@@ -22,6 +22,7 @@ import * as Iterable from "effect/Iterable"
 import * as Layer from "effect/Layer"
 import * as Mailbox from "effect/Mailbox"
 import * as Metric from "effect/Metric"
+import * as MetricLabel from "effect/MetricLabel"
 import * as MutableHashMap from "effect/MutableHashMap"
 import * as MutableHashSet from "effect/MutableHashSet"
 import * as Option from "effect/Option"
@@ -431,14 +432,21 @@ export const make = Effect.gen(function*() {
   const scope = yield* Effect.scope
   const events = yield* PubSub.unbounded<ShardingEvent>()
 
-  yield* Metric.incrementBy(ClusterMetrics.runners, MutableHashMap.size(state.allRunners))
+  function updateRunnerMetrics() {
+    ClusterMetrics.runners.unsafeUpdate(MutableHashMap.size(state.allRunners), [])
+  }
 
-  for (const [, address] of state.assignments) {
-    const metric = Option.isSome(address) ?
-      Metric.tagged(ClusterMetrics.assignedShards, "address", address.toString()) :
-      ClusterMetrics.unassignedShards
-    yield* Metric.increment(metric)
+  function updateShardMetrics() {
+    const stats = state.shardStats
+    for (const [address, shardCount] of stats.perRunner) {
+      ClusterMetrics.assignedShards.unsafeUpdate(
+        shardCount,
+        [MetricLabel.make("address", address)]
+      )
+    }
+    ClusterMetrics.unassignedShards.unsafeUpdate(stats.unassigned, [])
   }
+  updateShardMetrics()
 
   function withRetry<A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<void, never, R> {
     return effect.pipe(
@@ -493,10 +501,11 @@ export const make = Effect.gen(function*() {
   let machineId = 0
   const register = Effect.fnUntraced(function*(runner: Runner) {
     yield* Effect.logInfo(`Registering runner ${Runner.pretty(runner)}`)
-    state.addRunner(runner, clock.unsafeCurrentTimeMillis())
 
-    yield* Metric.increment(ClusterMetrics.runners)
+    state.addRunner(runner, clock.unsafeCurrentTimeMillis())
+    updateRunnerMetrics()
     yield* PubSub.publish(events, ShardingEvent.RunnerRegistered({ address: runner.address }))
+
     if (state.allUnassignedShards.length > 0) {
       yield* rebalance(false)
     }
@@ -516,13 +525,9 @@ export const make = Effect.gen(function*() {
     }
     state.addAssignments(unassignments, Option.none())
     state.removeRunner(address)
-    yield* Metric.incrementBy(ClusterMetrics.runners, -1)
+    updateRunnerMetrics()
 
     if (unassignments.length > 0) {
-      yield* Metric.incrementBy(
-        Metric.tagged(ClusterMetrics.unassignedShards, "runner_address", address.toString()),
-        unassignments.length
-      )
       yield* PubSub.publish(events, ShardingEvent.RunnerUnregistered({ address }))
     }
 
@@ -639,18 +644,8 @@ export const make = Effect.gen(function*() {
               MutableHashMap.remove(assignments, address)
               return Effect.void
             },
-            onSuccess: () => {
-              const shardCount = MutableHashSet.size(shards)
-              return Metric.incrementBy(
-                Metric.tagged(ClusterMetrics.assignedShards, "runner_address", address.toString()),
-                -shardCount
-              ).pipe(
-                Effect.zipRight(Metric.incrementBy(ClusterMetrics.unassignedShards, shardCount)),
-                Effect.zipRight(
-                  PubSub.publish(events, ShardingEvent.ShardsUnassigned({ address, shards: Array.from(shards) }))
-                )
-              )
-            }
+            onSuccess: () =>
+              PubSub.publish(events, ShardingEvent.ShardsUnassigned({ address, shards: Array.from(shards) }))
           })
         )
       )
@@ -677,24 +672,16 @@ export const make = Effect.gen(function*() {
               MutableHashSet.add(failedRunners, address)
               return Effect.void
             },
-            onSuccess: () => {
-              const shardCount = MutableHashSet.size(shards)
-              return Metric.incrementBy(
-                Metric.tagged(ClusterMetrics.assignedShards, "runner_address", address.toString()),
-                -shardCount
-              ).pipe(
-                Effect.zipRight(Metric.incrementBy(ClusterMetrics.unassignedShards, -shardCount)),
-                Effect.zipRight(
-                  PubSub.publish(events, ShardingEvent.ShardsAssigned({ address, shards: Array.from(shards) }))
-                )
-              )
-            }
+            onSuccess: () =>
+              PubSub.publish(events, ShardingEvent.ShardsAssigned({ address, shards: Array.from(shards) }))
           })
         )
       )
     }
     yield* FiberSet.awaitEmpty(rebalanceFibers)
 
+    updateShardMetrics()
+
     const wereFailures = MutableHashSet.size(failedRunners) > 0
     if (wereFailures) {
       // Check if the failing runners are still reachable

package/src/internal/shardManager.ts

@@ -162,6 +162,25 @@ export class State {
     )
   }
 
+  get shardStats(): {
+    readonly perRunner: Map<string, number>
+    readonly unassigned: number
+  } {
+    const perRunner = new Map<string, number>()
+    let unassigned = 0
+    for (const [, address] of this.assignments) {
+      if (Option.isNone(address)) {
+        unassigned++
+        continue
+      }
+      const runner = address.value.toString()
+      const count = perRunner.get(runner) ?? 0
+      perRunner.set(runner, count + 1)
+    }
+
+    return { perRunner, unassigned }
+  }
+
   shardsPerRunner(group: string): MutableHashMap.MutableHashMap<RunnerAddress, Set<number>> {
     const groupRunners = this.runners.get(group)
     const shards = MutableHashMap.empty<RunnerAddress, Set<number>>()