@effect/cluster 0.37.1 → 0.38.0

package/src/ClusterWorkflowEngine.ts

@@ -2,6 +2,7 @@
  * @since 1.0.0
  */
 import * as Rpc from "@effect/rpc/Rpc"
+import { DurableDeferred } from "@effect/workflow"
 import * as Activity from "@effect/workflow/Activity"
 import * as DurableClock from "@effect/workflow/DurableClock"
 import * as Workflow from "@effect/workflow/Workflow"
@@ -23,9 +24,7 @@ import * as Entity from "./Entity.js"
 import { EntityAddress } from "./EntityAddress.js"
 import { EntityId } from "./EntityId.js"
 import { EntityType } from "./EntityType.js"
-import * as Message from "./Message.js"
 import { MessageStorage } from "./MessageStorage.js"
-import * as Reply from "./Reply.js"
 import type { WithExitEncoded } from "./Reply.js"
 import * as Sharding from "./Sharding.js"
 import * as Snowflake from "./Snowflake.js"
@@ -37,8 +36,8 @@ import * as Snowflake from "./Snowflake.js"
 export const make = Effect.gen(function*() {
   const sharding = yield* Sharding.Sharding
   const storage = yield* MessageStorage
-  const snowflakeGen = yield* Snowflake.Generator
 
+  const workflows = new Map<string, Workflow.Any>()
   const entities = new Map<
     string,
     Entity.Entity<
@@ -69,16 +68,20 @@ export const make = Effect.gen(function*() {
   const deferredClient = yield* DeferredEntity.client
 
   const requestIdFor = Effect.fnUntraced(function*(options: {
+    readonly workflow: Workflow.Any
     readonly entityType: string
     readonly executionId: string
     readonly tag: string
     readonly id: string
   }) {
+    const shardGroup = Context.get(options.workflow.annotations, ClusterSchema.ShardGroup)(
+      options.executionId as EntityId
+    )
     const entityId = EntityId.make(options.executionId)
     const address = new EntityAddress({
       entityType: EntityType.make(options.entityType),
       entityId,
-      shardId: sharding.getShardId(entityId)
+      shardId: sharding.getShardId(entityId, shardGroup)
     })
     return yield* storage.requestIdForPrimaryKey({ address, tag: options.tag, id: options.id })
   })
@@ -94,6 +97,7 @@ export const make = Effect.gen(function*() {
   })
 
   const requestReply = Effect.fnUntraced(function*(options: {
+    readonly workflow: Workflow.Any
     readonly entityType: string
     readonly executionId: string
     readonly tag: string
@@ -114,6 +118,7 @@ export const make = Effect.gen(function*() {
       readonly attempt: number
     }) {
       const requestId = yield* requestIdFor({
+        workflow: options.workflow,
         entityType: `Workflow/${options.workflow.name}`,
         executionId: options.executionId,
         tag: "activity",
@@ -129,13 +134,33 @@ export const make = Effect.gen(function*() {
     Effect.orDie
   )
 
+  const clearClock = Effect.fnUntraced(function*(options: {
+    readonly workflow: Workflow.Any
+    readonly executionId: string
+  }) {
+    const shardGroup = Context.get(options.workflow.annotations, ClusterSchema.ShardGroup)(
+      options.executionId as EntityId
+    )
+    const entityId = EntityId.make(options.executionId)
+    const shardId = sharding.getShardId(entityId, shardGroup)
+    const clockAddress = new EntityAddress({
+      entityType: ClockEntity.type,
+      entityId,
+      shardId
+    })
+    yield* storage.clearAddress(clockAddress)
+  })
+
   return WorkflowEngine.of({
-    register: (workflow, execute) =>
-      Effect.suspend(() => {
+    register(workflow, execute) {
+      // eslint-disable-next-line @typescript-eslint/no-this-alias
+      const engine = this
+      return Effect.suspend(() => {
         if (entities.has(workflow.name)) {
           return Effect.dieMessage(`Workflow ${workflow.name} already registered`)
         }
         const entity = makeWorkflowEntity(workflow)
+        workflows.set(workflow.name, workflow)
         entities.set(workflow.name, entity as any)
         return sharding.registerEntity(
           entity,
@@ -143,18 +168,36 @@ export const make = Effect.gen(function*() {
             const address = yield* Entity.CurrentAddress
             const executionId = address.entityId
             return {
-              run: (request: Entity.Request<any>) =>
-                execute(request.payload, executionId).pipe(
+              run: (request: Entity.Request<any>) => {
+                const instance = WorkflowInstance.of({
+                  workflow,
+                  executionId,
+                  suspended: false
+                })
+                return execute(request.payload, executionId).pipe(
+                  Effect.onExit(() => {
+                    if (!instance.suspended) {
+                      return Effect.void
+                    }
+                    return engine.deferredResult(InterruptSignal).pipe(
+                      Effect.flatMap((maybeResult) => {
+                        if (Option.isNone(maybeResult)) {
+                          return Effect.void
+                        }
+                        instance.suspended = false
+                        return Effect.zipRight(
+                          Effect.ignore(clearClock({ workflow, executionId })),
+                          Effect.interrupt
+                        )
+                      }),
+                      Effect.orDie
+                    )
+                  }),
+                  Effect.scoped,
                   Workflow.intoResult,
-                  Effect.provideService(
-                    WorkflowInstance,
-                    WorkflowInstance.of({
-                      workflow,
-                      executionId,
-                      suspended: false
-                    })
-                  )
-                ) as any,
+                  Effect.provideService(WorkflowInstance, instance)
+                ) as any
+              },
               activity: Effect.fnUntraced(function*(request: Entity.Request<any>) {
                 const activityId = `${executionId}/${request.payload.name}`
                 let entry = activities.get(activityId)
@@ -185,7 +228,8 @@ export const make = Effect.gen(function*() {
             }
           })
         ) as Effect.Effect<void>
-      }),
+      })
+    },
 
     execute: ({ discard, executionId, payload, workflow }) =>
       RcMap.get(clients, workflow.name).pipe(
@@ -195,8 +239,9 @@ export const make = Effect.gen(function*() {
       ),
 
     interrupt: Effect.fnUntraced(
-      function*(workflow, executionId) {
+      function*(this: WorkflowEngine["Type"], workflow, executionId) {
         const requestId = yield* requestIdFor({
+          workflow,
           entityType: `Workflow/${workflow.name}`,
           executionId,
           tag: "run",
@@ -213,41 +258,12 @@ export const make = Effect.gen(function*() {
           return
         }
 
-        const entityId = EntityId.make(executionId)
-        const shardId = sharding.getShardId(entityId)
-        const workflowAddress = new EntityAddress({
-          entityType: EntityType.make(`Workflow/${workflow.name}`),
-          entityId,
-          shardId
-        })
-        const deferredAddress = new EntityAddress({
-          entityType: DeferredEntity.type,
-          entityId,
-          shardId
-        })
-        const clockAddress = new EntityAddress({
-          entityType: ClockEntity.type,
-          entityId,
-          shardId
+        yield* this.deferredDone({
+          workflowName: workflow.name,
+          executionId,
+          deferred: InterruptSignal,
+          exit: { _tag: "Success", value: void 0 }
         })
-        if (Option.isNone(reply)) {
-          yield* sharding.sendOutgoing(
-            Message.OutgoingEnvelope.interrupt({
-              address: workflowAddress,
-              id: snowflakeGen.unsafeNext(),
-              requestId: requestId.value
-            }),
-            true
-          )
-        } else {
-          yield* sharding.reset(requestId.value)
-        }
-        yield* storage.saveReply(Reply.ReplyWithContext.interrupt({
-          id: snowflakeGen.unsafeNext(),
-          requestId: requestId.value
-        }))
-        yield* storage.clearAddress(deferredAddress)
-        yield* storage.clearAddress(clockAddress)
       },
       Effect.retry({
         while: (e) => e._tag === "PersistenceError",
@@ -259,7 +275,12 @@ export const make = Effect.gen(function*() {
 
     resume: Effect.fnUntraced(
       function*(workflowName: string, executionId: string) {
+        const workflow = workflows.get(workflowName)
+        if (!workflow) {
+          return yield* Effect.dieMessage(`WorkflowEngine.resume: ${workflowName} not registered`)
+        }
         const maybeReply = yield* requestReply({
+          workflow,
           entityType: `Workflow/${workflowName}`,
           executionId,
           tag: "run",
@@ -313,6 +334,7 @@ export const make = Effect.gen(function*() {
       WorkflowInstance.pipe(
         Effect.flatMap((instance) =>
           requestReply({
+            workflow: instance.workflow,
             entityType: DeferredEntity.type,
             executionId: instance.executionId,
             tag: "set",
@@ -353,6 +375,17 @@ export const make = Effect.gen(function*() {
   })
 })
 
+const retryPolicy = Schedule.exponential(200, 1.5).pipe(
+  Schedule.union(Schedule.spaced("1 minute"))
+)
+
+const ensureSuccess = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+  effect.pipe(
+    Effect.sandbox,
+    Effect.retry(retryPolicy),
+    Effect.orDie
+  )
+
 const ActivityRpc = Rpc.make("activity", {
   payload: {
     name: Schema.String,
@@ -377,6 +410,7 @@ const makeWorkflowEntity = (workflow: Workflow.Any) =>
     }),
     ActivityRpc
   ])
+    .annotateContext(workflow.annotations)
     .annotateRpcs(ClusterSchema.Persisted, true)
     .annotateRpcs(ClusterSchema.Uninterruptible, true)
 
@@ -417,7 +451,7 @@ const DeferredEntityLayer = DeferredEntity.toLayer(Effect.gen(function*() {
   return {
     set: (request) =>
       Effect.as(
-        Effect.orDie(client.resume(request.payload, { discard: true })),
+        ensureSuccess(client.resume(request.payload, { discard: true })),
         request.payload.exit
       ),
     resume: (request) => engine.resume(request.payload.workflowName, executionId)
@@ -450,16 +484,18 @@ const ClockEntityLayer = ClockEntity.toLayer(Effect.gen(function*() {
   return {
     run(request) {
       const deferred = DurableClock.make({ name: request.payload.name, duration: Duration.zero }).deferred
-      return engine.deferredDone({
+      return ensureSuccess(engine.deferredDone({
         workflowName: request.payload.workflowName,
         executionId,
         deferred,
         exit: { _tag: "Success", value: void 0 }
-      })
+      }))
     }
   }
 }))
 
+const InterruptSignal = DurableDeferred.make("Workflow/InterruptSignal")
+
 /**
  * @since 1.0.0
  * @category Layers
@@ -470,6 +506,5 @@ export const layer: Layer.Layer<
   Sharding.Sharding | MessageStorage
 > = DeferredEntityLayer.pipe(
   Layer.merge(ClockEntityLayer),
-  Layer.provideMerge(Layer.scoped(WorkflowEngine, make)),
-  Layer.provide(Snowflake.layerGenerator)
+  Layer.provideMerge(Layer.scoped(WorkflowEngine, make))
 )

package/src/Entity.ts

@@ -26,6 +26,7 @@ import type {
   MailboxFull,
   PersistenceError
 } from "./ClusterError.js"
+import { ShardGroup } from "./ClusterSchema.js"
 import { EntityAddress } from "./EntityAddress.js"
 import type { EntityId } from "./EntityId.js"
 import { EntityType } from "./EntityType.js"
@@ -68,6 +69,16 @@ export interface Entity<in out Rpcs extends Rpc.Any> extends Equal.Equal {
    */
   readonly protocol: RpcGroup.RpcGroup<Rpcs>
 
+  /**
+   * Get the shard group for the given EntityId.
+   */
+  getShardGroup(entityId: EntityId): string
+
+  /**
+   * Get the ShardId for the given EntityId.
+   */
+  getShardId(entityId: EntityId): Effect.Effect<ShardId.ShardId, never, Sharding>
+
   /**
    * Annotate the entity with a value.
    */
@@ -205,6 +216,9 @@ const Proto = {
   annotateRpcsContext<S>(this: Entity<any>, context: Context.Context<S>) {
     return fromRpcGroup(this.type, this.protocol.annotateRpcsContext(context))
   },
+  getShardId(this: Entity<any>, entityId: EntityId) {
+    return Effect.map(shardingTag, (sharding) => sharding.getShardId(entityId, this.getShardGroup(entityId)))
+  },
   get client() {
     return shardingTag.pipe(
       Effect.flatMap((sharding) => sharding.makeClient(this as any))
@@ -341,6 +355,7 @@ export const fromRpcGroup = <Rpcs extends Rpc.Any>(
   const self = Object.create(Proto)
   self.type = EntityType.make(type)
   self.protocol = protocol
+  self.getShardGroup = Context.get(protocol.annotations, ShardGroup)
   return self
 }
 
@@ -470,7 +485,11 @@ export const makeTestClient: <Rpcs extends Rpc.Any, LA, LE, LR>(
   layer: Layer.Layer<LA, LE, LR>
 ) {
   const config = yield* ShardingConfig
-  const makeShardId = (entityId: string) => ShardId.make((Math.abs(hashString(entityId) % config.numberOfShards)) + 1)
+  const makeShardId = (entityId: string) =>
+    ShardId.make(
+      entity.getShardGroup(entityId as EntityId),
+      (Math.abs(hashString(entityId) % config.shardsPerGroup)) + 1
+    )
   const snowflakeGen = yield* Snowflake.makeGenerator
   const runnerAddress = new RunnerAddress({ host: "localhost", port: 3000 })
   const entityMap = new Map<string, {

package/src/EntityAddress.ts

@@ -1,6 +1,7 @@
 /**
  * @since 1.0.0
  */
+import * as Equal from "effect/Equal"
 import * as Hash from "effect/Hash"
 import * as Schema from "effect/Schema"
 import { EntityId } from "./EntityId.js"
@@ -36,11 +37,20 @@ export class EntityAddress extends Schema.Class<EntityAddress>(SymbolKey)({
    * @since 1.0.0
    */
   readonly [TypeId] = TypeId;
+
+  /**
+   * @since 1.0.0
+   */
+  [Equal.symbol](that: EntityAddress): boolean {
+    return this.entityType === that.entityType && this.entityId === that.entityId &&
+      this.shardId[Equal.symbol](that.shardId)
+  }
+
   /**
    * @since 1.0.0
    */
   [Hash.symbol]() {
-    return Hash.cached(this)(Hash.string(`${this.shardId}:${this.entityType}:${this.entityId}`))
+    return Hash.cached(this, Hash.string(`${this.entityType}:${this.entityId}:${this.shardId.toString()}`))
   }
 }
 

package/src/MessageStorage.ts

@@ -22,7 +22,7 @@ import type { EntityAddress } from "./EntityAddress.js"
 import * as Envelope from "./Envelope.js"
 import * as Message from "./Message.js"
 import * as Reply from "./Reply.js"
-import type { ShardId } from "./ShardId.js"
+import { ShardId } from "./ShardId.js"
 import type { ShardingConfig } from "./ShardingConfig.js"
 import * as Snowflake from "./Snowflake.js"
 
@@ -266,7 +266,7 @@ export type Encoded = {
    * - All Interrupt's for unprocessed requests
    */
   readonly unprocessedMessages: (
-    shardIds: ReadonlyArray<number>,
+    shardIds: ReadonlyArray<string>,
     now: number
   ) => Effect.Effect<
     Array<{
@@ -308,7 +308,7 @@ export type Encoded = {
    * Reset the mailbox state for the provided shards.
    */
   readonly resetShards: (
-    shardIds: ReadonlyArray<number>
+    shardIds: ReadonlyArray<string>
   ) => Effect.Effect<void, PersistenceError>
 }
 
@@ -441,7 +441,9 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
       const shards = Array.from(shardIds)
       if (shards.length === 0) return Effect.succeed([])
       return Effect.flatMap(
-        Effect.suspend(() => encoded.unprocessedMessages(shards, clock.unsafeCurrentTimeMillis())),
+        Effect.suspend(() =>
+          encoded.unprocessedMessages(shards.map((id) => id.toString()), clock.unsafeCurrentTimeMillis())
+        ),
         decodeMessages
       )
     },
@@ -455,7 +457,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
     },
     resetAddress: encoded.resetAddress,
     clearAddress: encoded.clearAddress,
-    resetShards: (shardIds) => encoded.resetShards(Array.from(shardIds))
+    resetShards: (shardIds) => encoded.resetShards(Array.from(shardIds, (id) => id.toString()))
   })
 
   const decodeMessages = (
@@ -666,7 +668,9 @@ export class MemoryDriver extends Effect.Service<MemoryDriver>()("@effect/cluste
           if (existing) {
             return SaveResultEncoded.Duplicate({
               originalId: Snowflake.Snowflake(existing.envelope.requestId),
-              lastReceivedReply: existing.lastReceivedChunk
+              lastReceivedReply: existing.replies.length === 1 && existing.replies[0]._tag === "WithExit"
+                ? Option.some(existing.replies[0])
+                : existing.lastReceivedChunk
             })
           }
           if (envelope._tag === "Request") {
@@ -726,7 +730,8 @@ export class MemoryDriver extends Effect.Service<MemoryDriver>()("@effect/cluste
           let index = journal.indexOf(Iterable.unsafeHead(unprocessed))
           for (; index < journal.length; index++) {
             const envelope = journal[index]
-            if (!shardIds.includes(envelope.address.shardId)) {
+            const shardId = ShardId.make(envelope.address.shardId)
+            if (!shardIds.includes(shardId.toString())) {
               continue
             }
             if (envelope._tag === "Request") {

package/src/Runner.ts

@@ -1,6 +1,8 @@
 /**
  * @since 1.0.0
  */
+import * as Equal from "effect/Equal"
+import * as Hash from "effect/Hash"
 import { NodeInspectSymbol } from "effect/Inspectable"
 import * as Pretty from "effect/Pretty"
 import * as Schema from "effect/Schema"
@@ -35,6 +37,7 @@ export type TypeId = typeof TypeId
  */
 export class Runner extends Schema.Class<Runner>(SymbolKey)({
   address: RunnerAddress,
+  groups: Schema.Array(Schema.String),
   version: Schema.Int
 }) {
   /**
@@ -63,6 +66,20 @@ export class Runner extends Schema.Class<Runner>(SymbolKey)({
   [NodeInspectSymbol](): string {
     return this.toString()
   }
+
+  /**
+   * @since 1.0.0
+   */
+  [Equal.symbol](that: Runner): boolean {
+    return this.address[Equal.symbol](that.address) && this.version === that.version
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  [Hash.symbol](): number {
+    return Hash.cached(this, Hash.string(`${this.address.toString()}:${this.version}`))
+  }
 }
 
 /**
@@ -80,5 +97,6 @@ export class Runner extends Schema.Class<Runner>(SymbolKey)({
  */
 export const make = (props: {
   readonly address: RunnerAddress
+  readonly groups: ReadonlyArray<string>
   readonly version: number
 }): Runner => new Runner(props)

package/src/RunnerAddress.ts

@@ -1,6 +1,7 @@
 /**
  * @since 1.0.0
  */
+import * as Equal from "effect/Equal"
 import * as Hash from "effect/Hash"
 import { NodeInspectSymbol } from "effect/Inspectable"
 import * as Schema from "effect/Schema"
@@ -32,11 +33,18 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(SymbolKey)({
    */
   readonly [TypeId] = TypeId;
 
+  /**
+   * @since 1.0.0
+   */
+  [Equal.symbol](that: RunnerAddress): boolean {
+    return this.host === that.host && this.port === that.port
+  }
+
   /**
    * @since 1.0.0
    */
   [Hash.symbol]() {
-    return Hash.cached(this)(Hash.string(this.toString()))
+    return Hash.cached(this, Hash.string(this.toString()))
   }
 
   /**

package/src/Runners.ts

@@ -164,6 +164,11 @@ export const make: (options: Omit<Runners["Type"], "sendLocal" | "notifyLocal">)
       MessageStorage.SaveResult.$match({
         Success: () => afterPersist(message, false),
         Duplicate: ({ lastReceivedReply, originalId }) => {
+          // If the last received reply is an exit, we can just return it
+          // as the response.
+          if (Option.isSome(lastReceivedReply) && lastReceivedReply.value._tag === "WithExit") {
+            return message.respond(lastReceivedReply.value.withRequestId(message.envelope.requestId))
+          }
           requestIdRewrites.set(message.envelope.requestId, originalId)
           return afterPersist(
             new Message.OutgoingRequest({

package/src/ShardId.ts

@@ -1,27 +1,97 @@
 /**
  * @since 1.0.0
  */
-import * as Schema from "effect/Schema"
+import * as Equal from "effect/Equal"
+import * as Hash from "effect/Hash"
+import * as S from "effect/Schema"
 
 /**
  * @since 1.0.0
- * @category constructors
+ * @category Symbols
  */
-export const ShardId = Schema.Int.pipe(
-  Schema.brand("ShardId"),
-  Schema.annotations({
-    pretty: () => (shardId) => `ShardId(${shardId})`
-  })
-)
+export const TypeId: unique symbol = Symbol.for("@effect/cluster/ShardId")
 
 /**
  * @since 1.0.0
- * @category models
+ * @category Symbols
  */
-export type ShardId = typeof ShardId.Type
+export type TypeId = typeof TypeId
+
+const constDisableValidation = { disableValidation: true }
 
 /**
  * @since 1.0.0
  * @category Constructors
  */
-export const make = (shardId: number): ShardId => ShardId.make(shardId)
+export const make = (group: string, id: number): ShardId => new ShardId({ group, id }, constDisableValidation)
+
+/**
+ * @since 1.0.0
+ * @category Models
+ */
+export class ShardId extends S.Class<ShardId>("@effect/cluster/ShardId")({
+  group: S.String,
+  id: S.Int
+}) {
+  /**
+   * @since 1.0.0
+   */
+  readonly [TypeId]: TypeId = TypeId;
+
+  /**
+   * @since 1.0.0
+   */
+  [Equal.symbol](that: ShardId): boolean {
+    return this.group === that.group && this.id === that.id
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  [Hash.symbol](): number {
+    return Hash.cached(this, Hash.string(this.toString()))
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  toString(): string {
+    return `${this.group}:${this.id}`
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  static toString(shardId: {
+    readonly group: string
+    readonly id: number
+  }): string {
+    return `${shardId.group}:${shardId.id}`
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  static fromStringEncoded(s: string): {
+    readonly group: string
+    readonly id: number
+  } {
+    const index = s.lastIndexOf(":")
+    if (index === -1) {
+      throw new Error(`Invalid ShardId format`)
+    }
+    const group = s.substring(0, index)
+    const id = Number(s.substring(index + 1))
+    if (isNaN(id)) {
+      throw new Error(`ShardId id must be a number`)
+    }
+    return { group, id }
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  static fromString(s: string): ShardId {
+    return new ShardId(ShardId.fromStringEncoded(s), constDisableValidation)
+  }
+}