@effect/cluster 0.28.4 → 0.29.0

package/src/Reply.ts

@@ -0,0 +1,295 @@
+/**
+ * @since 1.0.0
+ */
+import * as Rpc from "@effect/rpc/Rpc"
+import type * as RpcSchema from "@effect/rpc/RpcSchema"
+import type { NonEmptyReadonlyArray } from "effect/Array"
+import * as Context from "effect/Context"
+import * as Data from "effect/Data"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as FiberRef from "effect/FiberRef"
+import { identity } from "effect/Function"
+import type * as Option from "effect/Option"
+import { hasProperty } from "effect/Predicate"
+import * as Schema from "effect/Schema"
+import { MalformedMessage } from "./ClusterError.js"
+import type { OutgoingRequest } from "./Message.js"
+import { Snowflake, SnowflakeFromString } from "./Snowflake.js"
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export const TypeId: unique symbol = Symbol.for("@effect/cluster/Reply")
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export type TypeId = typeof TypeId
+
+/**
+ * @since 1.0.0
+ * @category guards
+ */
+export const isReply = (u: unknown): u is Reply<Rpc.Any> => hasProperty(u, TypeId)
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export type Reply<R extends Rpc.Any> = WithExit<R> | Chunk<R>
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("ReplyWithContext")<{
+  readonly reply: Reply<R>
+  readonly context: Context.Context<Rpc.Context<R>>
+  readonly rpc: R
+}> {
+  /**
+   * @since 1.0.0
+   */
+  static fromDefect(options: {
+    readonly id: Snowflake
+    readonly requestId: Snowflake
+    readonly defect: unknown
+  }): ReplyWithContext<any> {
+    return new ReplyWithContext({
+      reply: new WithExit({
+        requestId: options.requestId,
+        id: options.id,
+        exit: Exit.die(Schema.encodeSync(Schema.Defect)(options.defect))
+      }),
+      context: Context.empty() as any,
+      rpc: neverRpc
+    })
+  }
+}
+
+const neverRpc = Rpc.make("Never", {
+  success: Schema.Never as any,
+  error: Schema.Never,
+  payload: {}
+})
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export type ReplyEncoded<R extends Rpc.Any> = WithExitEncoded<R> | ChunkEncoded<R>
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export interface WithExitEncoded<R extends Rpc.Any> {
+  readonly _tag: "WithExit"
+  readonly requestId: string
+  readonly id: string
+  readonly exit: Rpc.ExitEncoded<R>
+}
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export interface ChunkEncoded<R extends Rpc.Any> {
+  readonly _tag: "Chunk"
+  readonly requestId: string
+  readonly id: string
+  readonly sequence: number
+  readonly values: NonEmptyReadonlyArray<Rpc.SuccessChunkEncoded<R>>
+}
+
+const schemaCache = new WeakMap<Rpc.Any, Schema.Schema<Reply<Rpc.Any>, ReplyEncoded<Rpc.Any>, Rpc.Context<Rpc.Any>>>()
+
+/**
+ * @since 1.0.0
+ * @category schemas
+ */
+export const Reply = <R extends Rpc.Any>(rpc: R): Schema.Schema<
+  Reply<R>,
+  ReplyEncoded<R>,
+  Rpc.Context<R>
+> => {
+  if (schemaCache.has(rpc)) {
+    return schemaCache.get(rpc) as any
+  }
+  const schema = Schema.Union(WithExit.schema(rpc), Chunk.schema(rpc))
+  schemaCache.set(rpc, schema)
+  return schema
+}
+
+/**
+ * @since 1.0.0
+ * @category schemas
+ */
+export const Encoded = Schema.Union(
+  Schema.Struct({
+    _tag: Schema.Literal("WithExit"),
+    requestId: Schema.String,
+    id: Schema.String,
+    exit: Schema.Unknown
+  }),
+  Schema.Struct({
+    _tag: Schema.Literal("Chunk"),
+    requestId: Schema.String,
+    id: Schema.String,
+    sequence: Schema.Number,
+    values: Schema.Array(Schema.Unknown)
+  })
+)
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
+  readonly requestId: Snowflake
+  readonly id: Snowflake
+  readonly sequence: number
+  readonly values: NonEmptyReadonlyArray<Rpc.SuccessChunk<R>>
+}> {
+  /**
+   * @since 1.0.0
+   */
+  readonly [TypeId] = TypeId
+
+  /**
+   * @since 1.0.0
+   */
+  static emptyFrom(requestId: Snowflake) {
+    return new Chunk({
+      requestId,
+      id: Snowflake(BigInt(0)),
+      sequence: 0,
+      values: [undefined]
+    })
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  static readonly schemaFromSelf: Schema.Schema<Chunk<never>> = Schema.declare((u): u is Chunk<never> =>
+    isReply(u) && u._tag === "Chunk"
+  )
+
+  /**
+   * @since 1.0.0
+   */
+  static schema<R extends Rpc.Any>(rpc: R): Schema.Schema<
+    Chunk<R>,
+    ChunkEncoded<R>,
+    Rpc.Context<R>
+  > {
+    const successSchema = ((rpc as any as Rpc.AnyWithProps).successSchema as RpcSchema.Stream<any, any>).success
+    if (!successSchema) {
+      return Schema.Never as any
+    }
+    return Schema.transform(
+      Schema.Struct({
+        _tag: Schema.Literal("Chunk"),
+        requestId: SnowflakeFromString,
+        id: SnowflakeFromString,
+        sequence: Schema.Number,
+        values: Schema.NonEmptyArray(successSchema)
+      }),
+      Chunk.schemaFromSelf,
+      {
+        decode: (encoded) => new Chunk(encoded as any),
+        encode: identity
+      }
+    ) as any
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  withRequestId(requestId: Snowflake): Chunk<R> {
+    return new Chunk({
+      ...this,
+      requestId
+    })
+  }
+}
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
+  readonly requestId: Snowflake
+  readonly id: Snowflake
+  readonly exit: Rpc.Exit<R>
+}> {
+  /**
+   * @since 1.0.0
+   */
+  readonly [TypeId] = TypeId
+
+  /**
+   * @since 1.0.0
+   */
+  static schema<R extends Rpc.Any>(rpc: R): Schema.Schema<
+    WithExit<R>,
+    WithExitEncoded<R>,
+    Rpc.Context<R>
+  > {
+    return Schema.transform(
+      Schema.Struct({
+        _tag: Schema.Literal("WithExit"),
+        requestId: SnowflakeFromString,
+        id: SnowflakeFromString,
+        exit: Rpc.exitSchema(rpc)
+      }),
+      Schema.declare((u): u is WithExit<R> => isReply(u) && u._tag === "WithExit"),
+      {
+        decode: (encoded) => new WithExit(encoded),
+        encode: identity
+      }
+    ) as any
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  withRequestId(requestId: Snowflake): WithExit<R> {
+    return new WithExit({
+      ...this,
+      requestId
+    })
+  }
+}
+
+/**
+ * @since 1.0.0
+ * @category serialization / deserialization
+ */
+export const serialize = <R extends Rpc.Any>(
+  self: ReplyWithContext<R>
+): Effect.Effect<ReplyEncoded<R>, MalformedMessage> => {
+  const schema = Reply(self.rpc)
+  return MalformedMessage.refail(
+    Effect.locally(Schema.encode(schema)(self.reply), FiberRef.currentContext, self.context)
+  )
+}
+
+/**
+ * @since 1.0.0
+ * @category serialization / deserialization
+ */
+export const serializeLastReceived = <R extends Rpc.Any>(
+  self: OutgoingRequest<R>
+): Effect.Effect<Option.Option<ReplyEncoded<R>>, MalformedMessage> => {
+  if (self.lastReceivedReply._tag === "None") {
+    return Effect.succeedNone
+  }
+  const schema = Reply(self.rpc)
+  return Effect.asSome(MalformedMessage.refail(
+    Effect.locally(Schema.encode(schema)(self.lastReceivedReply.value), FiberRef.currentContext, self.context)
+  ))
+}

package/src/Runner.ts

@@ -0,0 +1,84 @@
+/**
+ * @since 1.0.0
+ */
+import { NodeInspectSymbol } from "effect/Inspectable"
+import * as Pretty from "effect/Pretty"
+import * as Schema from "effect/Schema"
+import { RunnerAddress } from "./RunnerAddress.js"
+
+const SymbolKey = "@effect/cluster/Runner"
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export const TypeId: unique symbol = Symbol.for(SymbolKey)
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export type TypeId = typeof TypeId
+
+/**
+ * A `Runner` represents a physical application server that is capable of running
+ * entities.
+ *
+ * Because a Runner represents a physical application server, a Runner must have a
+ * unique `address` which can be used to communicate with the server.
+ *
+ * The version of a Runner is used during rebalancing to give priority to newer
+ * application servers and slowly decommission older ones.
+ *
+ * @since 1.0.0
+ * @category models
+ */
+export class Runner extends Schema.Class<Runner>(SymbolKey)({
+  address: RunnerAddress,
+  version: Schema.Int
+}) {
+  /**
+   * @since 1.0.0
+   */
+  static pretty = Pretty.make(this)
+
+  /**
+   * @since 1.0.0
+   */
+  readonly [TypeId] = TypeId
+
+  /**
+   * @since 1.0.0
+   */
+  static readonly decodeSync = Schema.decodeSync(Schema.parseJson(Runner))
+
+  /**
+   * @since 1.0.0
+   */
+  static readonly encodeSync = Schema.encodeSync(Schema.parseJson(Runner));
+
+  /**
+   * @since 1.0.0
+   */
+  [NodeInspectSymbol](): string {
+    return this.toString()
+  }
+}
+
+/**
+ * A `Runner` represents a physical application server that is capable of running
+ * entities.
+ *
+ * Because a Runner represents a physical application server, a Runner must have a
+ * unique `address` which can be used to communicate with the server.
+ *
+ * The version of a Runner is used during rebalancing to give priority to newer
+ * application servers and slowly decommission older ones.
+ *
+ * @since 1.0.0
+ * @category Constructors
+ */
+export const make = (props: {
+  readonly address: RunnerAddress
+  readonly version: number
+}): Runner => new Runner(props)

package/src/RunnerAddress.ts

@@ -0,0 +1,61 @@
+/**
+ * @since 1.0.0
+ */
+import * as Hash from "effect/Hash"
+import { NodeInspectSymbol } from "effect/Inspectable"
+import * as Schema from "effect/Schema"
+
+const SymbolKey = "@effect/cluster/RunnerAddress"
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export const TypeId: unique symbol = Symbol.for(SymbolKey)
+
+/**
+ * @since 1.0.0
+ * @category type ids
+ */
+export type TypeId = typeof TypeId
+
+/**
+ * @since 1.0.0
+ * @category models
+ */
+export class RunnerAddress extends Schema.Class<RunnerAddress>(SymbolKey)({
+  host: Schema.NonEmptyString,
+  port: Schema.Int
+}) {
+  /**
+   * @since 1.0.0
+   */
+  readonly [TypeId] = TypeId;
+
+  /**
+   * @since 1.0.0
+   */
+  [Hash.symbol]() {
+    return Hash.cached(this)(Hash.string(this.toString()))
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  toString(): string {
+    return `RunnerAddress(${this.host}:${this.port})`
+  }
+
+  /**
+   * @since 1.0.0
+   */
+  [NodeInspectSymbol](): string {
+    return this.toString()
+  }
+}
+
+/**
+ * @since 1.0.0
+ * @category constructors
+ */
+export const make = (host: string, port: number): RunnerAddress => new RunnerAddress({ host, port })

package/src/RunnerHealth.ts

@@ -0,0 +1,87 @@
+/**
+ * @since 1.0.0
+ */
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as MessageStorage from "./MessageStorage.js"
+import type { RunnerAddress } from "./RunnerAddress.js"
+import * as Runners from "./Runners.js"
+import type { ShardingConfig } from "./ShardingConfig.js"
+
+/**
+ * Represents the service used to check if a Runner is healthy.
+ *
+ * If a Runner is responsive, shards will not be re-assigned because the Runner may
+ * still be processing messages. If a Runner is not responsive, then its
+ * associated shards can and will be re-assigned to a different Runner.
+ *
+ * @since 1.0.0
+ * @category models
+ */
+export class RunnerHealth extends Context.Tag("@effect/cluster/RunnerHealth")<
+  RunnerHealth,
+  {
+    readonly isAlive: (address: RunnerAddress) => Effect.Effect<boolean>
+  }
+>() {}
+
+/**
+ * A layer which will **always** consider a Runner healthy.
+ *
+ * This is useful for testing.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerNoop = Layer.succeed(
+  RunnerHealth,
+  RunnerHealth.of({
+    isAlive: () => Effect.succeed(true)
+  })
+)
+
+/**
+ * @since 1.0.0
+ * @category Constructors
+ */
+export const make: Effect.Effect<
+  RunnerHealth["Type"],
+  never,
+  Runners.Runners
+> = Effect.gen(function*() {
+  const runners = yield* Runners.Runners
+
+  function isAlive(address: RunnerAddress): Effect.Effect<boolean> {
+    return Effect.isSuccess(Effect.timeout(runners.ping(address), 3000))
+  }
+
+  return RunnerHealth.of({ isAlive })
+})
+
+/**
+ * A layer which will ping a Runner directly to check if it is healthy.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layer: Layer.Layer<
+  RunnerHealth,
+  never,
+  Runners.Runners
+> = Layer.effect(RunnerHealth, make)
+
+/**
+ * A layer which will ping a Runner directly to check if it is healthy.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerRpc: Layer.Layer<
+  RunnerHealth,
+  never,
+  Runners.RpcClientProtocol | ShardingConfig
+> = layer.pipe(
+  Layer.provide(Runners.layerRpc),
+  Layer.provide(MessageStorage.layerNoop)
+)

package/src/RunnerServer.ts

@@ -0,0 +1,156 @@
+/**
+ * @since 1.0.0
+ */
+import * as RpcServer from "@effect/rpc/RpcServer"
+import * as Effect from "effect/Effect"
+import { constant } from "effect/Function"
+import * as Layer from "effect/Layer"
+import * as Mailbox from "effect/Mailbox"
+import * as Option from "effect/Option"
+import * as Message from "./Message.js"
+import type * as MessageStorage from "./MessageStorage.js"
+import * as Reply from "./Reply.js"
+import * as Runners from "./Runners.js"
+import * as Sharding from "./Sharding.js"
+import { ShardingConfig } from "./ShardingConfig.js"
+import * as ShardManager from "./ShardManager.js"
+import * as ShardStorage from "./ShardStorage.js"
+import * as SynchronizedClock from "./SynchronizedClock.js"
+
+const constVoid = constant(Effect.void)
+
+/**
+ * @since 1.0.0
+ * @category Layers
+ */
+export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
+  const sharding = yield* Sharding.Sharding
+
+  return {
+    Ping: () => Effect.void,
+    Notify: ({ envelope }) =>
+      sharding.notify(
+        envelope._tag === "Request"
+          ? new Message.IncomingRequest({
+            envelope,
+            respond: constVoid,
+            lastSentReply: Option.none()
+          })
+          : new Message.IncomingEnvelope({ envelope })
+      ),
+    Effect: ({ request }) => {
+      let resume: (reply: Effect.Effect<Reply.ReplyEncoded<any>>) => void
+      let replyEncoded: Reply.ReplyEncoded<any> | undefined
+      const message = new Message.IncomingRequest({
+        envelope: request,
+        lastSentReply: Option.none(),
+        respond(reply) {
+          return Effect.flatMap(Reply.serialize(reply), (reply) => {
+            if (resume) {
+              resume(Effect.succeed(reply))
+            } else {
+              replyEncoded = reply
+            }
+            return Effect.void
+          })
+        }
+      })
+      return Effect.zipRight(
+        sharding.send(message),
+        Effect.async<Reply.ReplyEncoded<any>>((resume_) => {
+          if (replyEncoded) {
+            resume_(Effect.succeed(replyEncoded))
+          } else {
+            resume = resume_
+          }
+        })
+      )
+    },
+    Stream: ({ request }) =>
+      Effect.flatMap(
+        Mailbox.make<Reply.ReplyEncoded<any>>(),
+        (mailbox) =>
+          Effect.as(
+            sharding.send(
+              new Message.IncomingRequest({
+                envelope: request,
+                lastSentReply: Option.none(),
+                respond(reply) {
+                  return Effect.flatMap(Reply.serialize(reply), (reply) => {
+                    mailbox.unsafeOffer(reply)
+                    return Effect.void
+                  })
+                }
+              })
+            ),
+            mailbox
+          )
+      ),
+    Envelope: ({ envelope }) => sharding.send(new Message.IncomingEnvelope({ envelope }))
+  }
+}))
+
+/**
+ * The `RunnerServer` recieves messages from other Runners and forwards them to the
+ * `Sharding` layer.
+ *
+ * It also responds to `Ping` requests.
+ *
+ * @since 1.0.0
+ * @category Layers
+ */
+export const layer: Layer.Layer<
+  never,
+  never,
+  RpcServer.Protocol | Sharding.Sharding | MessageStorage.MessageStorage
+> = RpcServer.layer(Runners.Rpcs, {
+  spanPrefix: "RunnerServer",
+  disableTracing: true
+}).pipe(Layer.provide(layerHandlers))
+
+/**
+ * A `RunnerServer` layer that includes the `Runners` & `Sharding` clients.
+ *
+ * @since 1.0.0
+ * @category Layers
+ */
+export const layerWithClients: Layer.Layer<
+  Sharding.Sharding | Runners.Runners,
+  never,
+  | RpcServer.Protocol
+  | ShardingConfig
+  | Runners.RpcClientProtocol
+  | MessageStorage.MessageStorage
+  | ShardStorage.ShardStorage
+> = layer.pipe(
+  Layer.provideMerge(Sharding.layer),
+  Layer.provideMerge(Runners.layerRpc),
+  Layer.provideMerge(SynchronizedClock.layer),
+  Layer.provide(ShardManager.layerClientRpc)
+)
+
+/**
+ * A `Runners` layer that is client only.
+ *
+ * It will not register with the ShardManager and recieve shard assignments,
+ * so this layer can be used to embed a cluster client inside another effect
+ * application.
+ *
+ * @since 1.0.0
+ * @category Layers
+ */
+export const layerClientOnly: Layer.Layer<
+  Sharding.Sharding | Runners.Runners,
+  never,
+  | ShardingConfig
+  | Runners.RpcClientProtocol
+  | MessageStorage.MessageStorage
+> = Sharding.layer.pipe(
+  Layer.provideMerge(Runners.layerRpc),
+  Layer.provide(ShardManager.layerClientRpc),
+  Layer.provide(ShardStorage.layerNoop),
+  Layer.updateService(ShardingConfig, (config) => ({
+    ...config,
+    runnerAddress: Option.none()
+  }))
+)