@effect/cluster 0.50.6 → 0.52.0

package/src/RunnerHealth.ts

@@ -1,15 +1,19 @@
 /**
  * @since 1.0.0
  */
+import * as FileSystem from "@effect/platform/FileSystem"
+import * as HttpClient from "@effect/platform/HttpClient"
+import * as HttpClientRequest from "@effect/platform/HttpClientRequest"
+import * as HttpClientResponse from "@effect/platform/HttpClientResponse"
 import * as Context from "effect/Context"
 import * as Effect from "effect/Effect"
+import { identity } from "effect/Function"
 import * as Layer from "effect/Layer"
-import * as RcMap from "effect/RcMap"
+import * as Schedule from "effect/Schedule"
+import * as Schema from "effect/Schema"
 import type * as Scope from "effect/Scope"
-import * as MessageStorage from "./MessageStorage.js"
 import type { RunnerAddress } from "./RunnerAddress.js"
 import * as Runners from "./Runners.js"
-import type { ShardingConfig } from "./ShardingConfig.js"
 
 /**
  * Represents the service used to check if a Runner is healthy.
@@ -24,46 +28,10 @@ import type { ShardingConfig } from "./ShardingConfig.js"
 export class RunnerHealth extends Context.Tag("@effect/cluster/RunnerHealth")<
   RunnerHealth,
   {
-    /**
-     * Used to indicate that a Runner is connected to this host and is healthy,
-     * while the Scope is active.
-     */
-    readonly onConnection: (address: RunnerAddress) => Effect.Effect<void, never, Scope.Scope>
     readonly isAlive: (address: RunnerAddress) => Effect.Effect<boolean>
   }
 >() {}
 
-/**
- * @since 1.0.0
- * @category Constructors
- */
-export const make: (
-  options: { readonly isAlive: (address: RunnerAddress) => Effect.Effect<boolean> }
-) => Effect.Effect<
-  RunnerHealth["Type"],
-  never,
-  Scope.Scope
-> = Effect.fnUntraced(function*(options: {
-  readonly isAlive: (address: RunnerAddress) => Effect.Effect<boolean>
-}) {
-  const connections = yield* RcMap.make({
-    lookup: (_address: RunnerAddress) => Effect.void
-  })
-
-  const onConnection = (address: RunnerAddress) => RcMap.get(connections, address)
-  const isAlive = Effect.fnUntraced(function*(address: RunnerAddress) {
-    if (yield* RcMap.has(connections, address)) {
-      return true
-    }
-    return yield* options.isAlive(address)
-  })
-
-  return RunnerHealth.of({
-    onConnection,
-    isAlive
-  })
-})
-
 /**
  * A layer which will **always** consider a Runner healthy.
  *
@@ -72,12 +40,9 @@ export const make: (
  * @since 1.0.0
  * @category layers
  */
-export const layerNoop = Layer.scoped(
-  RunnerHealth,
-  make({
-    isAlive: () => Effect.succeed(true)
-  })
-)
+export const layerNoop = Layer.succeed(RunnerHealth, {
+  isAlive: () => Effect.succeed(true)
+})
 
 /**
  * @since 1.0.0
@@ -89,16 +54,17 @@ export const makePing: Effect.Effect<
   Runners.Runners | Scope.Scope
 > = Effect.gen(function*() {
   const runners = yield* Runners.Runners
+  const schedule = Schedule.spaced(500)
 
   function isAlive(address: RunnerAddress): Effect.Effect<boolean> {
     return runners.ping(address).pipe(
-      Effect.timeout(3000),
-      Effect.retry({ times: 3 }),
+      Effect.timeout(10_000),
+      Effect.retry({ times: 5, schedule }),
       Effect.isSuccess
     )
   }
 
-  return yield* make({ isAlive })
+  return RunnerHealth.of({ isAlive })
 })
 
 /**
@@ -107,23 +73,120 @@ export const makePing: Effect.Effect<
  * @since 1.0.0
  * @category layers
  */
-export const layer: Layer.Layer<
+export const layerPing: Layer.Layer<
   RunnerHealth,
   never,
   Runners.Runners
 > = Layer.scoped(RunnerHealth, makePing)
 
 /**
- * A layer which will ping a Runner directly to check if it is healthy.
+ * @since 1.0.0
+ * @category Constructors
+ */
+export const makeK8s = Effect.fnUntraced(function*(options?: {
+  readonly namespace?: string | undefined
+  readonly labelSelector?: string | undefined
+}) {
+  const fs = yield* FileSystem.FileSystem
+  const token = yield* fs.readFileString("/var/run/secrets/kubernetes.io/serviceaccount/token").pipe(
+    Effect.option
+  )
+  const client = (yield* HttpClient.HttpClient).pipe(
+    HttpClient.filterStatusOk
+  )
+  const baseRequest = HttpClientRequest.get("https://kubernetes.default.svc/api").pipe(
+    token._tag === "Some" ? HttpClientRequest.bearerToken(token.value.trim()) : identity
+  )
+  const getPods = baseRequest.pipe(
+    HttpClientRequest.appendUrl(options?.namespace ? `/v1/namespaces/${options.namespace}/pods` : "/v1/pods"),
+    HttpClientRequest.setUrlParam("fieldSelector", "status.phase=Running"),
+    options?.labelSelector ? HttpClientRequest.setUrlParam("labelSelector", options.labelSelector) : identity
+  )
+  const allPods = yield* client.execute(getPods).pipe(
+    Effect.flatMap(HttpClientResponse.schemaBodyJson(PodList)),
+    Effect.map((list) => {
+      const pods = new Map<string, Pod>()
+      for (let i = 0; i < list.items.length; i++) {
+        const pod = list.items[i]
+        pods.set(pod.status.podIP, pod)
+      }
+      return pods
+    }),
+    Effect.tapErrorCause((cause) => Effect.logWarning("Failed to fetch pods from Kubernetes API", cause)),
+    Effect.cachedWithTTL("10 seconds")
+  )
+
+  return RunnerHealth.of({
+    isAlive: (address) =>
+      allPods.pipe(
+        Effect.map((pods) => pods.get(address.host)?.isReady ?? false),
+        Effect.catchAllCause(() => Effect.succeed(true))
+      )
+  })
+})
+
+class Pod extends Schema.Class<Pod>("effect/cluster/RunnerHealth/Pod")({
+  status: Schema.Struct({
+    phase: Schema.String,
+    conditions: Schema.Array(Schema.Struct({
+      type: Schema.String,
+      status: Schema.String,
+      lastTransitionTime: Schema.String
+    })),
+    podIP: Schema.String
+  })
+}) {
+  get isReady(): boolean {
+    let initializedAt: string | undefined
+    let readyAt: string | undefined
+    for (let i = 0; i < this.status.conditions.length; i++) {
+      const condition = this.status.conditions[i]
+      switch (condition.type) {
+        case "Initialized": {
+          if (condition.status !== "True") {
+            return true
+          }
+          initializedAt = condition.lastTransitionTime
+          break
+        }
+        case "Ready": {
+          if (condition.status === "True") {
+            return true
+          }
+          readyAt = condition.lastTransitionTime
+          break
+        }
+      }
+    }
+    // if the pod is still booting up, consider it ready as it would have
+    // already registered itself with RunnerStorage by now
+    return initializedAt === readyAt
+  }
+}
+
+const PodList = Schema.Struct({
+  items: Schema.Array(Pod)
+})
+
+/**
+ * A layer which will check the Kubernetes API to see if a Runner is healthy.
+ *
+ * The provided HttpClient will need to add the pod's CA certificate to its
+ * trusted root certificates in order to communicate with the Kubernetes API.
+ *
+ * The pod service account will also need to have permissions to list pods in
+ * order to use this layer.
  *
  * @since 1.0.0
  * @category layers
  */
-export const layerRpc: Layer.Layer<
+export const layerK8s = (
+  options?: {
+    readonly namespace?: string | undefined
+    readonly labelSelector?: string | undefined
+  } | undefined
+): Layer.Layer<
   RunnerHealth,
   never,
-  Runners.RpcClientProtocol | ShardingConfig
-> = layer.pipe(
-  Layer.provide(Runners.layerRpc),
-  Layer.provide(MessageStorage.layerNoop)
-)
+  HttpClient.HttpClient | FileSystem.FileSystem
+> => Layer.effect(RunnerHealth, makeK8s(options))

package/src/RunnerServer.ts

@@ -3,20 +3,22 @@
  */
 import * as RpcServer from "@effect/rpc/RpcServer"
 import * as Effect from "effect/Effect"
+import type * as Exit from "effect/Exit"
+import * as Fiber from "effect/Fiber"
 import { constant } from "effect/Function"
 import * as Layer from "effect/Layer"
 import * as Mailbox from "effect/Mailbox"
 import * as Option from "effect/Option"
-import * as ClusterError from "./ClusterError.js"
+import * as Runtime from "effect/Runtime"
+import type * as ClusterError from "./ClusterError.js"
 import * as Message from "./Message.js"
 import * as MessageStorage from "./MessageStorage.js"
 import * as Reply from "./Reply.js"
+import * as RunnerHealth from "./RunnerHealth.js"
 import * as Runners from "./Runners.js"
+import type * as RunnerStorage from "./RunnerStorage.js"
 import * as Sharding from "./Sharding.js"
 import { ShardingConfig } from "./ShardingConfig.js"
-import * as ShardManager from "./ShardManager.js"
-import * as ShardStorage from "./ShardStorage.js"
-import * as SynchronizedClock from "./SynchronizedClock.js"
 
 const constVoid = constant(Effect.void)
 
@@ -41,41 +43,60 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
           : new Message.IncomingEnvelope({ envelope })
       ),
     Effect: ({ persisted, request }) => {
-      let resume: (reply: Effect.Effect<Reply.ReplyEncoded<any>, ClusterError.EntityNotAssignedToRunner>) => void
-      let replyEncoded: Reply.ReplyEncoded<any> | undefined
+      let replyEncoded:
+        | Effect.Effect<
+          Reply.ReplyEncoded<any>,
+          ClusterError.EntityNotAssignedToRunner
+        >
+        | undefined = undefined
+      let resume = (reply: Effect.Effect<Reply.ReplyEncoded<any>, ClusterError.EntityNotAssignedToRunner>) => {
+        replyEncoded = reply
+      }
       const message = new Message.IncomingRequest({
         envelope: request,
         lastSentReply: Option.none(),
         respond(reply) {
-          return Effect.flatMap(Reply.serialize(reply), (reply) => {
-            if (resume) {
-              resume(Effect.succeed(reply))
-            } else {
-              replyEncoded = reply
-            }
-            return Effect.void
-          })
+          resume(Effect.orDie(Reply.serialize(reply)))
+          return Effect.void
         }
       })
+      if (persisted) {
+        return Effect.async<
+          Reply.ReplyEncoded<any>,
+          ClusterError.EntityNotAssignedToRunner
+        >((resume_) => {
+          resume = resume_
+          const parent = Option.getOrThrow(Fiber.getCurrentFiber())
+          const runtime = Runtime.make({
+            context: parent.currentContext,
+            runtimeFlags: Runtime.defaultRuntimeFlags,
+            fiberRefs: parent.getFiberRefs()
+          })
+          const onExit = (
+            exit: Exit.Exit<
+              any,
+              ClusterError.EntityNotAssignedToRunner
+            >
+          ) => {
+            if (exit._tag === "Failure") {
+              resume(exit as any)
+            }
+          }
+          const fiber = Runtime.runFork(runtime)(storage.registerReplyHandler(message))
+          fiber.addObserver(onExit)
+          Runtime.runFork(runtime)(Effect.catchTag(
+            sharding.notify(message, constWaitUntilRead),
+            "AlreadyProcessingMessage",
+            () => Effect.void
+          )).addObserver(onExit)
+          return Fiber.interrupt(fiber)
+        })
+      }
       return Effect.zipRight(
-        persisted ?
-          Effect.zipRight(
-            storage.registerReplyHandler(
-              message,
-              Effect.sync(() =>
-                resume(Effect.fail(
-                  new ClusterError.EntityNotAssignedToRunner({
-                    address: request.address
-                  })
-                ))
-              )
-            ),
-            sharding.notify(message)
-          ) :
-          sharding.send(message),
+        sharding.send(message),
         Effect.async<Reply.ReplyEncoded<any>, ClusterError.EntityNotAssignedToRunner>((resume_) => {
           if (replyEncoded) {
-            resume_(Effect.succeed(replyEncoded))
+            resume_(replyEncoded)
           } else {
             resume = resume_
           }
@@ -99,17 +120,12 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
           return Effect.as(
             persisted ?
               Effect.zipRight(
-                storage.registerReplyHandler(
-                  message,
-                  Effect.suspend(() =>
-                    mailbox.fail(
-                      new ClusterError.EntityNotAssignedToRunner({
-                        address: request.address
-                      })
-                    )
-                  )
+                storage.registerReplyHandler(message).pipe(
+                  Effect.onError((cause) => mailbox.failCause(cause)),
+                  Effect.forkScoped,
+                  Effect.interruptible
                 ),
-                sharding.notify(message)
+                sharding.notify(message, constWaitUntilRead)
               ) :
               sharding.send(message),
             mailbox
@@ -120,6 +136,8 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
   }
 }))
 
+const constWaitUntilRead = { waitUntilRead: true } as const
+
 /**
  * The `RunnerServer` recieves messages from other Runners and forwards them to the
  * `Sharding` layer.
@@ -151,18 +169,17 @@ export const layerWithClients: Layer.Layer<
   | ShardingConfig
   | Runners.RpcClientProtocol
   | MessageStorage.MessageStorage
-  | ShardStorage.ShardStorage
+  | RunnerStorage.RunnerStorage
+  | RunnerHealth.RunnerHealth
 > = layer.pipe(
   Layer.provideMerge(Sharding.layer),
-  Layer.provideMerge(Runners.layerRpc),
-  Layer.provideMerge(SynchronizedClock.layer),
-  Layer.provide(ShardManager.layerClientRpc)
+  Layer.provideMerge(Runners.layerRpc)
 )
 
 /**
  * A `Runners` layer that is client only.
  *
- * It will not register with the ShardManager and recieve shard assignments,
+ * It will not register with RunnerStorage and recieve shard assignments,
  * so this layer can be used to embed a cluster client inside another effect
  * application.
  *
@@ -175,10 +192,10 @@ export const layerClientOnly: Layer.Layer<
   | ShardingConfig
   | Runners.RpcClientProtocol
   | MessageStorage.MessageStorage
+  | RunnerStorage.RunnerStorage
 > = Sharding.layer.pipe(
   Layer.provideMerge(Runners.layerRpc),
-  Layer.provide(ShardManager.layerClientRpc),
-  Layer.provide(ShardStorage.layerNoop),
+  Layer.provide(RunnerHealth.layerNoop),
   Layer.updateService(ShardingConfig, (config) => ({
     ...config,
     runnerAddress: Option.none()

package/src/RunnerStorage.ts

@@ -0,0 +1,218 @@
+/**
+ * @since 1.0.0
+ */
+import { isNonEmptyArray, type NonEmptyArray } from "effect/Array"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as MutableHashMap from "effect/MutableHashMap"
+import type { PersistenceError } from "./ClusterError.js"
+import * as MachineId from "./MachineId.js"
+import { Runner } from "./Runner.js"
+import type { RunnerAddress } from "./RunnerAddress.js"
+import { ShardId } from "./ShardId.js"
+
+/**
+ * Represents a generic interface to the persistent storage required by the
+ * cluster.
+ *
+ * @since 1.0.0
+ * @category models
+ */
+export class RunnerStorage extends Context.Tag("@effect/cluster/RunnerStorage")<RunnerStorage, {
+  /**
+   * Register a new runner with the cluster.
+   */
+  readonly register: (runner: Runner, healthy: boolean) => Effect.Effect<MachineId.MachineId, PersistenceError>
+
+  /**
+   * Unregister the runner with the given address.
+   */
+  readonly unregister: (address: RunnerAddress) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Get all runners registered with the cluster.
+   */
+  readonly getRunners: Effect.Effect<Array<readonly [runner: Runner, healthy: boolean]>, PersistenceError>
+
+  /**
+   * Set the health status of the given runner.
+   */
+  readonly setRunnerHealth: (address: RunnerAddress, healthy: boolean) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Try to acquire the given shard ids for processing.
+   *
+   * It returns an array of shards it was able to acquire.
+   */
+  readonly acquire: (
+    address: RunnerAddress,
+    shardIds: Iterable<ShardId>
+  ) => Effect.Effect<Array<ShardId>, PersistenceError>
+
+  /**
+   * Refresh the locks owned by the given runner.
+   */
+  readonly refresh: (
+    address: RunnerAddress,
+    shardIds: Iterable<ShardId>
+  ) => Effect.Effect<Array<ShardId>, PersistenceError>
+
+  /**
+   * Release the given shard ids.
+   */
+  readonly release: (
+    address: RunnerAddress,
+    shardId: ShardId
+  ) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Release all the shards assigned to the given runner.
+   */
+  readonly releaseAll: (address: RunnerAddress) => Effect.Effect<void, PersistenceError>
+}>() {}
+
+/**
+ * @since 1.0.0
+ * @category Encoded
+ */
+export interface Encoded {
+  /**
+   * Get all runners registered with the cluster.
+   */
+  readonly getRunners: Effect.Effect<Array<readonly [runner: string, healthy: boolean]>, PersistenceError>
+
+  /**
+   * Register a new runner with the cluster.
+   */
+  readonly register: (address: string, runner: string, healthy: boolean) => Effect.Effect<number, PersistenceError>
+
+  /**
+   * Unregister the runner with the given address.
+   */
+  readonly unregister: (address: string) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Set the health status of the given runner.
+   */
+  readonly setRunnerHealth: (address: string, healthy: boolean) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Acquire the lock on the given shards, returning the shards that were
+   * successfully locked.
+   */
+  readonly acquire: (
+    address: string,
+    shardIds: NonEmptyArray<string>
+  ) => Effect.Effect<Array<string>, PersistenceError>
+
+  /**
+   * Refresh the lock on the given shards, returning the shards that were
+   * successfully locked.
+   */
+  readonly refresh: (
+    address: string,
+    shardIds: Array<string>
+  ) => Effect.Effect<ReadonlyArray<string>, PersistenceError>
+
+  /**
+   * Release the lock on the given shard.
+   */
+  readonly release: (
+    address: string,
+    shardId: string
+  ) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Release the lock on all shards for the given runner.
+   */
+  readonly releaseAll: (address: string) => Effect.Effect<void, PersistenceError>
+}
+
+/**
+ * @since 1.0.0
+ * @category layers
+ */
+export const makeEncoded = (encoded: Encoded) =>
+  RunnerStorage.of({
+    getRunners: Effect.gen(function*() {
+      const runners = yield* encoded.getRunners
+      const results: Array<[Runner, boolean]> = []
+      for (let i = 0; i < runners.length; i++) {
+        const [runner, healthy] = runners[i]
+        try {
+          results.push([Runner.decodeSync(runner), healthy])
+        } catch {
+          //
+        }
+      }
+      return results
+    }),
+    register: (runner, healthy) =>
+      Effect.map(
+        encoded.register(encodeRunnerAddress(runner.address), Runner.encodeSync(runner), healthy),
+        MachineId.make
+      ),
+    unregister: (address) => encoded.unregister(encodeRunnerAddress(address)),
+    setRunnerHealth: (address, healthy) => encoded.setRunnerHealth(encodeRunnerAddress(address), healthy),
+    acquire: (address, shardIds) => {
+      const arr = Array.from(shardIds, (id) => id.toString())
+      if (!isNonEmptyArray(arr)) return Effect.succeed([])
+      return encoded.acquire(encodeRunnerAddress(address), arr).pipe(
+        Effect.map((shards) => shards.map(ShardId.fromString))
+      )
+    },
+    refresh: (address, shardIds) =>
+      encoded.refresh(encodeRunnerAddress(address), Array.from(shardIds, (id) => id.toString())).pipe(
+        Effect.map((shards) => shards.map(ShardId.fromString))
+      ),
+    release(address, shardId) {
+      return encoded.release(encodeRunnerAddress(address), shardId.toString())
+    },
+    releaseAll(address) {
+      return encoded.releaseAll(encodeRunnerAddress(address))
+    }
+  })
+
+/**
+ * @since 1.0.0
+ * @category constructors
+ */
+export const makeMemory = Effect.gen(function*() {
+  const runners = MutableHashMap.empty<RunnerAddress, Runner>()
+  let acquired: Array<ShardId> = []
+  let id = 0
+
+  return RunnerStorage.of({
+    getRunners: Effect.sync(() => Array.from(MutableHashMap.values(runners), (runner) => [runner, true])),
+    register: (runner) =>
+      Effect.sync(() => {
+        MutableHashMap.set(runners, runner.address, runner)
+        return MachineId.make(id++)
+      }),
+    unregister: (address) =>
+      Effect.sync(() => {
+        MutableHashMap.remove(runners, address)
+      }),
+    setRunnerHealth: () => Effect.void,
+    acquire: (_address, shardIds) => {
+      acquired = Array.from(shardIds)
+      return Effect.succeed(Array.from(shardIds))
+    },
+    refresh: () => Effect.sync(() => acquired),
+    release: () => Effect.void,
+    releaseAll: () => Effect.void
+  })
+})
+
+/**
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerMemory: Layer.Layer<RunnerStorage> = Layer.effect(RunnerStorage)(makeMemory)
+
+// -------------------------------------------------------------------------------------
+// internal
+// -------------------------------------------------------------------------------------
+
+const encodeRunnerAddress = (runnerAddress: RunnerAddress) => `${runnerAddress.host}:${runnerAddress.port}`