effect 4.0.0-beta.66 → 4.0.0-beta.68

package/src/unstable/cluster/Runner.ts

@@ -1,4 +1,28 @@
 /**
+ * The `Runner` module defines the membership record used by the unstable
+ * cluster runtime to describe a process that can host entity shards.
+ *
+ * A runner combines the network address used by other runners to reach it, the
+ * shard groups it participates in, and a relative weight used when the sharding
+ * service assigns shards across the healthy runners in each group.
+ *
+ * **Common tasks**
+ *
+ * - Construct the runner registered by the local `Sharding` layer
+ * - Persist or exchange runner metadata through `RunnerStorage`
+ * - Encode and decode runner values at cluster transport or storage boundaries
+ * - Tune shard distribution by adjusting the runner's group membership and
+ *   relative weight
+ *
+ * **Gotchas**
+ *
+ * - Runner addresses must be stable and unique while a runner is registered,
+ *   because they identify the owner used for routing and shard locks.
+ * - Weights are relative within each shard group; changing weights or groups can
+ *   rebalance shard ownership as the cluster refreshes its runner view.
+ * - Runner equality and hashing are based on address and weight, so compare
+ *   `groups` explicitly when group membership is the important distinction.
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -10,17 +34,13 @@ import { RunnerAddress } from "./RunnerAddress.ts"
 const TypeId = "~effect/cluster/Runner"
 
 /**
- * A `Runner` represents a physical application server that is capable of running
- * entities.
+ * A cluster runner that can host entities.
  *
- * Because a Runner represents a physical application server, a Runner must have a
- * unique `address` which can be used to communicate with the server.
+ * Each runner has a unique network `address`, the shard `groups` it participates
+ * in, and a relative `weight` used when assigning shards across runners.
  *
- * The version of a Runner is used during rebalancing to give priority to newer
- * application servers and slowly decommission older ones.
- *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class Runner extends Schema.Class<Runner>(TypeId)({
   address: RunnerAddress,
@@ -28,26 +48,36 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   weight: Schema.Number
 }) {
   /**
+   * Formatter for rendering runner values consistently.
+   *
    * @since 4.0.0
    */
   static format = Schema.toFormatter(this)
 
   /**
+   * Marks this value as a cluster runner for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Decodes a runner from its JSON string representation.
+   *
    * @since 4.0.0
    */
   static readonly decodeSync = Schema.decodeSync(Schema.fromJsonString(Runner))
 
   /**
+   * Encodes a runner to its JSON string representation.
+   *
    * @since 4.0.0
    */
   static readonly encodeSync = Schema.encodeSync(Schema.fromJsonString(Runner))
 
   /**
+   * Formats this runner as a string.
+   *
    * @since 4.0.0
    */
   override toString(): string {
@@ -55,6 +85,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Formats this runner for Node.js inspection.
+   *
    * @since 4.0.0
    */
   [NodeInspectSymbol](): string {
@@ -62,6 +94,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Compares runners by address and shard-assignment weight.
+   *
    * @since 4.0.0
    */
   [Equal.symbol](that: Runner): boolean {
@@ -69,6 +103,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Computes a structural hash from the runner address and shard-assignment weight.
+   *
    * @since 4.0.0
    */
   [Hash.symbol](): number {
@@ -77,17 +113,11 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
 }
 
 /**
- * A `Runner` represents a physical application server that is capable of running
- * entities.
- *
- * Because a Runner represents a physical application server, a Runner must have a
- * unique `address` which can be used to communicate with the server.
- *
- * The version of a Runner is used during rebalancing to give priority to newer
- * application servers and slowly decommission older ones.
+ * Constructs a `Runner` from its network address, shard groups, and relative
+ * shard-assignment weight.
  *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const make = (props: {
   readonly address: RunnerAddress

package/src/unstable/cluster/RunnerAddress.ts

@@ -1,4 +1,20 @@
 /**
+ * The `RunnerAddress` module defines the network identity used to locate a
+ * cluster runner. A runner address is a host and port pair that can be encoded,
+ * compared, hashed, inspected, and used as a stable primary key.
+ *
+ * **Common use cases**
+ *
+ * - Representing the target runner for cluster routing and placement decisions
+ * - Persisting or exchanging runner endpoints through schemas
+ * - Using runner endpoints as keys in maps, registries, or shard ownership data
+ *
+ * **Gotchas**
+ *
+ * - Identity is structural: two addresses are equal when both host and port match
+ * - The primary key is formatted as `host:port`, so host strings should already
+ *   be normalized for the routing layer using them
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -10,19 +26,25 @@ import * as Schema from "../../Schema.ts"
 const TypeId = "~effect/cluster/RunnerAddress"
 
 /**
- * @since 4.0.0
+ * Network address of a cluster runner, identified by host and port.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   host: Schema.String,
   port: Schema.Number
 }) {
   /**
+   * Marks this value as a cluster runner address for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId;
 
   /**
+   * Compares runner addresses by host and port.
+   *
    * @since 4.0.0
    */
   [Equal.symbol](that: RunnerAddress): boolean {
@@ -30,6 +52,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Computes a structural hash from the host and port.
+   *
    * @since 4.0.0
    */
   [Hash.symbol]() {
@@ -37,6 +61,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Stable primary key used to identify the runner address.
+   *
    * @since 4.0.0
    */
   [PrimaryKey.symbol](): string {
@@ -44,6 +70,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Formats the runner address with its host and port.
+   *
    * @since 4.0.0
    */
   override toString(): string {
@@ -51,6 +79,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Formats the runner address for Node.js inspection.
+   *
    * @since 4.0.0
    */
   [NodeInspectSymbol](): string {
@@ -59,8 +89,10 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
 }
 
 /**
- * @since 4.0.0
+ * Constructs a `RunnerAddress` from a host and port.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const make = (host: string, port: number): RunnerAddress =>
   new RunnerAddress({ host, port }, { disableChecks: true })

package/src/unstable/cluster/RunnerHealth.ts

@@ -1,4 +1,24 @@
 /**
+ * The `RunnerHealth` module defines the health-check service used by cluster
+ * sharding to decide whether a runner may still own its assigned shards. A
+ * runner that is reported as alive is allowed to keep processing messages,
+ * while a runner that is reported as unavailable can have its shards moved to
+ * another runner.
+ *
+ * **Common tasks**
+ *
+ * - Provide a custom {@link RunnerHealth} service for a cluster deployment
+ * - Use {@link layerPing} to check runners through the cluster runner protocol
+ * - Use {@link layerK8s} when Kubernetes pod readiness should drive health
+ * - Use {@link layerNoop} in tests or environments where runners are always considered healthy
+ *
+ * **Gotchas**
+ *
+ * - Health checks affect shard reassignment, so false negatives can move shards
+ *   away from runners that may still be processing messages
+ * - The Kubernetes implementation treats API failures as healthy to avoid
+ *   reassignment caused by a temporary control-plane outage
+ *
  * @since 4.0.0
  */
 import * as Context from "../../Context.ts"
@@ -17,8 +37,8 @@ import * as Runners from "./Runners.ts"
  * still be processing messages. If a Runner is not responsive, then its
  * associated shards can and will be re-assigned to a different Runner.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class RunnerHealth extends Context.Service<
   RunnerHealth,
@@ -32,16 +52,20 @@ export class RunnerHealth extends Context.Service<
  *
  * This is useful for testing.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerNoop = Layer.succeed(RunnerHealth, {
   isAlive: () => Effect.succeed(true)
 })
 
 /**
+ * Creates a `RunnerHealth` service that pings runners through `Runners`, retrying
+ * failed pings on a short schedule and treating a successful ping within the
+ * timeout as healthy.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const makePing: Effect.Effect<
   RunnerHealth["Service"],
@@ -65,8 +89,8 @@ export const makePing: Effect.Effect<
 /**
  * A layer which will ping a Runner directly to check if it is healthy.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerPing: Layer.Layer<
   RunnerHealth,
@@ -75,8 +99,13 @@ export const layerPing: Layer.Layer<
 > = Layer.effect(RunnerHealth, makePing)
 
 /**
+ * Creates a `RunnerHealth` service that checks Kubernetes pod readiness for a
+ * runner host, optionally scoped by namespace and label selector.
+ *
+ * If the Kubernetes API check fails, the runner is treated as healthy.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const makeK8s = Effect.fnUntraced(function*(options?: {
   readonly namespace?: string | undefined
@@ -94,16 +123,15 @@ export const makeK8s = Effect.fnUntraced(function*(options?: {
 })
 
 /**
- * A layer which will check the Kubernetes API to see if a Runner is healthy.
- *
- * The provided HttpClient will need to add the pod's CA certificate to its
- * trusted root certificates in order to communicate with the Kubernetes API.
+ * A layer which checks Kubernetes pod readiness to determine whether a runner is
+ * healthy.
  *
- * The pod service account will also need to have permissions to list pods in
- * order to use this layer.
+ * The provided `HttpClient` must trust the pod CA certificate and the pod service
+ * account must be allowed to list pods. If the Kubernetes API check fails, the
+ * runner is treated as healthy.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerK8s = (
   options?: {

package/src/unstable/cluster/RunnerServer.ts

@@ -1,4 +1,28 @@
 /**
+ * The `RunnerServer` module provides the transport-agnostic server side of the
+ * cluster runner protocol. It turns the runner RPC group into handlers that
+ * receive ping, notification, request, stream, and envelope messages from other
+ * runners, then forwards them into `Sharding` and coordinates persisted replies
+ * through `MessageStorage`.
+ *
+ * **Common tasks**
+ *
+ * - Build a runner server once an `RpcServer.Protocol` has been supplied by a
+ *   transport such as HTTP, WebSocket, or sockets
+ * - Provide the complete runner runtime with `Sharding` and `Runners` clients
+ *   using {@link layerWithClients}
+ * - Embed a cluster client without serving runner RPCs or accepting shard
+ *   assignments using {@link layerClientOnly}
+ *
+ * **Gotchas**
+ *
+ * - This module does not choose a wire transport; transport-specific modules
+ *   provide the `RpcServer.Protocol`
+ * - Persisted requests register reply handlers in `MessageStorage` before the
+ *   message is delivered to `Sharding`
+ * - Client-only layers clear the configured runner address, so they can send
+ *   cluster messages but do not register as shard-owning runners
+ *
  * @since 4.0.0
  */
 import * as Effect from "../../Effect.ts"
@@ -22,8 +46,11 @@ import { ShardingConfig } from "./ShardingConfig.ts"
 const constVoid = constant(Effect.void)
 
 /**
+ * Layer of RPC handlers for the runner protocol, forwarding ping, notify, effect,
+ * stream, and envelope requests to `Sharding` and `MessageStorage`.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
   const sharding = yield* Sharding.Sharding
@@ -134,8 +161,8 @@ const constWaitUntilRead = { waitUntilRead: true } as const
  *
  * It also responds to `Ping` requests.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   never,
@@ -149,8 +176,8 @@ export const layer: Layer.Layer<
 /**
  * A `RunnerServer` layer that includes the `Runners` & `Sharding` clients.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerWithClients: Layer.Layer<
   Sharding.Sharding | Runners.Runners,
@@ -173,8 +200,8 @@ export const layerWithClients: Layer.Layer<
  * so this layer can be used to embed a cluster client inside another effect
  * application.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerClientOnly: Layer.Layer<
   Sharding.Sharding | Runners.Runners,

package/src/unstable/cluster/RunnerStorage.ts

@@ -1,4 +1,27 @@
 /**
+ * The `RunnerStorage` module defines the persistence boundary used by clustered
+ * runners to register themselves and coordinate shard ownership.
+ *
+ * Implementations keep track of runner metadata, health, machine ids, and shard
+ * locks so a cluster can rebalance work as runners join, leave, or lose their
+ * leases. Production adapters usually implement the string-encoded interface and
+ * adapt it with {@link makeEncoded}; tests and local setups can use
+ * {@link makeMemory}.
+ *
+ * **Common tasks**
+ *
+ * - Register and unregister runners in a shared store
+ * - Read runner health for scheduling and rebalancing decisions
+ * - Acquire, refresh, and release shard locks for distributed processing
+ * - Bridge typed cluster values to string or numeric database representations
+ *
+ * **Gotchas**
+ *
+ * - Shard acquisition may be partial; callers must use the returned shard list
+ * - Refreshing leases is part of keeping shard ownership during rebalancing
+ * - The in-memory implementation is process-local and does not persist runner
+ *   registrations or locks across restarts
+ *
  * @since 4.0.0
  */
 import { isArrayNonEmpty, type NonEmptyArray } from "../../Array.ts"
@@ -16,8 +39,8 @@ import * as ShardId from "./ShardId.ts"
  * Represents a generic interface to the persistent storage required by the
  * cluster.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class RunnerStorage extends Context.Service<RunnerStorage, {
   /**
@@ -73,8 +96,11 @@ export class RunnerStorage extends Context.Service<RunnerStorage, {
 }>()("effect/cluster/RunnerStorage") {}
 
 /**
- * @since 4.0.0
+ * String-encoded runner storage interface used by adapters that persist runner
+ * addresses, runners, machine ids, and shard ids outside the in-memory model.
+ *
  * @category Encoded
+ * @since 4.0.0
  */
 export interface Encoded {
   /**
@@ -130,8 +156,12 @@ export interface Encoded {
 }
 
 /**
- * @since 4.0.0
+ * Adapts an encoded runner storage implementation into `RunnerStorage`, converting
+ * runner addresses, runners, machine ids, and shard ids between typed values and
+ * their string or numeric storage forms.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const makeEncoded = (encoded: Encoded) =>
   RunnerStorage.of({
@@ -176,8 +206,13 @@ export const makeEncoded = (encoded: Encoded) =>
   })
 
 /**
- * @since 4.0.0
+ * Creates an in-memory `RunnerStorage` implementation for tests and local use.
+ *
+ * Registered runners are treated as healthy and shard acquisition is kept only in
+ * process memory.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const makeMemory = Effect.gen(function*() {
   const runners = MutableHashMap.empty<RunnerAddress, Runner>()
@@ -207,8 +242,10 @@ export const makeMemory = Effect.gen(function*() {
 })
 
 /**
- * @since 4.0.0
+ * Layer that provides the in-memory `RunnerStorage` implementation.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerMemory: Layer.Layer<RunnerStorage> = Layer.effect(RunnerStorage)(makeMemory)
 

package/src/unstable/cluster/Runners.ts

@@ -1,4 +1,34 @@
 /**
+ * The `Runners` module defines the service used by the unstable cluster runtime
+ * to communicate with processes that host entity shards. It is the transport
+ * boundary between sharding decisions and runner execution: callers can ping a
+ * runner, send requests or envelopes, notify a runner that persisted work is
+ * available, and report an address as unavailable.
+ *
+ * The default implementation wraps lower-level runner callbacks with cluster
+ * message semantics. Persisted messages are written to `MessageStorage` before
+ * delivery, duplicate requests can resume from stored replies, and local sends
+ * can optionally serialize and deserialize messages to exercise the same path as
+ * remote delivery.
+ *
+ * **Common tasks**
+ *
+ * - Provide runner communication with {@link layerRpc}
+ * - Build a custom implementation with {@link make}
+ * - Use {@link makeNoop} or {@link layerNoop} when no remote runners are
+ *   available
+ * - Define runner-to-runner protocol support with {@link Rpcs} and
+ *   {@link RpcClientProtocol}
+ *
+ * **Gotchas**
+ *
+ * - `notify` is only for RPCs annotated as persisted; non-persisted messages
+ *   should be sent directly.
+ * - Failed remote sends can fall back to reading replies from storage, so reply
+ *   polling and `entityReplyPollInterval` affect recovery latency.
+ * - Unavailable runners invalidate cached RPC clients, but shard ownership and
+ *   rebalancing are coordinated by the sharding layer rather than this module.
+ *
  * @since 4.0.0
  */
 import * as Context from "../../Context.ts"
@@ -28,8 +58,12 @@ import { ShardingConfig } from "./ShardingConfig.ts"
 import * as Snowflake from "./Snowflake.ts"
 
 /**
- * @since 1.0.0
+ * Service for communicating with cluster runners, including pinging runners,
+ * sending and notifying messages, coordinating persisted replies, and marking
+ * runners unavailable.
+ *
  * @category context
+ * @since 4.0.0
  */
 export class Runners extends Context.Service<Runners, {
   /**
@@ -112,8 +146,12 @@ export class Runners extends Context.Service<Runners, {
 }>()("effect/cluster/Runners") {}
 
 /**
- * @since 1.0.0
- * @category Constructors
+ * Builds the `Runners` service from remote runner callbacks and adds local
+ * message persistence, duplicate request handling, optional local serialization
+ * simulation, and polling for persisted replies.
+ *
+ * @category constructors
+ * @since 4.0.0
  */
 export const make: (options: Omit<Runners["Service"], "sendLocal" | "notifyLocal">) => Effect.Effect<
   Runners["Service"],
@@ -380,8 +418,12 @@ export const make: (options: Omit<Runners["Service"], "sendLocal" | "notifyLocal
 })
 
 /**
- * @since 1.0.0
+ * Creates a no-op `Runners` service that rejects sends with
+ * `EntityNotAssignedToRunner` and ignores notifications, pings, and unavailable
+ * runner reports.
+ *
  * @category No-op
+ * @since 4.0.0
  */
 export const makeNoop: Effect.Effect<
   Runners["Service"],
@@ -395,8 +437,11 @@ export const makeNoop: Effect.Effect<
 })
 
 /**
- * @since 1.0.0
- * @category Layers
+ * Layer that provides the no-op `Runners` service, using the default snowflake
+ * generator.
+ *
+ * @category layers
+ * @since 4.0.0
  */
 export const layerNoop: Layer.Layer<
   Runners,
@@ -415,8 +460,11 @@ const rpcErrors: Schema.Union<[
 ])
 
 /**
- * @since 1.0.0
+ * RPC group used for runner-to-runner communication, including ping, notify,
+ * effect, stream, and envelope messages.
+ *
  * @category Rpcs
+ * @since 4.0.0
  */
 export class Rpcs extends RpcGroup.make(
   Rpc.make("Ping"),
@@ -454,14 +502,19 @@ export class Rpcs extends RpcGroup.make(
 ) {}
 
 /**
- * @since 1.0.0
+ * Client interface generated from the runner RPC group.
+ *
  * @category Rpcs
+ * @since 4.0.0
  */
 export interface RpcClient extends RpcClient_.FromGroup<typeof Rpcs, RpcClientError> {}
 
 /**
- * @since 1.0.0
+ * Builds a runner RPC client from the current `RpcClient.Protocol`, using the
+ * `Runners` span prefix with tracing disabled.
+ *
  * @category Rpcs
+ * @since 4.0.0
  */
 export const makeRpcClient: Effect.Effect<
   RpcClient,
@@ -470,8 +523,12 @@ export const makeRpcClient: Effect.Effect<
 > = RpcClient_.make(Rpcs, { spanPrefix: "Runners", disableTracing: true })
 
 /**
- * @since 1.0.0
+ * Builds a `Runners` service backed by RPC clients, caching a client per runner
+ * address and dispatching ping, notify, effect, stream, and envelope messages over
+ * the runner protocol.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const makeRpc: Effect.Effect<
   Runners["Service"],
@@ -603,8 +660,11 @@ export const makeRpc: Effect.Effect<
 })
 
 /**
- * @since 1.0.0
- * @category Layers
+ * Layer that provides an RPC-backed `Runners` service using `RpcClientProtocol`,
+ * message storage, sharding configuration, and the default snowflake generator.
+ *
+ * @category layers
+ * @since 4.0.0
  */
 export const layerRpc: Layer.Layer<
   Runners,
@@ -615,8 +675,11 @@ export const layerRpc: Layer.Layer<
 )
 
 /**
- * @since 1.0.0
+ * Service that creates an RPC client protocol for communicating with a runner at a
+ * given address.
+ *
  * @category Client
+ * @since 4.0.0
  */
 export class RpcClientProtocol extends Context.Service<
   RpcClientProtocol,