effect 4.0.0-beta.66 → 4.0.0-beta.67

package/src/unstable/cluster/Reply.ts

@@ -1,4 +1,25 @@
 /**
+ * The `Reply` module models responses produced by clustered RPC execution. A
+ * reply belongs to a request and is either a terminal {@link WithExit}, which
+ * carries the final RPC `Exit`, or a streaming {@link Chunk}, which carries a
+ * non-empty batch of success values for RPCs that stream results.
+ *
+ * **Common tasks**
+ *
+ * - Represent runtime replies with {@link Reply}, {@link WithExit}, and {@link Chunk}
+ * - Encode and decode transport payloads with {@link Encoded} and {@link Reply}
+ * - Persist replies together with schema context via {@link ReplyWithContext}
+ * - Serialize the latest received reply when resuming or de-duplicating requests with {@link serializeLastReceived}
+ *
+ * **Streaming and acknowledgement notes**
+ *
+ * - Chunk replies are sequenced and can be replayed until acknowledged by the
+ *   receiver.
+ * - A `WithExit` reply is terminal and completes the request, while chunks only
+ *   represent intermediate streamed success values.
+ * - `Chunk.emptyFrom` is used as an acknowledgement marker for an empty streamed
+ *   reply; it is not a general-purpose success payload.
+ *
  * @since 4.0.0
  */
 import type { NonEmptyReadonlyArray } from "../../Array.ts"
@@ -22,32 +43,49 @@ import { Snowflake, SnowflakeFromBigInt } from "./Snowflake.ts"
 const TypeId = "~effect/cluster/Reply"
 
 /**
- * @since 4.0.0
+ * Returns `true` when the supplied value is a runtime cluster reply.
+ *
+ * The check is based on the reply type identifier.
+ *
  * @category guards
+ * @since 4.0.0
  */
 export const isReply = (u: unknown): u is Reply<Rpc.Any> => hasProperty(u, TypeId)
 
 /**
- * @since 4.0.0
+ * Runtime reply sent for an RPC request.
+ *
+ * A reply is either a final exit or a chunk of a streaming success value.
+ *
  * @category models
+ * @since 4.0.0
  */
 export type Reply<R extends Rpc.Any> = WithExit<R> | Chunk<R>
 
 /**
- * @since 4.0.0
+ * JSON-serializable form of a cluster reply.
+ *
  * @category models
+ * @since 4.0.0
  */
 export type Encoded = WithExitEncoded | ChunkEncoded
 
 /**
- * @since 4.0.0
+ * Codec used for reply values that are already in encoded form.
+ *
+ * Per-RPC payload validation is performed by `Reply(rpc)`.
+ *
  * @category models
+ * @since 4.0.0
  */
 export const Encoded: Schema.Codec<Encoded> = Schema.Any as any
 
 /**
- * @since 4.0.0
+ * A cluster reply paired with the RPC definition and service context required to
+ * serialize it for transport.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("ReplyWithContext")<{
   readonly reply: Reply<R>
@@ -55,6 +93,8 @@ export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("Reply
   readonly rpc: R
 }> {
   /**
+   * Creates a terminal reply context that dies with the supplied defect.
+   *
    * @since 4.0.0
    */
   static fromDefect(options: {
@@ -73,6 +113,8 @@ export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("Reply
     })
   }
   /**
+   * Creates a terminal reply context that interrupts the supplied request.
+   *
    * @since 4.0.0
    */
   static interrupt(options: {
@@ -98,8 +140,11 @@ const neverRpc = Rpc.make("Never", {
 })
 
 /**
- * @since 4.0.0
+ * Wire-format representation of a terminal reply containing the request id, reply
+ * id, and encoded RPC exit value.
+ *
  * @category models
+ * @since 4.0.0
  */
 export interface WithExitEncoded<A = unknown, E = unknown> {
   readonly _tag: "WithExit"
@@ -109,8 +154,11 @@ export interface WithExitEncoded<A = unknown, E = unknown> {
 }
 
 /**
- * @since 4.0.0
+ * Wire-format representation of a streaming reply chunk, including the request id,
+ * reply id, sequence number, and non-empty encoded values.
+ *
  * @category models
+ * @since 4.0.0
  */
 export interface ChunkEncoded {
   readonly _tag: "Chunk"
@@ -123,8 +171,11 @@ export interface ChunkEncoded {
 const schemaCache = new WeakMap<Rpc.Any, Schema.Top>()
 
 /**
- * @since 4.0.0
+ * A streaming RPC reply chunk for a request, carrying a non-empty batch of success
+ * values together with the reply id and sequence number.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   readonly requestId: Snowflake
@@ -133,11 +184,15 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   readonly values: NonEmptyReadonlyArray<Rpc.SuccessChunk<R>>
 }> {
   /**
+   * Marks this value as a runtime cluster reply.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Creates an empty chunk reply for the supplied request id.
+   *
    * @since 4.0.0
    */
   static emptyFrom(requestId: Snowflake) {
@@ -150,11 +205,15 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Schema that accepts any runtime chunk reply without validating payload values.
+   *
    * @since 4.0.0
    */
   static readonly Any = Schema.declare((u): u is Chunk<never> => isReply(u) && u._tag === "Chunk")
 
   /**
+   * Transformation between encoded chunk records and `Chunk` instances.
+   *
    * @since 4.0.0
    */
   static readonly transform: Transformation.Transformation<any, any> = Transformation.transform({
@@ -163,6 +222,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   })
 
   /**
+   * Builds a chunk schema from the streaming success schema of an RPC.
+   *
    * @since 4.0.0
    */
   static schema<R extends Rpc.Any>(
@@ -176,6 +237,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Builds a chunk schema that validates each success value with the supplied schema.
+   *
    * @since 4.0.0
    */
   static schemaFrom<Success extends Schema.Top>(
@@ -214,6 +277,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Returns a copy of this chunk associated with the supplied request id.
+   *
    * @since 4.0.0
    */
   withRequestId(requestId: Snowflake): Chunk<R> {
@@ -225,8 +290,11 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
 }
 
 /**
- * @since 4.0.0
+ * A terminal RPC reply for a request, carrying the final `Exit` for the remote
+ * call.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   readonly requestId: Snowflake
@@ -234,11 +302,15 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   readonly exit: Rpc.Exit<R>
 }> {
   /**
+   * Marks this value as a runtime cluster reply.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Returns `true` when the value is a terminal `WithExit` reply.
+   *
    * @since 4.0.0
    */
   static is(u: unknown): u is WithExit<any> {
@@ -246,6 +318,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Builds a terminal reply schema from the exit schema of an RPC.
+   *
    * @since 4.0.0
    */
   static schema<R extends Rpc.Any>(
@@ -259,6 +333,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Builds a terminal reply schema that validates the encoded exit value.
+   *
    * @since 4.0.0
    */
   static schemaFrom<Success extends Schema.Top, Error extends Schema.Top, Defect extends Schema.Top>(
@@ -300,6 +376,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Returns a copy of this terminal reply associated with the supplied request id.
+   *
    * @since 4.0.0
    */
   withRequestId(requestId: Snowflake): WithExit<R> {
@@ -311,8 +389,11 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
 }
 
 /**
- * @since 4.0.0
+ * Builds the transport codec for replies to the specified RPC, covering terminal
+ * `WithExit` replies and streaming `Chunk` replies.
+ *
  * @category schemas
+ * @since 4.0.0
  */
 export const Reply = <R extends Rpc.Any>(
   rpc: R
@@ -331,8 +412,12 @@ export const Reply = <R extends Rpc.Any>(
 }
 
 /**
- * @since 4.0.0
+ * Serializes a `ReplyWithContext` into its encoded wire representation, using the
+ * reply's RPC schema and context and refailing encoding errors as
+ * `MalformedMessage`.
+ *
  * @category serialization / deserialization
+ * @since 4.0.0
  */
 export const serialize = <R extends Rpc.Any>(
   self: ReplyWithContext<R>
@@ -347,8 +432,12 @@ export const serialize = <R extends Rpc.Any>(
 }
 
 /**
- * @since 4.0.0
+ * Serializes an outgoing request's last received reply when one exists, returning
+ * `None` when no reply has been received and refailing encoding errors as
+ * `MalformedMessage`.
+ *
  * @category serialization / deserialization
+ * @since 4.0.0
  */
 export const serializeLastReceived = <R extends Rpc.Any>(
   self: OutgoingRequest<R>

package/src/unstable/cluster/Runner.ts

@@ -1,4 +1,28 @@
 /**
+ * The `Runner` module defines the membership record used by the unstable
+ * cluster runtime to describe a process that can host entity shards.
+ *
+ * A runner combines the network address used by other runners to reach it, the
+ * shard groups it participates in, and a relative weight used when the sharding
+ * service assigns shards across the healthy runners in each group.
+ *
+ * **Common tasks**
+ *
+ * - Construct the runner registered by the local `Sharding` layer
+ * - Persist or exchange runner metadata through `RunnerStorage`
+ * - Encode and decode runner values at cluster transport or storage boundaries
+ * - Tune shard distribution by adjusting the runner's group membership and
+ *   relative weight
+ *
+ * **Gotchas**
+ *
+ * - Runner addresses must be stable and unique while a runner is registered,
+ *   because they identify the owner used for routing and shard locks.
+ * - Weights are relative within each shard group; changing weights or groups can
+ *   rebalance shard ownership as the cluster refreshes its runner view.
+ * - Runner equality and hashing are based on address and weight, so compare
+ *   `groups` explicitly when group membership is the important distinction.
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -10,17 +34,13 @@ import { RunnerAddress } from "./RunnerAddress.ts"
 const TypeId = "~effect/cluster/Runner"
 
 /**
- * A `Runner` represents a physical application server that is capable of running
- * entities.
- *
- * Because a Runner represents a physical application server, a Runner must have a
- * unique `address` which can be used to communicate with the server.
+ * A cluster runner that can host entities.
  *
- * The version of a Runner is used during rebalancing to give priority to newer
- * application servers and slowly decommission older ones.
+ * Each runner has a unique network `address`, the shard `groups` it participates
+ * in, and a relative `weight` used when assigning shards across runners.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class Runner extends Schema.Class<Runner>(TypeId)({
   address: RunnerAddress,
@@ -28,26 +48,36 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   weight: Schema.Number
 }) {
   /**
+   * Formatter for rendering runner values consistently.
+   *
    * @since 4.0.0
    */
   static format = Schema.toFormatter(this)
 
   /**
+   * Marks this value as a cluster runner for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Decodes a runner from its JSON string representation.
+   *
    * @since 4.0.0
    */
   static readonly decodeSync = Schema.decodeSync(Schema.fromJsonString(Runner))
 
   /**
+   * Encodes a runner to its JSON string representation.
+   *
    * @since 4.0.0
    */
   static readonly encodeSync = Schema.encodeSync(Schema.fromJsonString(Runner))
 
   /**
+   * Formats this runner as a string.
+   *
    * @since 4.0.0
    */
   override toString(): string {
@@ -55,6 +85,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Formats this runner for Node.js inspection.
+   *
    * @since 4.0.0
    */
   [NodeInspectSymbol](): string {
@@ -62,6 +94,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Compares runners by address and shard-assignment weight.
+   *
    * @since 4.0.0
    */
   [Equal.symbol](that: Runner): boolean {
@@ -69,6 +103,8 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
   }
 
   /**
+   * Computes a structural hash from the runner address and shard-assignment weight.
+   *
    * @since 4.0.0
    */
   [Hash.symbol](): number {
@@ -77,17 +113,11 @@ export class Runner extends Schema.Class<Runner>(TypeId)({
 }
 
 /**
- * A `Runner` represents a physical application server that is capable of running
- * entities.
- *
- * Because a Runner represents a physical application server, a Runner must have a
- * unique `address` which can be used to communicate with the server.
+ * Constructs a `Runner` from its network address, shard groups, and relative
+ * shard-assignment weight.
  *
- * The version of a Runner is used during rebalancing to give priority to newer
- * application servers and slowly decommission older ones.
- *
- * @since 4.0.0
  * @category Constructors
+ * @since 4.0.0
  */
 export const make = (props: {
   readonly address: RunnerAddress

package/src/unstable/cluster/RunnerAddress.ts

@@ -1,4 +1,20 @@
 /**
+ * The `RunnerAddress` module defines the network identity used to locate a
+ * cluster runner. A runner address is a host and port pair that can be encoded,
+ * compared, hashed, inspected, and used as a stable primary key.
+ *
+ * **Common use cases**
+ *
+ * - Representing the target runner for cluster routing and placement decisions
+ * - Persisting or exchanging runner endpoints through schemas
+ * - Using runner endpoints as keys in maps, registries, or shard ownership data
+ *
+ * **Gotchas**
+ *
+ * - Identity is structural: two addresses are equal when both host and port match
+ * - The primary key is formatted as `host:port`, so host strings should already
+ *   be normalized for the routing layer using them
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -10,19 +26,25 @@ import * as Schema from "../../Schema.ts"
 const TypeId = "~effect/cluster/RunnerAddress"
 
 /**
- * @since 4.0.0
+ * Network address of a cluster runner, identified by host and port.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   host: Schema.String,
   port: Schema.Number
 }) {
   /**
+   * Marks this value as a cluster runner address for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId;
 
   /**
+   * Compares runner addresses by host and port.
+   *
    * @since 4.0.0
    */
   [Equal.symbol](that: RunnerAddress): boolean {
@@ -30,6 +52,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Computes a structural hash from the host and port.
+   *
    * @since 4.0.0
    */
   [Hash.symbol]() {
@@ -37,6 +61,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Stable primary key used to identify the runner address.
+   *
    * @since 4.0.0
    */
   [PrimaryKey.symbol](): string {
@@ -44,6 +70,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Formats the runner address with its host and port.
+   *
    * @since 4.0.0
    */
   override toString(): string {
@@ -51,6 +79,8 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   }
 
   /**
+   * Formats the runner address for Node.js inspection.
+   *
    * @since 4.0.0
    */
   [NodeInspectSymbol](): string {
@@ -59,8 +89,10 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
 }
 
 /**
- * @since 4.0.0
+ * Constructs a `RunnerAddress` from a host and port.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const make = (host: string, port: number): RunnerAddress =>
   new RunnerAddress({ host, port }, { disableChecks: true })

package/src/unstable/cluster/RunnerHealth.ts

@@ -1,4 +1,24 @@
 /**
+ * The `RunnerHealth` module defines the health-check service used by cluster
+ * sharding to decide whether a runner may still own its assigned shards. A
+ * runner that is reported as alive is allowed to keep processing messages,
+ * while a runner that is reported as unavailable can have its shards moved to
+ * another runner.
+ *
+ * **Common tasks**
+ *
+ * - Provide a custom {@link RunnerHealth} service for a cluster deployment
+ * - Use {@link layerPing} to check runners through the cluster runner protocol
+ * - Use {@link layerK8s} when Kubernetes pod readiness should drive health
+ * - Use {@link layerNoop} in tests or environments where runners are always considered healthy
+ *
+ * **Gotchas**
+ *
+ * - Health checks affect shard reassignment, so false negatives can move shards
+ *   away from runners that may still be processing messages
+ * - The Kubernetes implementation treats API failures as healthy to avoid
+ *   reassignment caused by a temporary control-plane outage
+ *
  * @since 4.0.0
  */
 import * as Context from "../../Context.ts"
@@ -17,8 +37,8 @@ import * as Runners from "./Runners.ts"
  * still be processing messages. If a Runner is not responsive, then its
  * associated shards can and will be re-assigned to a different Runner.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class RunnerHealth extends Context.Service<
   RunnerHealth,
@@ -32,16 +52,20 @@ export class RunnerHealth extends Context.Service<
  *
  * This is useful for testing.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerNoop = Layer.succeed(RunnerHealth, {
   isAlive: () => Effect.succeed(true)
 })
 
 /**
- * @since 4.0.0
+ * Creates a `RunnerHealth` service that pings runners through `Runners`, retrying
+ * failed pings on a short schedule and treating a successful ping within the
+ * timeout as healthy.
+ *
  * @category Constructors
+ * @since 4.0.0
  */
 export const makePing: Effect.Effect<
   RunnerHealth["Service"],
@@ -65,8 +89,8 @@ export const makePing: Effect.Effect<
 /**
  * A layer which will ping a Runner directly to check if it is healthy.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerPing: Layer.Layer<
   RunnerHealth,
@@ -75,8 +99,13 @@ export const layerPing: Layer.Layer<
 > = Layer.effect(RunnerHealth, makePing)
 
 /**
- * @since 4.0.0
+ * Creates a `RunnerHealth` service that checks Kubernetes pod readiness for a
+ * runner host, optionally scoped by namespace and label selector.
+ *
+ * If the Kubernetes API check fails, the runner is treated as healthy.
+ *
  * @category Constructors
+ * @since 4.0.0
  */
 export const makeK8s = Effect.fnUntraced(function*(options?: {
   readonly namespace?: string | undefined
@@ -94,16 +123,15 @@ export const makeK8s = Effect.fnUntraced(function*(options?: {
 })
 
 /**
- * A layer which will check the Kubernetes API to see if a Runner is healthy.
- *
- * The provided HttpClient will need to add the pod's CA certificate to its
- * trusted root certificates in order to communicate with the Kubernetes API.
+ * A layer which checks Kubernetes pod readiness to determine whether a runner is
+ * healthy.
  *
- * The pod service account will also need to have permissions to list pods in
- * order to use this layer.
+ * The provided `HttpClient` must trust the pod CA certificate and the pod service
+ * account must be allowed to list pods. If the Kubernetes API check fails, the
+ * runner is treated as healthy.
  *
- * @since 4.0.0
  * @category layers
+ * @since 4.0.0
  */
 export const layerK8s = (
   options?: {

package/src/unstable/cluster/RunnerServer.ts

@@ -1,4 +1,28 @@
 /**
+ * The `RunnerServer` module provides the transport-agnostic server side of the
+ * cluster runner protocol. It turns the runner RPC group into handlers that
+ * receive ping, notification, request, stream, and envelope messages from other
+ * runners, then forwards them into `Sharding` and coordinates persisted replies
+ * through `MessageStorage`.
+ *
+ * **Common tasks**
+ *
+ * - Build a runner server once an `RpcServer.Protocol` has been supplied by a
+ *   transport such as HTTP, WebSocket, or sockets
+ * - Provide the complete runner runtime with `Sharding` and `Runners` clients
+ *   using {@link layerWithClients}
+ * - Embed a cluster client without serving runner RPCs or accepting shard
+ *   assignments using {@link layerClientOnly}
+ *
+ * **Gotchas**
+ *
+ * - This module does not choose a wire transport; transport-specific modules
+ *   provide the `RpcServer.Protocol`
+ * - Persisted requests register reply handlers in `MessageStorage` before the
+ *   message is delivered to `Sharding`
+ * - Client-only layers clear the configured runner address, so they can send
+ *   cluster messages but do not register as shard-owning runners
+ *
  * @since 4.0.0
  */
 import * as Effect from "../../Effect.ts"
@@ -22,8 +46,11 @@ import { ShardingConfig } from "./ShardingConfig.ts"
 const constVoid = constant(Effect.void)
 
 /**
- * @since 4.0.0
+ * Layer of RPC handlers for the runner protocol, forwarding ping, notify, effect,
+ * stream, and envelope requests to `Sharding` and `MessageStorage`.
+ *
  * @category Layers
+ * @since 4.0.0
  */
 export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
   const sharding = yield* Sharding.Sharding
@@ -134,8 +161,8 @@ const constWaitUntilRead = { waitUntilRead: true } as const
  *
  * It also responds to `Ping` requests.
  *
- * @since 4.0.0
  * @category Layers
+ * @since 4.0.0
  */
 export const layer: Layer.Layer<
   never,
@@ -149,8 +176,8 @@ export const layer: Layer.Layer<
 /**
  * A `RunnerServer` layer that includes the `Runners` & `Sharding` clients.
  *
- * @since 4.0.0
  * @category Layers
+ * @since 4.0.0
  */
 export const layerWithClients: Layer.Layer<
   Sharding.Sharding | Runners.Runners,
@@ -173,8 +200,8 @@ export const layerWithClients: Layer.Layer<
  * so this layer can be used to embed a cluster client inside another effect
  * application.
  *
- * @since 4.0.0
  * @category Layers
+ * @since 4.0.0
  */
 export const layerClientOnly: Layer.Layer<
   Sharding.Sharding | Runners.Runners,