effect 4.0.0-beta.66 → 4.0.0-beta.68

package/src/unstable/cluster/ShardId.ts

@@ -1,4 +1,27 @@
 /**
+ * The `ShardId` module models the address of a shard inside an Effect Cluster
+ * shard group. A shard id is made from a string `group` and numeric `id`, and
+ * the module gives that pair stable equality, hashing, primary-key behavior,
+ * schema support, and conversion to and from the `group:id` string form used by
+ * routing and storage boundaries.
+ *
+ * **Common tasks**
+ *
+ * - Create or reuse a cached shard identifier with {@link make}
+ * - Check runtime values with {@link isShardId}
+ * - Encode or decode shard identifiers with {@link ShardId}
+ * - Format for logs, persistence, or transport with {@link toString}
+ * - Parse encoded shard keys with {@link fromString} or {@link fromStringEncoded}
+ *
+ * **Gotchas**
+ *
+ * - Equality and hashing are based on the `group:id` representation, so both
+ *   fields must match for two shard ids to be equal
+ * - Encoded strings are split at the last `:`; groups may contain colons, but
+ *   ids must parse as numbers
+ * - This module identifies shards after a routing or hashing decision; it does
+ *   not choose a shard for an arbitrary entity key
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -11,8 +34,11 @@ import * as Getter from "../../SchemaGetter.ts"
 const TypeId = "~effect/cluster/ShardId"
 
 /**
+ * Identifier for a shard within a shard group, with equality, hashing, and primary
+ * key behavior based on the `group:id` string form.
+ *
+ * @category models
  * @since 4.0.0
- * @category Models
  */
 export interface ShardId extends Equal.Equal, Hash.Hash, PrimaryKey.PrimaryKey {
   readonly [TypeId]: typeof TypeId
@@ -21,14 +47,19 @@ export interface ShardId extends Equal.Equal, Hash.Hash, PrimaryKey.PrimaryKey {
 }
 
 /**
+ * Returns `true` when the value carries the `ShardId` runtime marker.
+ *
+ * @category guards
  * @since 4.0.0
- * @category Guards
  */
 export const isShardId = (u: unknown): u is ShardId => hasProperty(u, TypeId)
 
 /**
+ * Schema for `ShardId` values encoded as `{ group, id }` objects and decoded via
+ * `make`.
+ *
+ * @category schemas
  * @since 4.0.0
- * @category Schema
  */
 export const ShardId = S.declare(isShardId, {
   toCodecJson: () =>
@@ -45,8 +76,11 @@ export const ShardId = S.declare(isShardId, {
 })
 
 /**
+ * Creates or reuses the cached `ShardId` for the specified shard group and numeric
+ * id.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const make = (group: string, id: number): ShardId => {
   const key = `${group}:${id}`
@@ -84,8 +118,10 @@ const ShardIdProto = {
 }
 
 /**
+ * Formats a shard identifier as `group:id`.
+ *
+ * @category converting
  * @since 4.0.0
- * @category Conversions
  */
 export const toString = (shardId: {
   readonly group: string
@@ -94,6 +130,12 @@ export const toString = (shardId: {
   return `${shardId.group}:${shardId.id}`
 }
 /**
+ * Parses a `group:id` string into plain shard id parts.
+ *
+ * Throws an `Error` when the string has no colon separator or the id segment is
+ * not numeric.
+ *
+ * @category decoding
  * @since 4.0.0
  */
 export function fromStringEncoded(s: string): {
@@ -113,6 +155,12 @@ export function fromStringEncoded(s: string): {
 }
 
 /**
+ * Parses a `group:id` string into a cached `ShardId`.
+ *
+ * Throws an `Error` when the string has no colon separator or the id segment is
+ * not numeric.
+ *
+ * @category decoding
  * @since 4.0.0
  */
 export function fromString(s: string): ShardId {

package/src/unstable/cluster/Sharding.ts

@@ -1,4 +1,26 @@
 /**
+ * The `Sharding` module coordinates cluster-wide placement and delivery for
+ * entities and singletons. It hashes entity ids into shard ids, tracks which
+ * runner owns each shard, acquires local shard locks, and routes RPC messages
+ * to the runner that is responsible for the addressed entity.
+ *
+ * Use this module when building clustered services that need location
+ * transparency for stateful entities, singleton workloads that should run once
+ * per shard group, or durable message processing backed by cluster storage.
+ * Registered entity handlers are started on demand for shards owned by the
+ * current runner, while clients produced by the {@link Sharding} service route
+ * requests through the sharding service instead of calling handlers directly.
+ *
+ * **Gotchas**
+ *
+ * - Shard assignment and shard acquisition are distinct: a runner may be
+ *   assigned a shard before it has acquired the storage lock for that shard.
+ * - Routing depends on the entity shard group and the configured shard count,
+ *   so changing either value affects where entity ids are placed.
+ * - Persisted messages are only read and dispatched for shards currently owned
+ *   by the local runner; shutdown and runner health changes can temporarily
+ *   move work between runners.
+ *
  * @since 4.0.0
  */
 import * as Arr from "../../Array.ts"
@@ -56,14 +78,18 @@ import { Runners } from "./Runners.ts"
 import { RunnerStorage } from "./RunnerStorage.ts"
 import type { ShardId } from "./ShardId.ts"
 import { make as makeShardId } from "./ShardId.ts"
-import { ShardingConfig } from "./ShardingConfig.ts"
+import { shardGroupConfig, ShardingConfig } from "./ShardingConfig.ts"
 import { EntityRegistered, type ShardingRegistrationEvent, SingletonRegistered } from "./ShardingRegistrationEvent.ts"
 import { SingletonAddress } from "./SingletonAddress.ts"
 import * as Snowflake from "./Snowflake.ts"
 
 /**
- * @since 4.0.0
+ * Cluster sharding service for registering entities and singletons, routing
+ * messages to owned shards, generating runner-local snowflake ids, and polling
+ * storage for persisted work.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class Sharding extends Context.Service<Sharding, {
   /**
@@ -199,6 +225,7 @@ interface EntityManagerState {
 
 const make = Effect.gen(function*() {
   const config = yield* ShardingConfig
+  const shardGroups = shardGroupConfig(config)
   const getRunnerAddress = () => Option.getOrUndefined(config.runnerAddress)
   const clock = yield* Clock
 
@@ -871,7 +898,7 @@ const make = Effect.gen(function*() {
   const selfRunner = initialRunnerAddress ?
     new Runner({
       address: initialRunnerAddress,
-      groups: config.shardGroups,
+      groups: Array.from(shardGroups.assigned),
       weight: config.runnerShardWeight
     }) :
     undefined
@@ -1430,8 +1457,12 @@ const make = Effect.gen(function*() {
 })
 
 /**
- * @since 4.0.0
+ * Layer that constructs the `Sharding` service from sharding configuration,
+ * runner communication, message storage, runner storage, runner health, the
+ * snowflake generator, and the entity reaper.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layer: Layer.Layer<
   Sharding,

package/src/unstable/cluster/ShardingConfig.ts

@@ -1,4 +1,19 @@
 /**
+ * The `ShardingConfig` module defines the configuration used by a cluster
+ * runner to participate in Effect Cluster sharding. It describes how a runner is
+ * addressed by other runners, which shard groups it can host, how many shards
+ * are assigned per group, and the timing settings used for locks, assignment
+ * refreshes, health checks, entity lifecycle, and message polling.
+ *
+ * Use this module when wiring a sharded application locally with
+ * {@link layer}, loading deployment settings from environment variables with
+ * {@link layerFromEnv}, or overriding selected defaults for tests and
+ * single-node development. In production, keep cluster-wide values such as
+ * `shardsPerGroup` and shard groups consistent across runners, choose stable
+ * externally reachable runner addresses, and tune lock expiration and refresh
+ * intervals to match the storage backend and shutdown behavior of the
+ * deployment platform.
+ *
  * @since 4.0.0
  */
 import * as Config from "../../Config.ts"
@@ -14,8 +29,8 @@ import { RunnerAddress } from "./RunnerAddress.ts"
 /**
  * Represents the configuration for the `Sharding` service on a given runner.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export class ShardingConfig extends Context.Service<ShardingConfig, {
   /**
@@ -42,12 +57,18 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
    * shards as a runner with a weight of `1`.
    */
   readonly runnerShardWeight: number
+  /**
+   * The shard groups available across all runners.
+   *
+   * Defaults to `["default"]`.
+   */
+  readonly availableShardGroups: ReadonlyArray<string>
   /**
    * The shard groups that are assigned to this runner.
    *
    * Defaults to `["default"]`.
    */
-  readonly shardGroups: ReadonlyArray<string>
+  readonly assignedShardGroups: ReadonlyArray<string>
   /**
    * The number of shards to allocate per shard group.
    *
@@ -125,15 +146,20 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
 const defaultRunnerAddress = RunnerAddress.make({ host: "localhost", port: 34431 })
 
 /**
- * @since 4.0.0
+ * Default values for `ShardingConfig`, including the default local runner address,
+ * shard group, shard count, mailbox settings, polling intervals, and remote
+ * serialization simulation.
+ *
  * @category defaults
+ * @since 4.0.0
  */
 export const defaults: ShardingConfig["Service"] = {
   runnerAddress: Option.some(defaultRunnerAddress),
   runnerListenAddress: Option.none(),
   runnerShardWeight: 1,
   shardsPerGroup: 300,
-  shardGroups: ["default"],
+  availableShardGroups: ["default"],
+  assignedShardGroups: ["default"],
   preemptiveShutdown: true,
   shardLockRefreshInterval: Duration.seconds(10),
   shardLockExpiration: Duration.seconds(35),
@@ -151,21 +177,29 @@ export const defaults: ShardingConfig["Service"] = {
 }
 
 /**
+ * Creates a `ShardingConfig` layer by merging the provided partial options over
+ * `defaults`.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer = (options?: Partial<ShardingConfig["Service"]>): Layer.Layer<ShardingConfig> =>
   Layer.succeed(ShardingConfig)({ ...defaults, ...options })
 
 /**
- * @since 4.0.0
+ * Layer that provides the default `ShardingConfig` values.
+ *
  * @category defaults
+ * @since 4.0.0
  */
 export const layerDefaults: Layer.Layer<ShardingConfig> = layer()
 
 /**
- * @since 4.0.0
+ * Config descriptor for loading `ShardingConfig` values, applying the same
+ * defaults used by the in-memory `defaults` object.
+ *
  * @category Config
+ * @since 4.0.0
  */
 export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
   runnerAddress: Config.all({
@@ -190,7 +224,11 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
     Config.withDefault(defaults.runnerShardWeight)
     // Config.withDescription("A number that determines how many shards this runner will be assigned relative to other runners.")
   ),
-  shardGroups: Config.schema(Schema.Array(Schema.String), "shardGroups").pipe(
+  availableShardGroups: Config.schema(Schema.Array(Schema.String), "availableShardGroups").pipe(
+    Config.withDefault(["default"])
+    // Config.withDescription("The shard groups available across all runners.")
+  ),
+  assignedShardGroups: Config.schema(Schema.Array(Schema.String), "shardGroups").pipe(
     Config.withDefault(["default"])
     // Config.withDescription("The shard groups that are assigned to this runner.")
   ),
@@ -260,8 +298,11 @@ export const config: Config.Config<ShardingConfig["Service"]> = Config.all({
 })
 
 /**
- * @since 4.0.0
+ * Effect that loads `ShardingConfig` from environment variables using the
+ * constant-case config provider.
+ *
  * @category Config
+ * @since 4.0.0
  */
 export const configFromEnv = config.pipe(
   Effect.provideService(
@@ -273,8 +314,11 @@ export const configFromEnv = config.pipe(
 )
 
 /**
+ * Layer that loads `ShardingConfig` from environment variables and, when options
+ * are provided, overlays those options on top of the loaded values.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerFromEnv = (options?: Partial<ShardingConfig["Service"]> | undefined): Layer.Layer<
   ShardingConfig,
@@ -283,3 +327,24 @@ export const layerFromEnv = (options?: Partial<ShardingConfig["Service"]> | unde
   Layer.effect(ShardingConfig)(
     options ? Effect.map(configFromEnv, (config) => ({ ...config, ...options })) : configFromEnv
   )
+
+/**
+ * Normalizes the provided `ShardingConfig` to calculate the `available` and
+ * `assigned` shard groups.
+ *
+ * @category Shard groups
+ * @since 4.0.0
+ */
+export const shardGroupConfig = (config: ShardingConfig["Service"]): {
+  readonly available: ReadonlySet<string>
+  readonly assigned: ReadonlySet<string>
+} => {
+  const available = new Set(config.availableShardGroups.slice().sort())
+  const assigned = new Set<string>()
+  available.forEach((group) => {
+    if (config.assignedShardGroups.includes(group)) {
+      assigned.add(group)
+    }
+  })
+  return { available, assigned }
+}

package/src/unstable/cluster/ShardingRegistrationEvent.ts

@@ -1,4 +1,18 @@
 /**
+ * The `ShardingRegistrationEvent` module defines the events emitted by
+ * `Sharding` when the local runner registers entity handlers or singleton
+ * workloads. These events are useful for observing the set of capabilities a
+ * runner has made available, coordinating startup hooks, and writing tests or
+ * integrations that need to react when registrations are complete.
+ *
+ * Registration events describe local registration, not shard ownership or
+ * execution. A runner may register an entity or singleton before it owns the
+ * shard that will run it, and the events are in-memory notifications from the
+ * `Sharding` service rather than persisted cluster state. For persisted
+ * messages, treat registration as the point where the handler is available to
+ * the runner; it does not imply that existing storage work has already been
+ * read or processed.
+ *
  * @since 4.0.0
  */
 import * as Data from "../../Data.ts"
@@ -8,8 +22,8 @@ import type { SingletonAddress } from "./SingletonAddress.ts"
 /**
  * Represents events that can occur when a runner registers entities or singletons.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export type ShardingRegistrationEvent =
   | EntityRegistered
@@ -18,8 +32,8 @@ export type ShardingRegistrationEvent =
 /**
  * Represents an event that occurs when a new entity is registered with a runner.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export interface EntityRegistered {
   readonly _tag: "EntityRegistered"
@@ -30,8 +44,8 @@ export interface EntityRegistered {
  * Represents an event that occurs when a new singleton is registered with a
  * runner.
  *
- * @since 4.0.0
  * @category models
+ * @since 4.0.0
  */
 export interface SingletonRegistered {
   readonly _tag: "SingletonRegistered"
@@ -39,23 +53,33 @@ export interface SingletonRegistered {
 }
 
 /**
- * @since 4.0.0
+ * Generated helpers for pattern matching and constructing sharding registration
+ * events.
+ *
  * @category pattern matching
+ * @since 4.0.0
  */
 export const {
   /**
-   * @since 4.0.0
+   * Pattern matches on a sharding registration event and dispatches to the
+   * matching variant handler.
+   *
    * @category pattern matching
+   * @since 4.0.0
    */
   $match: match,
   /**
-   * @since 4.0.0
+   * Creates an event for an entity registered by the local runner.
+   *
    * @category constructors
+   * @since 4.0.0
    */
   EntityRegistered,
   /**
-   * @since 4.0.0
+   * Creates an event for a singleton registered by the local runner.
+   *
    * @category constructors
+   * @since 4.0.0
    */
   SingletonRegistered
 } = Data.taggedEnum<ShardingRegistrationEvent>()

package/src/unstable/cluster/SingleRunner.ts

@@ -1,4 +1,26 @@
 /**
+ * The `SingleRunner` module provides a ready-to-use layer for running the
+ * cluster sharding services in a single process. It wires together sharding,
+ * message storage, runner registration, runner health, and sharding
+ * configuration so durable entities and workflows can run without a fleet of
+ * external runners.
+ *
+ * **Common tasks**
+ *
+ * - Start a local or embedded cluster runner backed by SQL message storage
+ * - Run durable entities and workflows in development, tests, or small
+ *   single-node deployments
+ * - Choose SQL runner storage for persistence or in-memory runner storage for
+ *   short-lived scenarios
+ * - Override sharding configuration while still using the standard
+ *   environment-based defaults
+ *
+ * **Gotchas**
+ *
+ * - The layer still requires a `SqlClient` because message storage is SQL-backed
+ * - Runner health and runner coordination are no-op implementations, so this is
+ *   for single-node use rather than multi-runner cluster coordination
+ *
  * @since 4.0.0
  */
 import * as Layer from "effect/Layer"
@@ -17,8 +39,8 @@ import * as SqlRunnerStorage from "./SqlRunnerStorage.ts"
  * A sql backed single-node cluster, that can be used for running durable
  * entities and workflows.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer = (options?: {
   readonly shardingConfig?: Partial<ShardingConfig.ShardingConfig["Service"]> | undefined

package/src/unstable/cluster/Singleton.ts

@@ -1,4 +1,15 @@
 /**
+ * The `Singleton` module provides a small helper for registering effects that
+ * should run once across an Effect cluster. A singleton is coordinated through
+ * `Sharding`, which assigns ownership to one node at a time and can move that
+ * ownership when nodes leave or fail.
+ *
+ * Use singletons for cluster-wide background work such as schedulers, polling
+ * loops, maintenance jobs, or consumers that must not have one instance per
+ * process. Because ownership can change during failover, the registered effect
+ * should be interruptible, scoped, and able to resume work without assuming that
+ * the previous owner completed every in-flight action exactly once.
+ *
  * @since 4.0.0
  */
 import * as Effect from "../../Effect.ts"
@@ -7,8 +18,11 @@ import type { Scope } from "../../Scope.ts"
 import { Sharding } from "./Sharding.ts"
 
 /**
- * @since 4.0.0
+ * Creates a layer that registers a singleton effect with `Sharding` under the
+ * specified name and optional shard group.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const make = <E, R>(
   name: string,

package/src/unstable/cluster/SingletonAddress.ts

@@ -1,4 +1,18 @@
 /**
+ * The `SingletonAddress` module defines the address used by cluster sharding to
+ * identify a registered singleton. A singleton address combines the singleton
+ * name with the `ShardId` that owns it, giving the runtime a stable key for
+ * registration events, equality checks, hashing, and runner-local fiber
+ * tracking.
+ *
+ * Use this module when observing singleton registrations or working with
+ * sharding internals that need to tell which shard currently owns a singleton.
+ * The shard id is derived from the singleton name and shard group at
+ * registration time, so changing either value changes ownership and routing.
+ * Ownership can also move as shard locks are acquired or released, so an address
+ * identifies the target shard rather than guaranteeing that a particular runner
+ * is currently executing the singleton.
+ *
  * @since 4.0.0
  */
 import * as Equal from "../../Equal.ts"
@@ -11,24 +25,30 @@ const TypeId = "~effect/cluster/SingletonAddress"
 /**
  * Represents the unique address of an singleton within the cluster.
  *
- * @since 4.0.0
  * @category Address
+ * @since 4.0.0
  */
 export class SingletonAddress extends Schema.Class<SingletonAddress>(TypeId)({
   shardId: ShardId,
   name: Schema.String
 }) {
   /**
+   * Marks this value as a cluster singleton address for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId;
   /**
+   * Computes a structural hash from the singleton name and shard id.
+   *
    * @since 4.0.0
    */
   [Hash.symbol]() {
     return Hash.string(`${this.name}:${this.shardId.toString()}`)
   }
   /**
+   * Compares singleton addresses by name and shard id.
+   *
    * @since 4.0.0
    */
   [Equal.symbol](that: SingletonAddress): boolean {