effect 4.0.0-beta.66 → 4.0.0-beta.68

package/src/unstable/cluster/Snowflake.ts

@@ -1,4 +1,22 @@
 /**
+ * The `Snowflake` module provides compact, sortable identifiers for cluster
+ * resources and events. A snowflake id is a branded `bigint` made from a
+ * millisecond timestamp, a machine id, and a per-machine sequence number.
+ *
+ * **Common use cases**
+ *
+ * - Creating ids without coordinating through a central database
+ * - Ordering cluster events, entity ids, or log records by generation time
+ * - Encoding ids as strings at service boundaries with {@link SnowflakeFromString}
+ * - Decoding a generated id into timestamp, machine id, and sequence parts with {@link toParts}
+ *
+ * **Gotchas**
+ *
+ * - Uniqueness depends on each concurrent generator using a distinct machine id
+ * - Generated ids are time-sortable, but they are not random or secret values
+ * - The default generator prevents local clock drift from moving ids backward
+ * - More than 4096 ids in the same millisecond advance the logical timestamp
+ *
  * @since 4.0.0
  */
 import type * as Brand from "../../Brand.ts"
@@ -13,36 +31,51 @@ import * as Transformation from "../../SchemaTransformation.ts"
 import type { MachineId } from "./MachineId.ts"
 
 /**
+ * Runtime brand identifier for cluster snowflake ids.
+ *
+ * @category type IDs
  * @since 4.0.0
  */
 export const TypeId = "~effect/cluster/Snowflake"
 
 /**
+ * Type-level representation of the cluster snowflake brand identifier.
+ *
+ * @category type IDs
  * @since 4.0.0
  */
 export type TypeId = typeof TypeId
 
 /**
+ * Branded bigint identifier composed from a timestamp, machine id, and per-machine
+ * sequence number.
+ *
+ * @category models
  * @since 4.0.0
- * @category Models
  */
 export type Snowflake = Brand.Branded<bigint, TypeId>
 
 /**
+ * Constructs a branded `Snowflake` from a bigint or bigint-compatible string.
+ *
+ * @category models
  * @since 4.0.0
- * @category Models
  */
 export const Snowflake = (input: string | bigint): Snowflake =>
   typeof input === "string" ? BigInt(input) as Snowflake : input as Snowflake
 
 /**
+ * Namespace containing support types for snowflake parts and generators.
+ *
  * @since 4.0.0
- * @category Models
  */
 export declare namespace Snowflake {
   /**
+   * Decoded components of a snowflake id: Unix timestamp milliseconds, machine id,
+   * and sequence number.
+   *
+   * @category models
    * @since 4.0.0
-   * @category Models
    */
   export interface Parts {
     readonly timestamp: number
@@ -51,8 +84,11 @@ export declare namespace Snowflake {
   }
 
   /**
+   * Stateful generator for runner-local snowflake ids, exposing an unsafe
+   * synchronous `nextUnsafe` operation and an effectful machine id setter.
+   *
+   * @category models
    * @since 4.0.0
-   * @category Models
    */
   export interface Generator {
     readonly nextUnsafe: () => Snowflake
@@ -61,34 +97,45 @@ export declare namespace Snowflake {
 }
 
 /**
+ * Schema type for snowflake ids represented as branded bigints.
+ *
+ * @category schemas
  * @since 4.0.0
- * @category Schemas
  */
 export interface SnowflakeFromBigInt extends Schema.brand<Schema.BigInt, TypeId> {}
 
 /**
+ * Schema for snowflake ids represented as branded bigints.
+ *
+ * @category schemas
  * @since 4.0.0
- * @category Schemas
  */
 export const SnowflakeFromBigInt: SnowflakeFromBigInt = Schema.BigInt.pipe(Schema.brand(TypeId))
 
 /**
+ * Schema type for snowflake ids decoded from strings into branded bigints.
+ *
+ * @category schemas
  * @since 4.0.0
- * @category Schemas
  */
 export interface SnowflakeFromString extends Schema.decodeTo<SnowflakeFromBigInt, Schema.String> {}
 
 /**
+ * Schema that decodes snowflake ids from strings into branded bigints and encodes
+ * them back to strings.
+ *
+ * @category schemas
  * @since 4.0.0
- * @category Schemas
  */
 export const SnowflakeFromString: SnowflakeFromString = Schema.String.pipe(
   Schema.decodeTo(SnowflakeFromBigInt, Transformation.bigintFromString)
 )
 
 /**
- * @since 4.0.0
+ * Custom snowflake epoch in Unix milliseconds, set to January 1, 2025 UTC.
+ *
  * @category Epoch
+ * @since 4.0.0
  */
 export const constEpochMillis: number = Date.UTC(2025, 0, 1)
 
@@ -99,8 +146,12 @@ const constBigInt1024 = BigInt(1024)
 const constBigInt4096 = BigInt(4096)
 
 /**
- * @since 4.0.0
+ * Packs a timestamp, machine id, and sequence number into a branded snowflake id,
+ * using the custom snowflake epoch and 10-bit machine id and 12-bit sequence
+ * fields.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const make = (options: {
   readonly machineId: MachineId
@@ -112,33 +163,43 @@ export const make = (options: {
     | BigInt(options.sequence % 4096)) as Snowflake
 
 /**
- * @since 4.0.0
+ * Extracts the Unix timestamp in milliseconds from a snowflake id.
+ *
  * @category Parts
+ * @since 4.0.0
  */
 export const timestamp = (snowflake: Snowflake): number => Number(snowflake >> constBigInt22) + sinceUnixEpoch
 
 /**
- * @since 4.0.0
+ * Extracts the timestamp from a snowflake id as a `DateTime.Utc`.
+ *
  * @category Parts
+ * @since 4.0.0
  */
 export const dateTime = (snowflake: Snowflake): DateTime.Utc => DateTime.makeUnsafe(timestamp(snowflake))
 
 /**
- * @since 4.0.0
+ * Extracts the machine id component from a snowflake id.
+ *
  * @category Parts
+ * @since 4.0.0
  */
 export const machineId = (snowflake: Snowflake): MachineId =>
   Number((snowflake >> constBigInt12) % constBigInt1024) as MachineId
 
 /**
- * @since 4.0.0
+ * Extracts the per-machine sequence component from a snowflake id.
+ *
  * @category Parts
+ * @since 4.0.0
  */
 export const sequence = (snowflake: Snowflake): number => Number(snowflake % constBigInt4096)
 
 /**
- * @since 4.0.0
+ * Decomposes a snowflake id into its timestamp, machine id, and sequence parts.
+ *
  * @category Parts
+ * @since 4.0.0
  */
 export const toParts = (snowflake: Snowflake): Snowflake.Parts => ({
   timestamp: timestamp(snowflake),
@@ -147,8 +208,14 @@ export const toParts = (snowflake: Snowflake): Snowflake.Parts => ({
 })
 
 /**
- * @since 4.0.0
+ * Creates a stateful snowflake generator using `Clock`.
+ *
+ * The generator starts with a random machine id, never moves generated timestamps
+ * backward, resets the sequence each millisecond, and advances the timestamp when
+ * more than 4096 ids are requested in the same millisecond.
+ *
  * @category Generator
+ * @since 4.0.0
  */
 export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(function*() {
   let machineId = Math.floor(Math.random() * 1024) as MachineId
@@ -189,8 +256,10 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
 })
 
 /**
- * @since 4.0.0
+ * Context service for a stateful snowflake id generator.
+ *
  * @category Generator
+ * @since 4.0.0
  */
 export class Generator extends Context.Service<
   Generator,
@@ -198,7 +267,9 @@ export class Generator extends Context.Service<
 >()("effect/cluster/Snowflake/Generator") {}
 
 /**
- * @since 4.0.0
+ * Layer that provides the default snowflake `Generator` service.
+ *
  * @category Generator
+ * @since 4.0.0
  */
 export const layerGenerator: Layer.Layer<Generator> = Layer.effect(Generator)(makeGenerator)

package/src/unstable/cluster/SocketRunner.ts

@@ -1,4 +1,25 @@
 /**
+ * The `SocketRunner` module wires cluster runner RPCs to socket transports. It
+ * provides a complete runner layer that serves RPC handlers on a `SocketServer`
+ * and installs `Sharding` and `Runners` clients for talking to other runners
+ * through the socket RPC protocol.
+ *
+ * **Common tasks**
+ *
+ * - Run a cluster worker over TCP or Unix sockets with {@link layer}
+ * - Connect to other runners while exposing `Sharding` and `Runners` clients
+ * - Embed a client-only cluster participant with {@link layerClientOnly} when
+ *   the process should send messages but not receive shard assignments
+ *
+ * **Transport gotchas**
+ *
+ * - The server listen address comes from the provided `SocketServer` and is
+ *   logged when {@link layer} starts
+ * - TCP addresses are logged as `hostname:port`, while Unix socket addresses
+ *   are logged as their filesystem path
+ * - The client-only layer does not start a socket server; provide the full
+ *   layer when the process must accept runner RPCs
+ *
  * @since 4.0.0
  */
 import * as Effect from "../../Effect.ts"
@@ -27,8 +48,11 @@ const withLogAddress = <A, E, R>(layer: Layer.Layer<A, E, R>): Layer.Layer<A, E,
   })).pipe(Layer.provideMerge(layer))
 
 /**
+ * Layer that runs a cluster runner over the socket RPC protocol, providing
+ * `Sharding` and `Runners` clients and logging the socket listen address.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   Sharding.Sharding | Runners.Runners,
@@ -46,8 +70,11 @@ export const layer: Layer.Layer<
 )
 
 /**
+ * Client-only socket runner layer that provides `Sharding` and `Runners` clients
+ * without starting a runner server or receiving shard assignments.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerClientOnly: Layer.Layer<
   Sharding.Sharding | Runners.Runners,

package/src/unstable/cluster/SqlMessageStorage.ts

@@ -1,4 +1,20 @@
 /**
+ * SQL-backed message storage for the unstable cluster runtime.
+ *
+ * This module persists encoded cluster envelopes and replies in SQL tables so
+ * shards can resume work after process restarts, redeliver unprocessed messages,
+ * deduplicate requests by primary key, and replay outstanding reply chunks until
+ * they are acknowledged. It is the storage implementation to use when a cluster
+ * needs durable request / reply state backed by `SqlClient` rather than an
+ * in-memory store.
+ *
+ * The storage layer runs its own migrations and creates messages, replies, and
+ * migration tables using the configured prefix (`cluster` by default). Choose a
+ * stable prefix before deploying, because changing it points the runtime at a
+ * different set of tables. Existing deployments should also keep the generated
+ * migration history table with the message tables so future schema changes can
+ * be applied consistently across supported SQL dialects.
+ *
  * @since 4.0.0
  */
 // eslint-disable effect/no-bigint-literals
@@ -23,8 +39,11 @@ import * as Snowflake from "./Snowflake.ts"
 const withTracerDisabled = Effect.withTracerEnabled(false)
 
 /**
+ * Creates a SQL-backed `MessageStorage` implementation, running its migrations
+ * and using the optional table prefix.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const make: (options?: {
   readonly prefix?: string | undefined
@@ -603,8 +622,11 @@ export const make: (options?: {
 }, withTracerDisabled)
 
 /**
+ * Layer that provides SQL-backed `MessageStorage` using the default table prefix
+ * and the default snowflake generator.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   MessageStorage.MessageStorage,
@@ -615,8 +637,10 @@ export const layer: Layer.Layer<
 )
 
 /**
+ * Layer that provides SQL-backed `MessageStorage` using a custom table prefix.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerWith = (options: {
   readonly prefix?: string | undefined

package/src/unstable/cluster/SqlRunnerStorage.ts

@@ -1,4 +1,33 @@
 /**
+ * SQL-backed storage for Effect Cluster runner metadata and shard ownership.
+ *
+ * The `SqlRunnerStorage` module builds a `RunnerStorage` implementation from a
+ * `SqlClient`, creating the runner and lock tables it needs using the configured
+ * table prefix. It is used by clustered applications that need runner
+ * registration, health tracking, shard acquisition, refresh, and release to be
+ * coordinated through an external database instead of in-memory state.
+ *
+ * **Common tasks**
+ *
+ * - Provide the default storage layer with {@link layer}
+ * - Use {@link layerWith} when multiple clusters share the same database and
+ *   need distinct table prefixes
+ * - Create a storage implementation directly with {@link make} for custom layer
+ *   composition
+ *
+ * **Gotchas**
+ *
+ * - Runner heartbeats and persisted shard locks expire according to
+ *   `ShardingConfig.shardLockExpiration`; stale rows may be reused or cleaned up
+ *   by later storage operations.
+ * - PostgreSQL and MySQL use advisory locks by default, keeping shard ownership
+ *   tied to a reserved database connection. Set
+ *   `ShardingConfig.shardLockDisableAdvisory` when persisted lock rows should be
+ *   used instead.
+ * - The selected table prefix controls the generated `runners` and `locks`
+ *   table names, so changing it points the cluster at a different storage
+ *   namespace.
+ *
  * @since 4.0.0
  */
 import * as Arr from "../../Array.ts"
@@ -18,13 +47,19 @@ import * as ShardingConfig from "./ShardingConfig.ts"
 const withTracerDisabled = Effect.withTracerEnabled(false)
 
 /**
+ * Creates a SQL-backed `RunnerStorage` implementation for registered runners and
+ * shard locks, using the configured table prefix and advisory locks where
+ * supported and enabled.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const make = Effect.fnUntraced(function*(options: {
   readonly prefix?: string | undefined
 }) {
   const config = yield* ShardingConfig.ShardingConfig
+  const shardGroups = ShardingConfig.shardGroupConfig(config)
+  const availableShardGroups = Array.from(shardGroups.available)
   const disableAdvisoryLocks = config.shardLockDisableAdvisory
   const sql = (yield* SqlClient.SqlClient).withoutTransforms()
   const prefix = options?.prefix ?? "cluster"
@@ -391,8 +426,8 @@ export const make = Effect.fnUntraced(function*(options: {
 
   const lockNumbers = new Map<string, number>()
   const lockNumbersReverse = new Map<number, string>()
-  for (let i = 0; i < config.shardGroups.length; i++) {
-    const group = config.shardGroups[i]
+  for (let i = 0; i < availableShardGroups.length; i++) {
+    const group = availableShardGroups[i]
     const base = (i + 1) * 1000000
     for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
       const shardId = ShardId.make(group, shard).toString()
@@ -407,8 +442,8 @@ export const make = Effect.fnUntraced(function*(options: {
   const lockNamesReverse = new Map<string, string>()
   {
     let index = 0
-    for (let i = 0; i < config.shardGroups.length; i++) {
-      const group = config.shardGroups[i]
+    for (let i = 0; i < availableShardGroups.length; i++) {
+      const group = availableShardGroups[i]
       for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
         const shardId = ShardId.make(group, shard).toString()
         const lockName = `${prefix}.${shardId}`
@@ -643,8 +678,10 @@ export const make = Effect.fnUntraced(function*(options: {
 }, withTracerDisabled)
 
 /**
+ * Layer that provides SQL-backed `RunnerStorage` using the default table prefix.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   RunnerStorage.RunnerStorage,
@@ -653,8 +690,10 @@ export const layer: Layer.Layer<
 > = Layer.effect(RunnerStorage.RunnerStorage)(make({}))
 
 /**
+ * Layer that provides SQL-backed `RunnerStorage` using a custom table prefix.
+ *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layerWith = (options: {
   readonly prefix?: string | undefined

package/src/unstable/cluster/TestRunner.ts

@@ -1,4 +1,21 @@
 /**
+ * The `TestRunner` module provides a lightweight in-memory cluster layer for
+ * tests that need the cluster sharding services without starting real runners
+ * or relying on external storage.
+ *
+ * Use it when exercising sharding behavior, message storage, or code that
+ * depends on the cluster runner services in unit and integration tests. The
+ * layer wires the normal sharding service to in-memory message and runner
+ * storage, along with no-op runner and health implementations.
+ *
+ * **Testing gotchas**
+ *
+ * - State is held in memory and scoped to the layer lifetime; it is not shared
+ *   across independently constructed layers or persisted between test runs
+ * - Runner execution and health checks are no-ops, so this layer is best suited
+ *   for testing coordination and storage behavior rather than real distributed
+ *   runner processes
+ *
  * @since 4.0.0
  */
 import * as Layer from "../../Layer.ts"
@@ -15,8 +32,8 @@ import * as ShardingConfig from "./ShardingConfig.ts"
  * MessageStorage is backed by an in-memory driver, and RunnerStorage is backed
  * by an in-memory driver.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   Sharding.Sharding | Runners.Runners | MessageStorage.MessageStorage | MessageStorage.MemoryDriver