effect 4.0.0-beta.66 → 4.0.0-beta.68

package/src/unstable/cluster/MessageStorage.ts

@@ -1,4 +1,28 @@
 /**
+ * The `MessageStorage` module defines the persistence boundary used by Effect
+ * Cluster to store mailbox messages and replies. Storage implementations keep
+ * requests, envelopes, and reply chunks durable enough for runners to recover
+ * work after restarts, replay unprocessed messages for assigned shards, and
+ * deliver replies back to locally registered handlers.
+ *
+ * **Common use cases**
+ *
+ * - Persist outgoing requests and control envelopes before delivery
+ * - Detect duplicate requests by primary key and resume from an existing reply
+ * - Query unprocessed messages when shards are assigned to a runner
+ * - Store, load, and clear replies for request streams and completions
+ * - Reset or clear mailbox state during shard or address lifecycle changes
+ *
+ * **Gotchas**
+ *
+ * - Implementations should make save and reply operations transactional when
+ *   possible so recovery does not observe partial mailbox state
+ * - Duplicate detection depends on stable request primary keys and persisted
+ *   request ids
+ * - Reply handlers are local process state; persisted replies may need to be
+ *   loaded again after restarts or reassignment
+ * - Concurrent runners must only process messages for shards they currently own
+ *
  * @since 4.0.0
  */
 import * as Arr from "../../Array.ts"
@@ -25,8 +49,14 @@ import type { ShardingConfig } from "./ShardingConfig.ts"
 import * as Snowflake from "./Snowflake.ts"
 
 /**
- * @since 4.0.0
+ * Service for cluster mailbox persistence and reply delivery.
+ *
+ * It stores outgoing requests, control envelopes, and replies; reads unprocessed
+ * messages; manages reply handlers; and provides transaction wrapping for storage
+ * operations.
+ *
  * @category context
+ * @since 4.0.0
  */
 export class MessageStorage extends Context.Service<MessageStorage, {
   /**
@@ -151,45 +181,68 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 }>()("effect/cluster/MessageStorage") {}
 
 /**
- * @since 4.0.0
+ * Result of saving a request or envelope into message storage.
+ *
+ * A duplicate result carries the original request ID and the last reply already
+ * received for the duplicated request.
+ *
  * @category SaveResult
+ * @since 4.0.0
  */
 export type SaveResult<R extends Rpc.Any> = SaveResult.Success | SaveResult.Duplicate<R>
 
 /**
- * @since 4.0.0
+ * Tagged enum constructor for creating and matching `SaveResult` values.
+ *
  * @category SaveResult
+ * @since 4.0.0
  */
 export const SaveResult = Data.taggedEnum<SaveResult.Constructor>()
 
 /**
- * @since 4.0.0
+ * Tagged enum constructor for encoded save results returned by encoded storage
+ * drivers.
+ *
  * @category SaveResult
+ * @since 4.0.0
  */
 export const SaveResultEncoded = Data.taggedEnum<SaveResult.Encoded>()
 
 /**
+ * Variants and helper types for `SaveResult`.
+ *
  * @since 4.0.0
- * @category SaveResult
  */
 export declare namespace SaveResult {
   /**
-   * @since 4.0.0
+   * Encoded storage-driver form of `SaveResult`.
+   *
+   * Duplicate results contain an encoded last received reply instead of a decoded
+   * reply.
+   *
    * @category SaveResult
+   * @since 4.0.0
    */
   export type Encoded = SaveResult.Success | SaveResult.DuplicateEncoded
 
   /**
-   * @since 4.0.0
+   * Variant indicating that the message was saved as a new storage entry.
+   *
    * @category SaveResult
+   * @since 4.0.0
    */
   export interface Success {
     readonly _tag: "Success"
   }
 
   /**
-   * @since 4.0.0
+   * Variant indicating that the request duplicates an existing stored request.
+   *
+   * It carries the original request ID and the latest decoded reply, when one is
+   * available.
+   *
    * @category SaveResult
+   * @since 4.0.0
    */
   export interface Duplicate<R extends Rpc.Any> {
     readonly _tag: "Duplicate"
@@ -198,8 +251,13 @@ export declare namespace SaveResult {
   }
 
   /**
-   * @since 4.0.0
+   * Encoded duplicate-save variant returned by lower-level storage drivers.
+   *
+   * It carries the original request ID and the latest encoded reply, when one is
+   * available.
+   *
    * @category SaveResult
+   * @since 4.0.0
    */
   export interface DuplicateEncoded {
     readonly _tag: "Duplicate"
@@ -208,8 +266,10 @@ export declare namespace SaveResult {
   }
 
   /**
-   * @since 4.0.0
+   * Generic tagged enum constructor type for `SaveResult`.
+   *
    * @category SaveResult
+   * @since 4.0.0
    */
   export interface Constructor extends Data.TaggedEnum.WithGenerics<1> {
     readonly taggedEnum: SaveResult<this["A"] extends Rpc.Any ? this["A"] : never>
@@ -217,8 +277,13 @@ export declare namespace SaveResult {
 }
 
 /**
- * @since 4.0.0
+ * Low-level storage-driver contract for encoded envelopes and replies.
+ *
+ * Implementations persist encoded messages, track primary keys and delayed
+ * delivery, read unprocessed messages, and provide transaction wrapping.
+ *
  * @category Encoded
+ * @since 4.0.0
  */
 export type Encoded = {
   /**
@@ -333,8 +398,13 @@ export type Encoded = {
 }
 
 /**
- * @since 4.0.0
+ * Cursor options for reading encoded unprocessed messages across shard sets.
+ *
+ * The fields distinguish existing shards from newly assigned shards and carry the
+ * driver-specific pagination cursor.
+ *
  * @category Encoded
+ * @since 4.0.0
  */
 export type EncodedUnprocessedOptions<A> = {
   readonly existingShards: Array<number>
@@ -343,8 +413,13 @@ export type EncodedUnprocessedOptions<A> = {
 }
 
 /**
- * @since 4.0.0
+ * Cursor options for reading encoded replies across request sets.
+ *
+ * The fields distinguish existing requests from new requests and carry the
+ * driver-specific pagination cursor.
+ *
  * @category Encoded
+ * @since 4.0.0
  */
 export type EncodedRepliesOptions<A> = {
   readonly existingRequests: Array<string>
@@ -353,8 +428,13 @@ export type EncodedRepliesOptions<A> = {
 }
 
 /**
- * @since 4.0.0
+ * Wraps a concrete message storage implementation with reply-handler management.
+ *
+ * The returned service can register waiting reply handlers, notify them when
+ * replies are saved, and fail them when a request or shard is unregistered.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const make = (
   storage: Omit<
@@ -455,8 +535,14 @@ export const make = (
   })
 
 /**
- * @since 4.0.0
+ * Builds a `MessageStorage` service from an encoded storage driver.
+ *
+ * The adapter handles envelope and reply encoding and decoding, primary-key
+ * generation, delayed delivery checks, duplicate decoding, and malformed-message
+ * defect replies.
+ *
  * @category constructors
+ * @since 4.0.0
  */
 export const makeEncoded: (encoded: Encoded) => Effect.Effect<
   MessageStorage["Service"],
@@ -663,8 +749,10 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
 })
 
 /**
+ * No-op `MessageStorage` service that does not persist messages or replies.
+ *
+ * @category constructors
  * @since 4.0.0
- * @category Constructors
  */
 export const noop: MessageStorage["Service"] = Effect.runSync(make({
   saveRequest: () => Effect.succeed(SaveResult.Success()),
@@ -683,8 +771,13 @@ export const noop: MessageStorage["Service"] = Effect.runSync(make({
 }))
 
 /**
- * @since 4.0.0
+ * In-memory storage entry for a request envelope.
+ *
+ * It stores the encoded envelope, last acknowledged chunk, accumulated replies,
+ * and optional delivery time.
+ *
  * @category Memory
+ * @since 4.0.0
  */
 export type MemoryEntry = {
   readonly envelope: Envelope.Encoded
@@ -696,16 +789,22 @@ export type MemoryEntry = {
 /**
  * Can be used in tests to simulate a transaction.
  *
- * @since 4.0.0
  * @category Memory
+ * @since 4.0.0
  */
 export const MemoryTransaction = Context.Reference<boolean>("effect/cluster/MessageStorage/MemoryTransaction", {
   defaultValue: constFalse
 })
 
 /**
- * @since 4.0.0
+ * In-memory message storage driver with inspectable backing state.
+ *
+ * It provides a `MessageStorage` service, the encoded driver implementation, and
+ * maps used to track requests, primary keys, unprocessed envelopes, reply IDs,
+ * and the journal.
+ *
  * @category Memory
+ * @since 4.0.0
  */
 export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluster/MessageStorage/MemoryDriver", {
   make: Effect.gen(function*() {
@@ -908,6 +1007,8 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
   })
 }) {
   /**
+   * Layer that provides the in-memory message storage driver.
+   *
    * @since 4.0.0
    */
   static readonly layer: Layer.Layer<MemoryDriver> = Layer.effect(this)(this.make).pipe(
@@ -916,14 +1017,18 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
 }
 
 /**
- * @since 4.0.0
+ * Layer that provides the no-op `MessageStorage` service.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerNoop: Layer.Layer<MessageStorage> = Layer.succeed(MessageStorage, noop)
 
 /**
- * @since 4.0.0
+ * Layer that provides in-memory message storage and its backing `MemoryDriver`.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerMemory: Layer.Layer<
   MessageStorage | MemoryDriver,

package/src/unstable/cluster/Reply.ts

@@ -1,4 +1,25 @@
 /**
+ * The `Reply` module models responses produced by clustered RPC execution. A
+ * reply belongs to a request and is either a terminal {@link WithExit}, which
+ * carries the final RPC `Exit`, or a streaming {@link Chunk}, which carries a
+ * non-empty batch of success values for RPCs that stream results.
+ *
+ * **Common tasks**
+ *
+ * - Represent runtime replies with {@link Reply}, {@link WithExit}, and {@link Chunk}
+ * - Encode and decode transport payloads with {@link Encoded} and {@link Reply}
+ * - Persist replies together with schema context via {@link ReplyWithContext}
+ * - Serialize the latest received reply when resuming or de-duplicating requests with {@link serializeLastReceived}
+ *
+ * **Streaming and acknowledgement notes**
+ *
+ * - Chunk replies are sequenced and can be replayed until acknowledged by the
+ *   receiver.
+ * - A `WithExit` reply is terminal and completes the request, while chunks only
+ *   represent intermediate streamed success values.
+ * - `Chunk.emptyFrom` is used as an acknowledgement marker for an empty streamed
+ *   reply; it is not a general-purpose success payload.
+ *
  * @since 4.0.0
  */
 import type { NonEmptyReadonlyArray } from "../../Array.ts"
@@ -22,32 +43,49 @@ import { Snowflake, SnowflakeFromBigInt } from "./Snowflake.ts"
 const TypeId = "~effect/cluster/Reply"
 
 /**
- * @since 4.0.0
+ * Returns `true` when the supplied value is a runtime cluster reply.
+ *
+ * The check is based on the reply type identifier.
+ *
  * @category guards
+ * @since 4.0.0
  */
 export const isReply = (u: unknown): u is Reply<Rpc.Any> => hasProperty(u, TypeId)
 
 /**
- * @since 4.0.0
+ * Runtime reply sent for an RPC request.
+ *
+ * A reply is either a final exit or a chunk of a streaming success value.
+ *
  * @category models
+ * @since 4.0.0
  */
 export type Reply<R extends Rpc.Any> = WithExit<R> | Chunk<R>
 
 /**
- * @since 4.0.0
+ * JSON-serializable form of a cluster reply.
+ *
  * @category models
+ * @since 4.0.0
  */
 export type Encoded = WithExitEncoded | ChunkEncoded
 
 /**
- * @since 4.0.0
+ * Codec used for reply values that are already in encoded form.
+ *
+ * Per-RPC payload validation is performed by `Reply(rpc)`.
+ *
  * @category models
+ * @since 4.0.0
  */
 export const Encoded: Schema.Codec<Encoded> = Schema.Any as any
 
 /**
- * @since 4.0.0
+ * A cluster reply paired with the RPC definition and service context required to
+ * serialize it for transport.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("ReplyWithContext")<{
   readonly reply: Reply<R>
@@ -55,6 +93,8 @@ export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("Reply
   readonly rpc: R
 }> {
   /**
+   * Creates a terminal reply context that dies with the supplied defect.
+   *
    * @since 4.0.0
    */
   static fromDefect(options: {
@@ -73,6 +113,8 @@ export class ReplyWithContext<R extends Rpc.Any> extends Data.TaggedClass("Reply
     })
   }
   /**
+   * Creates a terminal reply context that interrupts the supplied request.
+   *
    * @since 4.0.0
    */
   static interrupt(options: {
@@ -98,8 +140,11 @@ const neverRpc = Rpc.make("Never", {
 })
 
 /**
- * @since 4.0.0
+ * Wire-format representation of a terminal reply containing the request id, reply
+ * id, and encoded RPC exit value.
+ *
  * @category models
+ * @since 4.0.0
  */
 export interface WithExitEncoded<A = unknown, E = unknown> {
   readonly _tag: "WithExit"
@@ -109,8 +154,11 @@ export interface WithExitEncoded<A = unknown, E = unknown> {
 }
 
 /**
- * @since 4.0.0
+ * Wire-format representation of a streaming reply chunk, including the request id,
+ * reply id, sequence number, and non-empty encoded values.
+ *
  * @category models
+ * @since 4.0.0
  */
 export interface ChunkEncoded {
   readonly _tag: "Chunk"
@@ -123,8 +171,11 @@ export interface ChunkEncoded {
 const schemaCache = new WeakMap<Rpc.Any, Schema.Top>()
 
 /**
- * @since 4.0.0
+ * A streaming RPC reply chunk for a request, carrying a non-empty batch of success
+ * values together with the reply id and sequence number.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   readonly requestId: Snowflake
@@ -133,11 +184,15 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   readonly values: NonEmptyReadonlyArray<Rpc.SuccessChunk<R>>
 }> {
   /**
+   * Marks this value as a runtime cluster reply.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Creates an empty chunk reply for the supplied request id.
+   *
    * @since 4.0.0
    */
   static emptyFrom(requestId: Snowflake) {
@@ -150,11 +205,15 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Schema that accepts any runtime chunk reply without validating payload values.
+   *
    * @since 4.0.0
    */
   static readonly Any = Schema.declare((u): u is Chunk<never> => isReply(u) && u._tag === "Chunk")
 
   /**
+   * Transformation between encoded chunk records and `Chunk` instances.
+   *
    * @since 4.0.0
    */
   static readonly transform: Transformation.Transformation<any, any> = Transformation.transform({
@@ -163,6 +222,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   })
 
   /**
+   * Builds a chunk schema from the streaming success schema of an RPC.
+   *
    * @since 4.0.0
    */
   static schema<R extends Rpc.Any>(
@@ -176,6 +237,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Builds a chunk schema that validates each success value with the supplied schema.
+   *
    * @since 4.0.0
    */
   static schemaFrom<Success extends Schema.Top>(
@@ -214,6 +277,8 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
   }
 
   /**
+   * Returns a copy of this chunk associated with the supplied request id.
+   *
    * @since 4.0.0
    */
   withRequestId(requestId: Snowflake): Chunk<R> {
@@ -225,8 +290,11 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
 }
 
 /**
- * @since 4.0.0
+ * A terminal RPC reply for a request, carrying the final `Exit` for the remote
+ * call.
+ *
  * @category models
+ * @since 4.0.0
  */
 export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   readonly requestId: Snowflake
@@ -234,11 +302,15 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   readonly exit: Rpc.Exit<R>
 }> {
   /**
+   * Marks this value as a runtime cluster reply.
+   *
    * @since 4.0.0
    */
   readonly [TypeId] = TypeId
 
   /**
+   * Returns `true` when the value is a terminal `WithExit` reply.
+   *
    * @since 4.0.0
    */
   static is(u: unknown): u is WithExit<any> {
@@ -246,6 +318,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Builds a terminal reply schema from the exit schema of an RPC.
+   *
    * @since 4.0.0
    */
   static schema<R extends Rpc.Any>(
@@ -259,6 +333,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Builds a terminal reply schema that validates the encoded exit value.
+   *
    * @since 4.0.0
    */
   static schemaFrom<Success extends Schema.Top, Error extends Schema.Top, Defect extends Schema.Top>(
@@ -300,6 +376,8 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
   }
 
   /**
+   * Returns a copy of this terminal reply associated with the supplied request id.
+   *
    * @since 4.0.0
    */
   withRequestId(requestId: Snowflake): WithExit<R> {
@@ -311,8 +389,11 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
 }
 
 /**
- * @since 4.0.0
+ * Builds the transport codec for replies to the specified RPC, covering terminal
+ * `WithExit` replies and streaming `Chunk` replies.
+ *
  * @category schemas
+ * @since 4.0.0
  */
 export const Reply = <R extends Rpc.Any>(
   rpc: R
@@ -331,8 +412,12 @@ export const Reply = <R extends Rpc.Any>(
 }
 
 /**
- * @since 4.0.0
+ * Serializes a `ReplyWithContext` into its encoded wire representation, using the
+ * reply's RPC schema and context and refailing encoding errors as
+ * `MalformedMessage`.
+ *
  * @category serialization / deserialization
+ * @since 4.0.0
  */
 export const serialize = <R extends Rpc.Any>(
   self: ReplyWithContext<R>
@@ -347,8 +432,12 @@ export const serialize = <R extends Rpc.Any>(
 }
 
 /**
- * @since 4.0.0
+ * Serializes an outgoing request's last received reply when one exists, returning
+ * `None` when no reply has been received and refailing encoding errors as
+ * `MalformedMessage`.
+ *
  * @category serialization / deserialization
+ * @since 4.0.0
  */
 export const serializeLastReceived = <R extends Rpc.Any>(
   self: OutgoingRequest<R>