effect 4.0.0-beta.99 → 4.0.0-rc.108

package/src/unstable/cluster/Message.ts

@@ -31,7 +31,7 @@ import type { Snowflake } from "./Snowflake.ts"
  * An incoming message is either a persisted request with an encoded payload or an
  * incoming control envelope.
  *
- * @category incoming
+ * @category models
  * @since 4.0.0
  */
 export type Incoming<R extends Rpc.Any> = IncomingRequest<R> | IncomingEnvelope
@@ -43,7 +43,7 @@ export type Incoming<R extends Rpc.Any> = IncomingRequest<R> | IncomingEnvelope
  *
  * It is either a request with a decoded payload or an incoming control envelope.
  *
- * @category incoming
+ * @category models
  * @since 4.0.0
  */
 export type IncomingLocal<R extends Rpc.Any> = IncomingRequestLocal<R> | IncomingEnvelope
@@ -56,7 +56,7 @@ export type IncomingLocal<R extends Rpc.Any> = IncomingRequestLocal<R> | Incomin
  * Request messages keep their decoded payload and response callback, while
  * control envelopes are wrapped as incoming envelopes.
  *
- * @category incoming
+ * @category converting
  * @since 4.0.0
  */
 export const incomingLocalFromOutgoing = <R extends Rpc.Any>(self: Outgoing<R>): IncomingLocal<R> => {
@@ -83,7 +83,7 @@ export const incomingLocalFromOutgoing = <R extends Rpc.Any>(self: Outgoing<R>):
  * It carries the last reply that was sent and a callback for persisting encoded
  * replies.
  *
- * @category incoming
+ * @category models
  * @since 4.0.0
  */
 export class IncomingRequest<R extends Rpc.Any> extends Data.TaggedClass("IncomingRequest")<{
@@ -100,7 +100,7 @@ export class IncomingRequest<R extends Rpc.Any> extends Data.TaggedClass("Incomi
  * It includes dynamic annotations, the last sent reply, and a callback for
  * replying with decoded replies.
  *
- * @category incoming
+ * @category models
  * @since 4.0.0
  */
 export class IncomingRequestLocal<R extends Rpc.Any> extends Data.TaggedClass("IncomingRequestLocal")<{
@@ -113,7 +113,7 @@ export class IncomingRequestLocal<R extends Rpc.Any> extends Data.TaggedClass("I
 /**
  * Represents an incoming control envelope carrying an `AckChunk` or `Interrupt`.
  *
- * @category incoming
+ * @category models
  * @since 4.0.0
  */
 export class IncomingEnvelope extends Data.TaggedClass("IncomingEnvelope")<{
@@ -128,7 +128,7 @@ export class IncomingEnvelope extends Data.TaggedClass("IncomingEnvelope")<{
  *
  * An outgoing message is either an entity request or a control envelope.
  *
- * @category outgoing
+ * @category models
  * @since 4.0.0
  */
 export type Outgoing<R extends Rpc.Any> = OutgoingRequest<R> | OutgoingEnvelope
@@ -141,7 +141,7 @@ export type Outgoing<R extends Rpc.Any> = OutgoingRequest<R> | OutgoingEnvelope
  * It carries the service context used for serialization, the last received reply,
  * the reply callback, dynamic annotations, and an optional encoded request cache.
  *
- * @category outgoing
+ * @category models
  * @since 4.0.0
  */
 export class OutgoingRequest<R extends Rpc.Any> extends Data.TaggedClass("OutgoingRequest")<{
@@ -168,7 +168,7 @@ export class OutgoingRequest<R extends Rpc.Any> extends Data.TaggedClass("Outgoi
  * Use to construct an interrupt envelope for an
  * in-flight request.
  *
- * @category outgoing
+ * @category models
  * @since 4.0.0
  */
 export class OutgoingEnvelope extends Data.TaggedClass("OutgoingEnvelope")<{

package/src/unstable/cluster/MessageStorage.ts

@@ -42,7 +42,7 @@ import * as Snowflake from "./Snowflake.ts"
  * messages; manages reply handlers; and provides transaction wrapping for storage
  * operations.
  *
- * @category context
+ * @category services
  * @since 4.0.0
  */
 export class MessageStorage extends Context.Service<MessageStorage, {
@@ -180,7 +180,7 @@ export class MessageStorage extends Context.Service<MessageStorage, {
  * A duplicate result carries the original request ID and the last reply already
  * received for the duplicated request.
  *
- * @category SaveResult
+ * @category models
  * @since 4.0.0
  */
 export type SaveResult<R extends Rpc.Any> = SaveResult.Success | SaveResult.Duplicate<R>
@@ -188,7 +188,7 @@ export type SaveResult<R extends Rpc.Any> = SaveResult.Success | SaveResult.Dupl
 /**
  * Constructors and matchers for decoded save results.
  *
- * @category SaveResult
+ * @category constructors
  * @since 4.0.0
  */
 export const SaveResult = Data.taggedEnum<SaveResult.Constructor>()
@@ -197,7 +197,7 @@ export const SaveResult = Data.taggedEnum<SaveResult.Constructor>()
  * Constructors and matchers for encoded save results returned by storage
  * drivers.
  *
- * @category SaveResult
+ * @category constructors
  * @since 4.0.0
  */
 export const SaveResultEncoded = Data.taggedEnum<SaveResult.Encoded>()
@@ -216,7 +216,7 @@ export declare namespace SaveResult {
    * Duplicate results contain an encoded last received reply instead of a decoded
    * reply.
    *
-   * @category SaveResult
+   * @category models
    * @since 4.0.0
    */
   export type Encoded = SaveResult.Success | SaveResult.DuplicateEncoded
@@ -224,7 +224,7 @@ export declare namespace SaveResult {
   /**
    * Variant indicating that the message was saved as a new storage entry.
    *
-   * @category SaveResult
+   * @category models
    * @since 4.0.0
    */
   export interface Success {
@@ -239,7 +239,7 @@ export declare namespace SaveResult {
    * It carries the original request ID and the latest decoded reply, when one is
    * available.
    *
-   * @category SaveResult
+   * @category models
    * @since 4.0.0
    */
   export interface Duplicate<R extends Rpc.Any> {
@@ -256,7 +256,7 @@ export declare namespace SaveResult {
    * It carries the original request ID and the latest encoded reply, when one is
    * available.
    *
-   * @category SaveResult
+   * @category models
    * @since 4.0.0
    */
   export interface DuplicateEncoded {
@@ -268,7 +268,7 @@ export declare namespace SaveResult {
   /**
    * Generic tagged enum constructor type for `SaveResult`.
    *
-   * @category SaveResult
+   * @category utility types
    * @since 4.0.0
    */
   export interface Constructor extends Data.TaggedEnum.WithGenerics<1> {
@@ -284,7 +284,7 @@ export declare namespace SaveResult {
  * Implementations persist encoded messages, track primary keys and delayed
  * delivery, read unprocessed messages, and provide transaction wrapping.
  *
- * @category Encoded
+ * @category services
  * @since 4.0.0
  */
 export type Encoded = {
@@ -412,7 +412,7 @@ export type Encoded = {
  * The fields distinguish existing shards from newly assigned shards and carry the
  * driver-specific pagination cursor.
  *
- * @category Encoded
+ * @category models
  * @since 4.0.0
  */
 export type EncodedUnprocessedOptions<A> = {
@@ -429,7 +429,7 @@ export type EncodedUnprocessedOptions<A> = {
  * The fields distinguish existing requests from new requests and carry the
  * driver-specific pagination cursor.
  *
- * @category Encoded
+ * @category models
  * @since 4.0.0
  */
 export type EncodedRepliesOptions<A> = {
@@ -606,11 +606,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
         ),
         Effect.asVoid
       ),
-    saveReply: (reply) =>
-      Effect.flatMap(
-        Reply.serialize(reply),
-        encoded.saveReply
-      ),
+    saveReply: (reply) => Effect.flatMap(Reply.serializeOrDefect(reply), encoded.saveReply),
     clearReplies: encoded.clearReplies,
     repliesFor: Effect.fnUntraced(function*(messages) {
       const requestIds = Arr.empty<string>()
@@ -793,7 +789,7 @@ export const noop: MessageStorage["Service"] = Effect.runSync(make({
  * It stores the encoded envelope, last acknowledged chunk, accumulated replies,
  * and optional delivery time.
  *
- * @category memory
+ * @category models
  * @since 4.0.0
  */
 export type MemoryEntry = {
@@ -806,7 +802,7 @@ export type MemoryEntry = {
 /**
  * Provides a context reference used in tests to simulate a transaction.
  *
- * @category memory
+ * @category services
  * @since 4.0.0
  */
 export const MemoryTransaction = Context.Reference<boolean>("effect/cluster/MessageStorage/MemoryTransaction", {
@@ -822,7 +818,7 @@ export const MemoryTransaction = Context.Reference<boolean>("effect/cluster/Mess
  * maps used to track requests, primary keys, unprocessed envelopes, reply IDs,
  * and the journal.
  *
- * @category memory
+ * @category services
  * @since 4.0.0
  */
 export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluster/MessageStorage/MemoryDriver", {
@@ -995,6 +991,14 @@ export class MemoryDriver extends Context.Service<MemoryDriver>()("effect/cluste
       resetAddress: () => Effect.void,
       clearAddress: (address) =>
         Effect.sync(() => {
+          for (const [primaryKey, entry] of requestsByPrimaryKey) {
+            const envelope = entry.envelope
+            const sameAddress = address.entityType === envelope.address.entityType &&
+              address.entityId === envelope.address.entityId
+            if (sameAddress) {
+              requestsByPrimaryKey.delete(primaryKey)
+            }
+          }
           for (let i = journal.length - 1; i >= 0; i--) {
             const envelope = journal[i]
             const sameAddress = address.entityType === envelope.address.entityType &&

package/src/unstable/cluster/Reply.ts

@@ -242,11 +242,10 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
       [success],
       ([success]) => (input, ast, options) => {
         if (!isReply(input) || input._tag !== "Chunk") {
-          return Effect.fail(new SchemaIssue.InvalidType(ast, Option.some(input)))
+          return Effect.fail(new SchemaIssue.InvalidType(ast, input, options))
         }
         return Effect.mapBothEager(SchemaParser.decodeEffect(Schema.NonEmptyArray(success))(input.values, options), {
-          onFailure: (issue) =>
-            new SchemaIssue.Composite(ast, Option.some(input), [new SchemaIssue.Pointer(["values"], issue)]),
+          onFailure: (issue) => SchemaIssue.makeCompositeAtKey(ast, "values", issue, input, options),
           onSuccess: (values) => new Chunk({ ...input, values } as any)
         })
       },
@@ -258,7 +257,7 @@ export class Chunk<R extends Rpc.Any> extends Data.TaggedClass("Chunk")<{
               _tag: Schema.Literal("Chunk"),
               requestId: SnowflakeFromBigInt,
               id: SnowflakeFromBigInt,
-              sequence: Schema.Number,
+              sequence: Schema.Int,
               values: Schema.NonEmptyArray(success)
             }),
             SchemaTransformation.transform({
@@ -352,11 +351,10 @@ export class WithExit<R extends Rpc.Any> extends Data.TaggedClass("WithExit")<{
       [exitSchema],
       ([exit]) => (input, ast, options) => {
         if (!isReply(input) || input._tag !== "WithExit") {
-          return Effect.fail(new SchemaIssue.InvalidType(ast, Option.some(input)))
+          return Effect.fail(new SchemaIssue.InvalidType(ast, input, options))
         }
         return Effect.mapBothEager(SchemaParser.decodeEffect(exit)(input.exit, options), {
-          onFailure: (issue) =>
-            new SchemaIssue.Composite(ast, Option.some(input), [new SchemaIssue.Pointer(["exit"], issue)]),
+          onFailure: (issue) => SchemaIssue.makeCompositeAtKey(ast, "exit", issue, input, options),
           onSuccess: (exit) => new WithExit({ ...input, exit: exit as any })
         })
       },
@@ -435,6 +433,27 @@ export const serialize = <R extends Rpc.Any>(
   )
 }
 
+/**
+ * Serializes a `ReplyWithContext`, falling back to a serializable defect reply
+ * when the original reply cannot be encoded.
+ *
+ * @category serialization
+ * @since 4.0.0
+ */
+export const serializeOrDefect = <R extends Rpc.Any>(
+  self: ReplyWithContext<R>
+): Effect.Effect<Encoded> =>
+  Effect.catchTag(
+    serialize(self),
+    "MalformedMessage",
+    (error) =>
+      Effect.orDie(serialize(ReplyWithContext.fromDefect({
+        id: self.reply.id,
+        requestId: self.reply.requestId,
+        defect: error
+      })))
+  )
+
 /**
  * Serializes an outgoing request's last received reply when one exists, returning
  * `None` when no reply has been received and refailing encoding errors as

package/src/unstable/cluster/Runner.ts

@@ -29,7 +29,7 @@ const TypeId = "~effect/cluster/Runner"
 export class Runner extends Schema.Class<Runner>(TypeId)({
   address: RunnerAddress,
   groups: Schema.Array(Schema.String),
-  weight: Schema.Number
+  weight: Schema.Finite
 }) {
   /**
    * Formatter for rendering runner values consistently.

package/src/unstable/cluster/RunnerAddress.ts

@@ -28,7 +28,7 @@ const TypeId = "~effect/cluster/RunnerAddress"
  */
 export class RunnerAddress extends Schema.Class<RunnerAddress>(TypeId)({
   host: Schema.String,
-  port: Schema.Number
+  port: Schema.Int
 }) {
   /**
    * Marks this value as a cluster runner address for runtime guards.

package/src/unstable/cluster/RunnerHealth.ts

@@ -27,7 +27,7 @@ import * as Runners from "./Runners.ts"
  * still be processing messages. If a Runner is not responsive, then its
  * associated shards can and will be re-assigned to a different Runner.
  *
- * @category models
+ * @category services
  * @since 4.0.0
  */
 export class RunnerHealth extends Context.Service<

package/src/unstable/cluster/RunnerServer.ts

@@ -10,6 +10,7 @@
  *
  * @since 4.0.0
  */
+import type * as Cause from "../../Cause.ts"
 import * as Effect from "../../Effect.ts"
 import type * as Exit from "../../Exit.ts"
 import * as Fiber from "../../Fiber.ts"
@@ -17,6 +18,7 @@ import { constant } from "../../Function.ts"
 import * as Layer from "../../Layer.ts"
 import * as Option from "../../Option.ts"
 import * as Queue from "../../Queue.ts"
+import type * as Rpc from "../rpc/Rpc.ts"
 import * as RpcServer from "../rpc/RpcServer.ts"
 import type * as ClusterError from "./ClusterError.ts"
 import * as Message from "./Message.ts"
@@ -30,6 +32,16 @@ import { ShardingConfig } from "./ShardingConfig.ts"
 
 const constVoid = constant(Effect.void)
 
+const serializeDefectReply = <R extends Rpc.Any>(
+  reply: Reply.ReplyWithContext<R>,
+  defect: unknown
+): Effect.Effect<Reply.Encoded> =>
+  Effect.orDie(Reply.serialize(Reply.ReplyWithContext.fromDefect({
+    id: reply.reply.id,
+    requestId: reply.reply.requestId,
+    defect
+  })))
+
 /**
  * Layer that handles runner protocol RPCs by forwarding requests to `Sharding`
  * and `MessageStorage`.
@@ -43,16 +55,16 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
 
   return {
     Ping: () => Effect.void,
-    Notify: ({ envelope }) =>
-      sharding.notify(
-        envelope._tag === "Request"
-          ? new Message.IncomingRequest({
-            envelope,
-            respond: constVoid,
-            lastSentReply: Option.none()
-          })
-          : new Message.IncomingEnvelope({ envelope })
-      ),
+    Notify: ({ envelope, persisted }) => {
+      const message = envelope._tag === "Request"
+        ? new Message.IncomingRequest({
+          envelope,
+          respond: constVoid,
+          lastSentReply: Option.none()
+        })
+        : new Message.IncomingEnvelope({ envelope })
+      return persisted ? sharding.notify(message) : sharding.send(message)
+    },
     Effect: ({ persisted, request }) => {
       let replyEncoded: Option.Option<Effect.Effect<Reply.Encoded, ClusterError.EntityNotAssignedToRunner>> = Option
         .none()
@@ -63,7 +75,7 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
         envelope: request,
         lastSentReply: Option.none(),
         respond(reply) {
-          resume(Effect.orDie(Reply.serialize(reply)))
+          resume(Reply.serializeOrDefect(reply))
           return Effect.void
         }
       })
@@ -108,23 +120,37 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
     },
     Stream: ({ persisted, request }) =>
       Effect.flatMap(
-        Queue.make<Reply.Encoded, ClusterError.EntityNotAssignedToRunner>(),
+        Queue.make<Reply.Encoded, ClusterError.EntityNotAssignedToRunner | Cause.Done>(),
         (queue) => {
           const message = new Message.IncomingRequest({
             envelope: request,
             lastSentReply: Option.none(),
             respond(reply) {
-              return Effect.flatMap(Reply.serialize(reply), (reply) => {
-                Queue.offerUnsafe(queue, reply)
-                return Effect.void
-              })
+              return Reply.serialize(reply).pipe(
+                Effect.flatMap((reply) => {
+                  Queue.offerUnsafe(queue, reply)
+                  if (reply._tag === "WithExit") {
+                    Queue.endUnsafe(queue)
+                  }
+                  return Effect.void
+                }),
+                Effect.catchTag("MalformedMessage", (error) =>
+                  Effect.flatMap(serializeDefectReply(reply, error), (reply) => {
+                    // the fallback defect reply is terminal, so end the stream
+                    Queue.offerUnsafe(queue, reply)
+                    Queue.endUnsafe(queue)
+                    return Effect.void
+                  }))
+              )
             }
           })
           return Effect.as(
             persisted ?
               Effect.andThen(
                 storage.registerReplyHandler(message).pipe(
-                  Effect.onError((cause) => Queue.failCause(queue, cause)),
+                  Effect.onError((cause) =>
+                    Queue.failCause(queue, cause)
+                  ),
                   Effect.forkScoped
                 ),
                 sharding.notify(message, constWaitUntilRead)
@@ -168,7 +194,8 @@ export const layer: Layer.Layer<
   RpcServer.Protocol | Sharding.Sharding | MessageStorage.MessageStorage
 > = RpcServer.layer(Runners.Rpcs, {
   spanPrefix: "RunnerServer",
-  disableTracing: true
+  disableTracing: true,
+  disableFatalDefects: true
 }).pipe(Layer.provide(layerHandlers))
 
 /**

package/src/unstable/cluster/RunnerStorage.ts

@@ -24,7 +24,7 @@ import * as ShardId from "./ShardId.ts"
  * Represents a generic interface to the persistent storage required by the
  * cluster.
  *
- * @category models
+ * @category services
  * @since 4.0.0
  */
 export class RunnerStorage extends Context.Service<RunnerStorage, {
@@ -84,7 +84,7 @@ export class RunnerStorage extends Context.Service<RunnerStorage, {
  * String-encoded runner storage interface used by adapters that persist runner
  * addresses, runners, machine ids, and shard ids outside the in-memory model.
  *
- * @category Encoded
+ * @category services
  * @since 4.0.0
  */
 export interface Encoded {
@@ -195,29 +195,31 @@ export const makeEncoded = (encoded: Encoded) =>
  *
  * **Details**
  *
- * Registered runners are treated as healthy and shard acquisition is kept only in
- * process memory.
+ * Runner health and shard acquisition are kept only in process memory.
  *
  * @category constructors
  * @since 4.0.0
  */
 export const makeMemory = Effect.gen(function*() {
-  const runners = MutableHashMap.empty<RunnerAddress, Runner>()
+  const runners = MutableHashMap.empty<RunnerAddress, readonly [runner: Runner, healthy: boolean]>()
   let acquired: Array<ShardId.ShardId> = []
   let id = 0
 
   return RunnerStorage.of({
-    getRunners: Effect.sync(() => Array.from(MutableHashMap.values(runners), (runner) => [runner, true])),
-    register: (runner) =>
+    getRunners: Effect.sync(() => Array.from(MutableHashMap.values(runners))),
+    register: (runner, healthy) =>
       Effect.sync(() => {
-        MutableHashMap.set(runners, runner.address, runner)
+        MutableHashMap.set(runners, runner.address, [runner, healthy])
         return MachineId.make(id++)
       }),
     unregister: (address) =>
       Effect.sync(() => {
         MutableHashMap.remove(runners, address)
       }),
-    setRunnerHealth: () => Effect.void,
+    setRunnerHealth: (address, healthy) =>
+      Effect.sync(() => {
+        MutableHashMap.modify(runners, address, ([runner]) => [runner, healthy] as const)
+      }),
     acquire: (_address, shardIds) => {
       acquired = Array.from(shardIds)
       return Effect.succeed(acquired)