effect 4.0.0-beta.102 → 4.0.0-beta.103

package/src/unstable/cluster/RunnerHealth.ts

@@ -27,7 +27,7 @@ import * as Runners from "./Runners.ts"
  * still be processing messages. If a Runner is not responsive, then its
  * associated shards can and will be re-assigned to a different Runner.
  *
- * @category models
+ * @category services
  * @since 4.0.0
  */
 export class RunnerHealth extends Context.Service<

package/src/unstable/cluster/RunnerServer.ts

@@ -10,6 +10,7 @@
  *
  * @since 4.0.0
  */
+import type * as Cause from "../../Cause.ts"
 import * as Effect from "../../Effect.ts"
 import type * as Exit from "../../Exit.ts"
 import * as Fiber from "../../Fiber.ts"
@@ -17,6 +18,7 @@ import { constant } from "../../Function.ts"
 import * as Layer from "../../Layer.ts"
 import * as Option from "../../Option.ts"
 import * as Queue from "../../Queue.ts"
+import type * as Rpc from "../rpc/Rpc.ts"
 import * as RpcServer from "../rpc/RpcServer.ts"
 import type * as ClusterError from "./ClusterError.ts"
 import * as Message from "./Message.ts"
@@ -30,6 +32,25 @@ import { ShardingConfig } from "./ShardingConfig.ts"
 
 const constVoid = constant(Effect.void)
 
+const serializeDefectReply = <R extends Rpc.Any>(
+  reply: Reply.ReplyWithContext<R>,
+  defect: unknown
+): Effect.Effect<Reply.Encoded> =>
+  Effect.orDie(Reply.serialize(Reply.ReplyWithContext.fromDefect({
+    id: reply.reply.id,
+    requestId: reply.reply.requestId,
+    defect
+  })))
+
+const serializeReply = <R extends Rpc.Any>(
+  reply: Reply.ReplyWithContext<R>
+): Effect.Effect<Reply.Encoded> =>
+  Effect.catchTag(
+    Reply.serialize(reply),
+    "MalformedMessage",
+    (error) => serializeDefectReply(reply, error)
+  )
+
 /**
  * Layer that handles runner protocol RPCs by forwarding requests to `Sharding`
  * and `MessageStorage`.
@@ -63,7 +84,7 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
         envelope: request,
         lastSentReply: Option.none(),
         respond(reply) {
-          resume(Effect.orDie(Reply.serialize(reply)))
+          resume(serializeReply(reply))
           return Effect.void
         }
       })
@@ -108,23 +129,34 @@ export const layerHandlers = Runners.Rpcs.toLayer(Effect.gen(function*() {
     },
     Stream: ({ persisted, request }) =>
       Effect.flatMap(
-        Queue.make<Reply.Encoded, ClusterError.EntityNotAssignedToRunner>(),
+        Queue.make<Reply.Encoded, ClusterError.EntityNotAssignedToRunner | Cause.Done>(),
         (queue) => {
           const message = new Message.IncomingRequest({
             envelope: request,
             lastSentReply: Option.none(),
             respond(reply) {
-              return Effect.flatMap(Reply.serialize(reply), (reply) => {
-                Queue.offerUnsafe(queue, reply)
-                return Effect.void
-              })
+              return Reply.serialize(reply).pipe(
+                Effect.flatMap((reply) => {
+                  Queue.offerUnsafe(queue, reply)
+                  return Effect.void
+                }),
+                Effect.catchTag("MalformedMessage", (error) =>
+                  Effect.flatMap(serializeDefectReply(reply, error), (reply) => {
+                    // the fallback defect reply is terminal, so end the stream
+                    Queue.offerUnsafe(queue, reply)
+                    Queue.endUnsafe(queue)
+                    return Effect.void
+                  }))
+              )
             }
           })
           return Effect.as(
             persisted ?
               Effect.andThen(
                 storage.registerReplyHandler(message).pipe(
-                  Effect.onError((cause) => Queue.failCause(queue, cause)),
+                  Effect.onError((cause) =>
+                    Queue.failCause(queue, cause)
+                  ),
                   Effect.forkScoped
                 ),
                 sharding.notify(message, constWaitUntilRead)
@@ -168,7 +200,8 @@ export const layer: Layer.Layer<
   RpcServer.Protocol | Sharding.Sharding | MessageStorage.MessageStorage
 > = RpcServer.layer(Runners.Rpcs, {
   spanPrefix: "RunnerServer",
-  disableTracing: true
+  disableTracing: true,
+  disableFatalDefects: true
 }).pipe(Layer.provide(layerHandlers))
 
 /**

package/src/unstable/cluster/RunnerStorage.ts

@@ -24,7 +24,7 @@ import * as ShardId from "./ShardId.ts"
  * Represents a generic interface to the persistent storage required by the
  * cluster.
  *
- * @category models
+ * @category services
  * @since 4.0.0
  */
 export class RunnerStorage extends Context.Service<RunnerStorage, {
@@ -84,7 +84,7 @@ export class RunnerStorage extends Context.Service<RunnerStorage, {
  * String-encoded runner storage interface used by adapters that persist runner
  * addresses, runners, machine ids, and shard ids outside the in-memory model.
  *
- * @category Encoded
+ * @category services
  * @since 4.0.0
  */
 export interface Encoded {

package/src/unstable/cluster/Runners.ts

@@ -41,7 +41,7 @@ import * as Snowflake from "./Snowflake.ts"
  * sending and notifying messages, coordinating persisted replies, and marking
  * runners unavailable.
  *
- * @category context
+ * @category services
  * @since 4.0.0
  */
 export class Runners extends Context.Service<Runners, {
@@ -425,7 +425,7 @@ export const make: (options: Omit<Runners["Service"], "sendLocal" | "notifyLocal
  * `EntityNotAssignedToRunner` and ignores notifications, pings, and unavailable
  * runner reports.
  *
- * @category No-op
+ * @category constructors
  * @since 4.0.0
  */
 export const makeNoop: Effect.Effect<
@@ -466,7 +466,7 @@ const rpcErrors: Schema.Union<[
  * RPC group used for runner-to-runner communication, including ping, notify,
  * effect, stream, and envelope messages.
  *
- * @category Rpcs
+ * @category models
  * @since 4.0.0
  */
 export class Rpcs extends RpcGroup.make(
@@ -507,7 +507,7 @@ export class Rpcs extends RpcGroup.make(
 /**
  * Client interface generated from the runner RPC group.
  *
- * @category Rpcs
+ * @category models
  * @since 4.0.0
  */
 export interface RpcClient extends RpcClient_.FromGroup<typeof Rpcs, RpcClientError> {}
@@ -516,7 +516,7 @@ export interface RpcClient extends RpcClient_.FromGroup<typeof Rpcs, RpcClientEr
  * Builds a runner RPC client from the current `RpcClient.Protocol`, using the
  * `Runners` span prefix with tracing disabled.
  *
- * @category Rpcs
+ * @category constructors
  * @since 4.0.0
  */
 export const makeRpcClient: Effect.Effect<
@@ -574,11 +574,22 @@ export const makeRpc: Effect.Effect<
               persisted: isPersisted
             })
           ),
-          Effect.catchTag("RpcClientError", Effect.die),
           Effect.scoped,
-          Effect.catchDefect(() => Effect.fail(new RunnerUnavailable({ address })))
+          Effect.catchTag("RpcClientError", () => Effect.fail(new RunnerUnavailable({ address })))
         )
       }
+      // Persisted requests can recover their reply from storage via the
+      // `RunnerUnavailable` path, volatile requests receive the defect as their reply.
+      const respondDefect = (defect: unknown) =>
+        isPersisted
+          ? Effect.fail(new RunnerUnavailable({ address }))
+          : message.respond(
+            new Reply.WithExit({
+              id: snowflakeGen.nextUnsafe(),
+              requestId: message.envelope.requestId,
+              exit: Exit.die(defect)
+            })
+          )
       const isStream = RpcSchema.isStreamSchema(rpc.successSchema)
       if (!isStream) {
         return Effect.matchEffect(Message.serializeRequest(message), {
@@ -590,7 +601,6 @@ export const makeRpc: Effect.Effect<
                   persisted: isPersisted
                 })
               ),
-              Effect.catchTag("RpcClientError", Effect.die),
               Effect.flatMap((reply) =>
                 Schema.decodeEffect(Reply.Reply(message.rpc))(reply).pipe(
                   Effect.provideContext(message.context),
@@ -599,7 +609,8 @@ export const makeRpc: Effect.Effect<
               ),
               Effect.flatMap(message.respond),
               Effect.scoped,
-              Effect.catchDefect(() => Effect.fail(new RunnerUnavailable({ address })))
+              Effect.catchTag("RpcClientError", () => Effect.fail(new RunnerUnavailable({ address }))),
+              Effect.catchDefect(respondDefect)
             ),
           onFailure: (error) =>
             message.respond(
@@ -626,10 +637,10 @@ export const makeRpc: Effect.Effect<
                 Effect.flatMap((reply) => Effect.orDie(decode(reply))),
                 Effect.flatMap(message.respond),
                 Effect.forever,
-                Effect.catchTag("RpcClientError", Effect.die),
                 Effect.provideContext(message.context),
                 Effect.catchTag("Done", (_) => Effect.void),
-                Effect.catchDefect(() => Effect.fail(new RunnerUnavailable({ address })))
+                Effect.catchTag("RpcClientError", () => Effect.fail(new RunnerUnavailable({ address }))),
+                Effect.catchDefect(respondDefect)
               )
             }),
             Effect.scoped
@@ -681,7 +692,7 @@ export const layerRpc: Layer.Layer<
  * Service that creates an RPC client protocol for communicating with a runner at a
  * given address.
  *
- * @category client
+ * @category services
  * @since 4.0.0
  */
 export class RpcClientProtocol extends Context.Service<

package/src/unstable/cluster/Sharding.ts

@@ -771,7 +771,7 @@ const make = Effect.gen(function*() {
         const resumptionState = Option.getOrThrow(MutableHashMap.get(entityResumptionState, address))
         let done = false
 
-        while (!done) {
+        while (!done) { // oxlint-disable-line no-unmodified-loop-condition
           // if the shard is no longer assigned to this runner, we stop
           if (!MutableHashSet.has(acquiredShards, address.shardId)) {
             return
@@ -1202,7 +1202,7 @@ const make = Effect.gen(function*() {
               if (!options.discard) {
                 const entry: ClientRequestEntry = {
                   rpc: rpc as any,
-                  context: fiber.currentContext,
+                  context: fiber.context,
                   message
                 }
                 clientRequests.set(id, entry)
@@ -1436,12 +1436,11 @@ const make = Effect.gen(function*() {
         runnerAddress,
         sharding
       }).pipe(
-        Effect.provideContext(Context.mutate(services, (services) =>
-          services.pipe(
-            Context.add(EntityReaper, reaper),
-            Context.add(Scope.Scope, scope),
-            Context.add(Snowflake.Generator, snowflakeGen)
-          )))
+        Effect.provideContext(services.pipe(
+          Context.add(EntityReaper, reaper),
+          Context.add(Scope.Scope, scope),
+          Context.add(Snowflake.Generator, snowflakeGen)
+        ))
       ) as Effect.Effect<EntityManager.EntityManager>
       const state: EntityManagerState = {
         entity,

package/src/unstable/cluster/ShardingConfig.ts

@@ -23,7 +23,7 @@ import { RunnerAddress } from "./RunnerAddress.ts"
 /**
  * Represents the configuration for the `Sharding` service on a given runner.
  *
- * @category models
+ * @category services
  * @since 4.0.0
  */
 export class ShardingConfig extends Context.Service<ShardingConfig, {
@@ -210,7 +210,7 @@ export const layer = (options?: Partial<ShardingConfig["Service"]>): Layer.Layer
 /**
  * Layer that provides the default `ShardingConfig` values.
  *
- * @category defaults
+ * @category layers
  * @since 4.0.0
  */
 export const layerDefaults: Layer.Layer<ShardingConfig> = layer()
@@ -353,7 +353,7 @@ export const layerFromEnv = (options?: Partial<ShardingConfig["Service"]> | unde
  * Normalizes the provided `ShardingConfig` to calculate the `available` and
  * `assigned` shard groups.
  *
- * @category Shard groups
+ * @category converting
  * @since 4.0.0
  */
 export const shardGroupConfig = (config: ShardingConfig["Service"]): {

package/src/unstable/cluster/SingletonAddress.ts

@@ -17,7 +17,7 @@ const TypeId = "~effect/cluster/SingletonAddress"
 /**
  * Represents the unique address of an singleton within the cluster.
  *
- * @category address
+ * @category schemas
  * @since 4.0.0
  */
 export class SingletonAddress extends Schema.Class<SingletonAddress>(TypeId)({

package/src/unstable/cluster/Snowflake.ts

@@ -170,7 +170,7 @@ export const make = (options: {
 /**
  * Extracts the Unix timestamp in milliseconds from a snowflake id.
  *
- * @category parts
+ * @category getters
  * @since 4.0.0
  */
 export const timestamp = (snowflake: Snowflake): number => Number(snowflake >> constBigInt22) + sinceUnixEpoch
@@ -178,7 +178,7 @@ export const timestamp = (snowflake: Snowflake): number => Number(snowflake >> c
 /**
  * Extracts the timestamp from a snowflake id as a `DateTime.Utc`.
  *
- * @category parts
+ * @category getters
  * @since 4.0.0
  */
 export const dateTime = (snowflake: Snowflake): DateTime.Utc => DateTime.makeUnsafe(timestamp(snowflake))
@@ -186,7 +186,7 @@ export const dateTime = (snowflake: Snowflake): DateTime.Utc => DateTime.makeUns
 /**
  * Extracts the machine id component from a snowflake id.
  *
- * @category parts
+ * @category getters
  * @since 4.0.0
  */
 export const machineId = (snowflake: Snowflake): MachineId =>
@@ -195,7 +195,7 @@ export const machineId = (snowflake: Snowflake): MachineId =>
 /**
  * Extracts the per-machine sequence component from a snowflake id.
  *
- * @category parts
+ * @category getters
  * @since 4.0.0
  */
 export const sequence = (snowflake: Snowflake): number => Number(snowflake % constBigInt4096)
@@ -203,7 +203,7 @@ export const sequence = (snowflake: Snowflake): number => Number(snowflake % con
 /**
  * Decomposes a snowflake id into its timestamp, machine id, and sequence parts.
  *
- * @category parts
+ * @category converting
  * @since 4.0.0
  */
 export const toParts = (snowflake: Snowflake): Snowflake.Parts => ({
@@ -221,7 +221,7 @@ export const toParts = (snowflake: Snowflake): Snowflake.Parts => ({
  * backward, resets the sequence each millisecond, and advances the timestamp when
  * more than 4096 ids are requested in the same millisecond.
  *
- * @category Generator
+ * @category constructors
  * @since 4.0.0
  */
 export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(function*() {
@@ -265,7 +265,7 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
 /**
  * Context service for a stateful snowflake id generator.
  *
- * @category Generator
+ * @category services
  * @since 4.0.0
  */
 export class Generator extends Context.Service<
@@ -276,7 +276,7 @@ export class Generator extends Context.Service<
 /**
  * Layer that provides the default snowflake `Generator` service.
  *
- * @category Generator
+ * @category layers
  * @since 4.0.0
  */
 export const layerGenerator: Layer.Layer<Generator> = Layer.effect(Generator)(makeGenerator)

package/src/unstable/cluster/SqlMessageStorage.ts

@@ -1041,7 +1041,7 @@ const replyKind = {
 } as const satisfies Record<Reply.Reply<any>["_tag"], number | null>
 
 const replyFromRow = (row: ReplyRow): Reply.Encoded =>
-  Number(row.kind) === replyKind.WithExit ?
+  row.kind !== null && Number(row.kind) === replyKind.WithExit ?
     {
       _tag: "WithExit",
       id: String(row.id),

package/src/unstable/cluster/SqlRunnerStorage.ts

@@ -28,6 +28,17 @@ import * as ShardingConfig from "./ShardingConfig.ts"
 
 const withTracerDisabled = Effect.withTracerEnabled(false)
 
+// This exact FNV-1a hash, including its tag and UTF-8 encoding, is a persistent
+// advisory-lock wire format and must never change.
+const postgresLockNamespace = (prefix: string): number => {
+  const bytes = new TextEncoder().encode(`effect-cluster:${prefix}`)
+  let hash = 0x811c9dc5
+  for (let i = 0; i < bytes.length; i++) {
+    hash = Math.imul(hash ^ bytes[i], 0x01000193)
+  }
+  return hash | 0
+}
+
 /**
  * Creates a SQL-backed `RunnerStorage` implementation for registered runners and
  * shard locks, using the configured table prefix and advisory locks where
@@ -68,6 +79,9 @@ export const make = Effect.fnUntraced(function*(options: {
   const layerScope = yield* Effect.scope
   const prefix = options?.prefix ?? "cluster"
   const table = (name: string) => `${prefix}_${name}`
+  const pgLockNamespace = postgresLockNamespace(prefix)
+  // PostgreSQL exposes the signed int4 lock key through pg_locks as an unsigned oid.
+  const pgLockNamespaceOid = pgLockNamespace >>> 0
 
   // Keep all PostgreSQL and MySQL shard-lock operations on a rebuildable
   // reserved connection, including when advisory locks are disabled.
@@ -408,12 +422,14 @@ export const make = Effect.fnUntraced(function*(options: {
         const acquiredShardIds: Array<string> = []
         const toAcquire = new Map(shardIds.map((shardId) => [lockNumbers.get(shardId)!, shardId]))
         const takenLocks = yield* conn.executeValues(
-          `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = ${pid} ORDER BY objid`,
+          `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND classid = ${pgLockNamespaceOid} AND objsubid = 2 AND pid = ${pid} ORDER BY objid`,
           []
         )
         for (let i = 0; i < takenLocks.length; i++) {
           const lockNum = takenLocks[i][0] as number
-          acquiredShardIds.push(lockNumbersReverse.get(lockNum)!)
+          const shardId = lockNumbersReverse.get(lockNum)
+          if (shardId === undefined) continue
+          acquiredShardIds.push(shardId)
           toAcquire.delete(lockNum)
         }
         if (toAcquire.size === 0) {
@@ -546,7 +562,7 @@ export const make = Effect.fnUntraced(function*(options: {
   const pgLocks = (shardIdsMap: Map<number, string>) =>
     Array.from(
       shardIdsMap.entries(),
-      ([lockNum, shardId]) => `pg_try_advisory_lock(${lockNum}) AS "${shardId}"`
+      ([lockNum, shardId]) => `pg_try_advisory_lock(${pgLockNamespace}, ${lockNum}) AS "${shardId}"`
     ).join(", ")
 
   const mysqlLocks = (shardIds: ReadonlyArray<string>) =>
@@ -634,9 +650,9 @@ export const make = Effect.fnUntraced(function*(options: {
           const lockNum = lockNumbers.get(shardId)!
           for (let i = 0; i < 5; i++) {
             const [conn] = yield* lockConn!.await
-            yield* conn.executeRaw(`SELECT pg_advisory_unlock(${lockNum})`, [])
+            yield* conn.executeRaw(`SELECT pg_advisory_unlock(${pgLockNamespace}, ${lockNum})`, [])
             const takenLocks = yield* conn.executeValues(
-              `SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = pg_backend_pid() AND objid = ${lockNum}`,
+              `SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND classid = ${pgLockNamespaceOid} AND objid = ${lockNum} AND objsubid = 2 AND pid = pg_backend_pid()`,
               []
             )
             if (takenLocks.length === 0) return

package/src/unstable/cluster/internal/entityManager.ts

@@ -163,14 +163,13 @@ export const make = Effect.fnUntraced(function*<
       Effect.fnUntraced(function*(scope) {
         let isShuttingDown = false
 
-        const handlerContext = Context.mutate(context, (context) =>
-          context.pipe(
-            Context.add(CurrentAddress, address),
-            Context.add(CurrentRunnerAddress, options.runnerAddress),
-            Context.add(KeepAliveLatch, keepAliveLatch),
-            Context.add(Scope.Scope, scope),
-            Context.add(CurrentLogAnnotations, {})
-          ))
+        const handlerContext = context.pipe(
+          Context.add(CurrentAddress, address),
+          Context.add(CurrentRunnerAddress, options.runnerAddress),
+          Context.add(KeepAliveLatch, keepAliveLatch),
+          Context.add(Scope.Scope, scope),
+          Context.add(CurrentLogAnnotations, {})
+        )
 
         // Initiate the behavior for the entity
         const handlers = yield* (entity.protocol.toHandlers(buildHandlers as any).pipe(
@@ -546,7 +545,7 @@ export const make = Effect.fnUntraced(function*<
     )
   }
 
-  const decodeMessage = makeMessageDecode(entity, entityRpcs)
+  const decodeMessage = makeMessageDecode(entityRpcs)
 
   const runFork = Effect.runForkWith(context)
 
@@ -650,10 +649,7 @@ const defaultRetryPolicy = Schedule.min([
   Schedule.spaced("10 seconds")
 ])
 
-const makeMessageDecode = <Type extends string, Rpcs extends Rpc.Any>(
-  entity: Entity<Type, Rpcs>,
-  entityRpcs: Map<string, Rpcs>
-) => {
+const makeMessageDecode = <Rpcs extends Rpc.Any>(entityRpcs: Map<string, Rpcs>) => {
   const decodeRequest = Effect.fnUntracedEager(function*(
     message: Message.IncomingRequest<Rpcs>,
     rpc: Rpc.AnyWithProps
@@ -691,8 +687,8 @@ const makeMessageDecode = <Type extends string, Rpcs extends Rpc.Any>(
     if (!rpc) {
       return Effect.fail(
         new Schema.SchemaError(
-          new SchemaIssue.InvalidValue(Option.some(message), {
-            message: `Unknown tag ${message.envelope.tag} for entity type ${entity.type}`
+          new SchemaIssue.InvalidValue({
+            message: "Expected a known entity RPC tag"
           })
         )
       )

package/src/unstable/encoding/Msgpack.ts

@@ -17,8 +17,6 @@ import * as ChannelSchema from "../../ChannelSchema.ts"
 import * as Data from "../../Data.ts"
 import * as Effect from "../../Effect.ts"
 import { dual } from "../../Function.ts"
-import * as Option from "../../Option.ts"
-import * as Predicate from "../../Predicate.ts"
 import type * as Pull from "../../Pull.ts"
 import * as Schema from "../../Schema.ts"
 import * as SchemaIssue from "../../SchemaIssue.ts"
@@ -373,10 +371,10 @@ export const transformation: SchemaTransformation.Transformation<
   decode(e, _options) {
     try {
       return Effect.succeed(Msgpackr.decode(e))
-    } catch (cause) {
+    } catch {
       return Effect.fail(
-        new SchemaIssue.InvalidValue(Option.some(e), {
-          message: Predicate.hasProperty(cause, "message") ? String(cause.message) : String(cause)
+        new SchemaIssue.InvalidValue({
+          message: "Expected valid MessagePack bytes"
         })
       )
     }
@@ -384,10 +382,10 @@ export const transformation: SchemaTransformation.Transformation<
   encode(t, _options) {
     try {
       return Effect.succeed(Msgpackr.encode(t) as Uint8Array<ArrayBuffer>)
-    } catch (cause) {
+    } catch {
       return Effect.fail(
-        new SchemaIssue.InvalidValue(Option.some(t), {
-          message: Predicate.hasProperty(cause, "message") ? String(cause.message) : String(cause)
+        new SchemaIssue.InvalidValue({
+          message: "Expected a MessagePack-serializable value"
         })
       )
     }