effect 4.0.0-beta.40 → 4.0.0-beta.42

package/src/unstable/cluster/MessageStorage.ts

@@ -6,6 +6,7 @@ import { Clock } from "../../Clock.ts"
 import * as Data from "../../Data.ts"
 import * as Effect from "../../Effect.ts"
 import * as Exit from "../../Exit.ts"
+import { constFalse, identity } from "../../Function.ts"
 import * as Latch from "../../Latch.ts"
 import * as Layer from "../../Layer.ts"
 import * as Option from "../../Option.ts"
@@ -140,6 +141,13 @@ export class MessageStorage extends ServiceMap.Service<MessageStorage, {
   readonly clearAddress: (
     address: EntityAddress
   ) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Used to wrap requests with transactions.
+   */
+  readonly withTransaction: <A, E, R>(
+    effect: Effect.Effect<A, E, R>
+  ) => Effect.Effect<A, E, R>
 }>()("effect/cluster/MessageStorage") {}
 
 /**
@@ -315,6 +323,13 @@ export type Encoded = {
   readonly resetShards: (
     shardIds: Arr.NonEmptyArray<string>
   ) => Effect.Effect<void, PersistenceError>
+
+  /**
+   * Used to wrap requests with transactions.
+   */
+  readonly withTransaction: <A, E, R>(
+    effect: Effect.Effect<A, E, R>
+  ) => Effect.Effect<A, E, R>
 }
 
 /**
@@ -517,20 +532,22 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
       const primaryKey = Envelope.primaryKeyByAddress(options)
       return encoded.requestIdForPrimaryKey(primaryKey)
     },
-    unprocessedMessages: (shardIds) => {
+    unprocessedMessages(shardIds) {
+      const storage = this as MessageStorage["Service"]
       const shards = Array.from(shardIds, (id) => id.toString())
       if (!Arr.isArrayNonEmpty(shards)) return Effect.succeed([])
       return Effect.flatMap(
         Effect.suspend(() => encoded.unprocessedMessages(shards, clock.currentTimeMillisUnsafe())),
-        decodeMessages
+        (messages) => decodeMessages(storage, messages)
       )
     },
     unprocessedMessagesById(messageIds) {
+      const storage = this as MessageStorage["Service"]
       const ids = Array.from(messageIds)
       if (!Arr.isArrayNonEmpty(ids)) return Effect.succeed([])
       return Effect.flatMap(
         Effect.suspend(() => encoded.unprocessedMessagesById(ids, clock.currentTimeMillisUnsafe())),
-        decodeMessages
+        (messages) => decodeMessages(storage, messages)
       )
     },
     resetAddress: encoded.resetAddress,
@@ -539,10 +556,12 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
       const shards = Array.from(shardIds, (id) => id.toString())
       if (!Arr.isArrayNonEmpty(shards)) return Effect.void
       return encoded.resetShards(shards)
-    }
+    },
+    withTransaction: encoded.withTransaction
   })
 
   const decodeMessages = (
+    storage: MessageStorage["Service"],
     envelopes: Array<{
       readonly envelope: Envelope.Encoded
       readonly lastSentReply: Option.Option<Reply.Encoded>
@@ -659,7 +678,8 @@ export const noop: MessageStorage["Service"] = Effect.runSync(make({
   unprocessedMessagesById: () => Effect.succeed([]),
   resetAddress: () => Effect.void,
   clearAddress: () => Effect.void,
-  resetShards: () => Effect.void
+  resetShards: () => Effect.void,
+  withTransaction: identity
 }))
 
 /**
@@ -673,6 +693,16 @@ export type MemoryEntry = {
   deliverAt: number | null
 }
 
+/**
+ * Can be used in tests to simulate a transaction.
+ *
+ * @since 4.0.0
+ * @category Memory
+ */
+export const MemoryTransaction = ServiceMap.Reference<boolean>("effect/cluster/MessageStorage/MemoryTransaction", {
+  defaultValue: constFalse
+})
+
 /**
  * @since 4.0.0
  * @category Memory
@@ -859,7 +889,8 @@ export class MemoryDriver extends ServiceMap.Service<MemoryDriver>()("effect/clu
             journal.splice(i, 1)
           }
         }),
-      resetShards: () => Effect.void
+      resetShards: () => Effect.void,
+      withTransaction: Effect.provideService(MemoryTransaction, true)
     }
 
     const storage = yield* makeEncoded(encoded)

package/src/unstable/cluster/Sharding.ts

@@ -820,7 +820,10 @@ const make = Effect.gen(function*() {
     return Effect.catchFilter(
       Effect.suspend(() => {
         const address = message.envelope.address
-        const isPersisted = ServiceMap.get(message.rpc.annotations, Persisted)
+        const isPersisted = ServiceMap.get(
+          message._tag === "OutgoingRequest" ? message.annotations : message.rpc.annotations,
+          Persisted
+        )
         if (isPersisted && !storageEnabled) {
           return Effect.die("Sharding.sendOutgoing: Persisted messages require MessageStorage")
         }
@@ -1000,6 +1003,7 @@ const make = Effect.gen(function*() {
   type ClientRequestEntry = {
     readonly rpc: Rpc.AnyWithProps
     readonly services: ServiceMap.ServiceMap<never>
+    readonly message: Message.OutgoingRequest<any>
     lastChunkId?: Snowflake.Snowflake
   }
   const clientRequests = new Map<Snowflake.Snowflake, ClientRequestEntry>()
@@ -1029,35 +1033,39 @@ const make = Effect.gen(function*() {
             const id = Snowflake.Snowflake(options.message.id)
             const rpc = entity.protocol.requests.get(options.message.tag)!
             let respond: (reply: Reply.Reply<any>) => Effect.Effect<void>
+            const envelope = Envelope.makeRequest<any>({
+              requestId: id,
+              address,
+              tag: options.message.tag,
+              payload: options.message.payload,
+              headers: options.message.headers,
+              traceId: options.message.traceId,
+              spanId: options.message.spanId,
+              sampled: options.message.sampled
+            })
+            const message = new Message.OutgoingRequest({
+              envelope,
+              lastReceivedReply: Option.none(),
+              rpc,
+              services: fiber.services as ServiceMap.ServiceMap<any>,
+              respond: (reply) => respond(reply),
+              annotations: ServiceMap.get(rpc.annotations, ClusterSchema.Dynamic)(
+                rpc.annotations,
+                envelope as any
+              )
+            })
             if (!options.discard) {
               const entry: ClientRequestEntry = {
                 rpc: rpc as any,
-                services: fiber.currentContext
+                services: fiber.currentContext,
+                message
               }
               clientRequests.set(id, entry)
               respond = makeClientRespond(entry, client.write)
             } else {
               respond = clientRespondDiscard
             }
-            return sendOutgoing(
-              new Message.OutgoingRequest({
-                envelope: Envelope.makeRequest({
-                  requestId: id,
-                  address,
-                  tag: options.message.tag,
-                  payload: options.message.payload,
-                  headers: options.message.headers,
-                  traceId: options.message.traceId,
-                  spanId: options.message.spanId,
-                  sampled: options.message.sampled
-                }),
-                lastReceivedReply: Option.none(),
-                rpc,
-                services: fiber.services as ServiceMap.ServiceMap<any>,
-                respond
-              }),
-              options.discard
-            )
+            return sendOutgoing(message, options.discard)
           }
           case "Ack": {
             const requestId = Snowflake.Snowflake(options.message.requestId)
@@ -1081,14 +1089,14 @@ const make = Effect.gen(function*() {
             const entry = clientRequests.get(requestId)!
             if (!entry) return Effect.void
             clientRequests.delete(requestId)
-            if (ClusterSchema.isUninterruptibleForClient(entry.rpc.annotations)) {
+            if (ClusterSchema.isUninterruptibleForClient(entry.message.annotations)) {
               return Effect.void
             }
             // for durable messages, we ignore interrupts on shutdown or as a
             // result of a shard being resassigned
             const isTransientInterrupt = MutableRef.get(isShutdown) ||
               options.message.interruptors.some((id) => internalInterruptors.has(id))
-            if (isTransientInterrupt && ServiceMap.get(entry.rpc.annotations, Persisted)) {
+            if (isTransientInterrupt && ServiceMap.get(entry.message.annotations, Persisted)) {
               return Effect.void
             }
             return Effect.ignore(sendOutgoing(

package/src/unstable/cluster/SqlMessageStorage.ts

@@ -9,7 +9,7 @@ import * as Schedule from "../../Schedule.ts"
 import * as Migrator from "../sql/Migrator.ts"
 import * as SqlClient from "../sql/SqlClient.ts"
 import type { Row } from "../sql/SqlConnection.ts"
-import type { SqlError } from "../sql/SqlError.ts"
+import { isSqlError, type SqlError } from "../sql/SqlError.ts"
 import { PersistenceError } from "./ClusterError.ts"
 import type * as Envelope from "./Envelope.ts"
 import * as MessageStorage from "./MessageStorage.ts"
@@ -592,6 +592,11 @@ export const make: (options?: {
         Effect.asVoid,
         PersistenceError.refail,
         withTracerDisabled
+      ),
+
+    withTransaction: (effect) =>
+      sql.withTransaction(effect).pipe(
+        Effect.catchIf(isSqlError, Effect.die)
       )
   })
 }, withTracerDisabled)

package/src/unstable/cluster/internal/entityManager.ts

@@ -22,7 +22,8 @@ import { RequestId } from "../../rpc/RpcMessage.ts"
 import * as RpcServer from "../../rpc/RpcServer.ts"
 import { AlreadyProcessingMessage, EntityNotAssignedToRunner, MailboxFull, MalformedMessage } from "../ClusterError.ts"
 import * as ClusterMetrics from "../ClusterMetrics.ts"
-import { isUninterruptibleForServer, Persisted } from "../ClusterSchema.ts"
+import { isUninterruptibleForServer, Persisted, WithTransaction } from "../ClusterSchema.ts"
+import * as ClusterSchema from "../ClusterSchema.ts"
 import type { Entity, HandlersFrom } from "../Entity.ts"
 import { CurrentAddress, CurrentRunnerAddress, KeepAliveLatch, KeepAliveRpc, Request } from "../Entity.ts"
 import type { EntityAddress } from "../EntityAddress.ts"
@@ -187,9 +188,9 @@ export const make = Effect.fnUntraced(function*<
                 // interrupt.
                 if (
                   storageEnabled &&
-                  ServiceMap.get(request.rpc.annotations, Persisted) &&
+                  ServiceMap.get(request.message.annotations, Persisted) &&
                   Exit.hasInterrupts(response.exit) &&
-                  (isShuttingDown || isUninterruptibleForServer(request.rpc.annotations))
+                  (isShuttingDown || isUninterruptibleForServer(request.message.annotations))
                 ) {
                   if (!isShuttingDown) {
                     return server.write(0, {
@@ -395,7 +396,7 @@ export const make = Effect.fnUntraced(function*<
               }
 
               const rpc = entityRpcs.get(message.envelope.tag)! as any as Rpc.AnyWithProps
-              if (!storageEnabled && ServiceMap.get(rpc.annotations, Persisted)) {
+              if (!storageEnabled && ServiceMap.get(message.annotations, Persisted)) {
                 return Effect.die(
                   "EntityManager.sendLocal: Cannot process a persisted message without MessageStorage"
                 )
@@ -448,7 +449,7 @@ export const make = Effect.fnUntraced(function*<
                 })
               }
               server.activeRequests.set(message.envelope.requestId, entry)
-              return server.write(0, {
+              let write = server.write(0, {
                 ...message.envelope,
                 id: RequestId(message.envelope.requestId),
                 payload: new Request({
@@ -459,6 +460,10 @@ export const make = Effect.fnUntraced(function*<
                   )
                 })
               })
+              if (ServiceMap.get(message.annotations, WithTransaction)) {
+                write = options.storage.withTransaction(write)
+              }
+              return write
             }
             case "IncomingEnvelope": {
               const entry = server.activeRequests.get(message.envelope.requestId)
@@ -558,6 +563,10 @@ export const make = Effect.fnUntraced(function*<
             const rpc = entityRpcs.get(decoded.envelope.tag)!
             return sendLocal(
               new Message.IncomingRequestLocal({
+                annotations: ServiceMap.get(rpc.annotations, ClusterSchema.Dynamic)(
+                  rpc.annotations,
+                  decoded.envelope as any
+                ),
                 envelope: decoded.envelope,
                 lastSentReply: decoded.lastSentReply,
                 respond: (reply) =>

package/src/unstable/http/index.ts

@@ -137,9 +137,9 @@ export * as Template from "./Template.ts"
 /**
  * @since 4.0.0
  */
-export * as UrlParams from "./UrlParams.ts"
+export * as Url from "./Url.ts"
 
 /**
  * @since 4.0.0
  */
-export * as Url from "./Url.ts"
+export * as UrlParams from "./UrlParams.ts"

package/src/unstable/httpapi/HttpApiSchema.ts

@@ -299,7 +299,7 @@ export function asJson(options?: {
 export function asFormUrlEncoded(options?: {
   readonly contentType?: string
 }) {
-  return <S extends Schema.Top & { readonly Encoded: Record<string, string | ReadonlyArray<string> | undefined> }>(
+  return <S extends Schema.Top>(
     self: S
   ) => asNonMultipartEncoding(self, { _tag: "FormUrlEncoded", ...options })
 }

package/src/unstable/httpapi/OpenApi.ts

@@ -745,6 +745,7 @@ export interface OpenAPISpecExternalDocs {
 export interface OpenAPISpecLicense {
   name: string
   url?: string
+  [key: string]: unknown
 }
 
 /**

package/src/unstable/rpc/Rpc.ts

@@ -171,6 +171,7 @@ export interface Any extends Pipeable {
   readonly [TypeId]: typeof TypeId
   readonly _tag: string
   readonly key: string
+  readonly annotations: ServiceMap.ServiceMap<never>
 }
 
 /**

package/src/unstable/rpc/RpcClient.ts

@@ -825,10 +825,20 @@ export const makeProtocolHttp = (client: HttpClient.HttpClient): Effect.Effect<
   Protocol.make(Effect.fnUntraced(function*(writeResponse) {
     const serialization = yield* RpcSerialization.RpcSerialization
     const isFramed = serialization.includesFraming
+    const httpClientError = (cause: any) =>
+      new RpcClientError({
+        reason: HttpClientErrorSchema.fromHttpClientError(cause)
+      })
+    const protocolDefect = (message: string, cause: unknown) =>
+      new RpcClientError({
+        reason: new RpcClientDefect({ message, cause })
+      })
+    const emptyResponseError = (request: FromClientEncoded) =>
+      protocolDefect("Received empty HTTP response from RPC server", request)
 
-    const send = (request: FromClientEncoded): Effect.Effect<void, RpcClientError> => {
+    const send = Effect.fnUntraced(function*(request: FromClientEncoded) {
       if (request._tag !== "Request") {
-        return Effect.void
+        return
       }
 
       const parser = serialization.makeUnsafe()
@@ -838,34 +848,37 @@ export const makeProtocolHttp = (client: HttpClient.HttpClient): Effect.Effect<
         HttpBody.text(encoded, serialization.contentType) :
         HttpBody.uint8Array(encoded, serialization.contentType)
 
+      const response = yield* client.post("", { body }).pipe(Effect.mapError(httpClientError))
+
       if (!isFramed) {
-        return client.post("", { body }).pipe(
-          Effect.flatMap((r) => r.text),
-          Effect.mapError((cause) =>
-            new RpcClientError({
-              reason: HttpClientErrorSchema.fromHttpClientError(cause)
-            })
-          ),
-          Effect.flatMap((text) => {
-            const u = parser.decode(text)
-            if (!Array.isArray(u)) {
-              return Effect.die(`Expected an array of responses, but got: ${u}`)
-            }
-            let i = 0
-            return Effect.whileLoop({
-              while: () => i < u.length,
-              body: () => writeResponse(u[i++]),
-              step: constVoid
-            })
-          })
-        )
+        const text = yield* response.text.pipe(Effect.mapError(httpClientError))
+        const responses = yield* Effect.try({
+          try: () => parser.decode(text),
+          catch: (cause) => protocolDefect("Error decoding HTTP response", cause)
+        })
+        if (!Array.isArray(responses)) {
+          return yield* Effect.fail(protocolDefect("Expected an array of responses", responses))
+        }
+        if (responses.length === 0) {
+          return yield* Effect.fail(emptyResponseError(request))
+        }
+        let i = 0
+        return yield* Effect.whileLoop({
+          while: () => i < responses.length,
+          body: () => writeResponse(responses[i++]),
+          step: constVoid
+        })
       }
 
-      return client.post("", { body }).pipe(
-        Effect.flatMap((r) =>
-          Stream.runForEachArray(r.stream, (chunk) => {
-            const responses = chunk.flatMap(parser.decode) as Array<FromServerEncoded>
+      let hasResponse = false
+      yield* Stream.runForEachArray(response.stream, (chunk) =>
+        Effect.try({
+          try: () => chunk.flatMap(parser.decode) as Array<FromServerEncoded>,
+          catch: (cause) => protocolDefect("Error decoding HTTP response", cause)
+        }).pipe(
+          Effect.flatMap((responses) => {
             if (responses.length === 0) return Effect.void
+            hasResponse = true
             let i = 0
             return Effect.whileLoop({
               while: () => i < responses.length,
@@ -873,14 +886,13 @@ export const makeProtocolHttp = (client: HttpClient.HttpClient): Effect.Effect<
               step: constVoid
             })
           })
-        ),
-        Effect.mapError((cause) =>
-          new RpcClientError({
-            reason: HttpClientErrorSchema.fromHttpClientError(cause)
-          })
+        )).pipe(
+          Effect.mapError((cause) => cause instanceof RpcClientError ? cause : httpClientError(cause))
         )
-      )
-    }
+      if (!hasResponse) {
+        return yield* Effect.fail(emptyResponseError(request))
+      }
+    })
 
     return {
       send,

package/src/unstable/socket/Socket.ts

@@ -392,7 +392,7 @@ export const makeChannel = <IE = never>(): Channel.Channel<
 /**
  * @since 4.0.0
  */
-export const defaultCloseCodeIsError = (code: number) => code !== 1000 && code !== 1006
+export const defaultCloseCodeIsError = (_code: number) => true
 
 /**
  * @since 4.0.0

package/src/unstable/workflow/Activity.ts

@@ -39,6 +39,14 @@ export interface Activity<
   readonly successSchema: Success
   readonly errorSchema: Error
   readonly exitSchema: Schema.Exit<Success, Error, Schema.Defect>
+  readonly annotations: ServiceMap.ServiceMap<never>
+  annotate<I, S>(
+    key: ServiceMap.Key<I, S>,
+    value: S
+  ): Activity<Success, Error, R>
+  annotateMerge<I>(
+    annotations: ServiceMap.ServiceMap<I>
+  ): Activity<Success, Error, R>
   readonly execute: Effect.Effect<
     Success["Type"],
     Error["Type"],
@@ -73,6 +81,7 @@ export interface Any {
   readonly [TypeId]: typeof TypeId
   readonly name: string
   readonly executeEncoded: Effect.Effect<any, any, any>
+  readonly annotations: ServiceMap.ServiceMap<never>
 }
 
 /**
@@ -101,6 +110,7 @@ export const make = <
   readonly error?: Error | undefined
   readonly execute: Effect.Effect<Success["Type"], Error["Type"], R>
   readonly interruptRetryPolicy?: Schedule.Schedule<any, Cause.Cause<unknown>> | undefined
+  readonly annotations?: ServiceMap.ServiceMap<never> | undefined
 }): Activity<Success, Error, Exclude<R, WorkflowInstance | WorkflowEngine | Scope>> => {
   const successSchema = options.success ?? (Schema.Void as any as Success)
   const errorSchema = options.error ?? (Schema.Never as any as Error)
@@ -120,6 +130,19 @@ export const make = <
     successSchema,
     errorSchema,
     exitSchema: Schema.Exit(successSchemaJson, errorSchemaJson, Schema.Defect),
+    annotations: options.annotations ?? ServiceMap.empty(),
+    annotate(tag: ServiceMap.Key<any, any>, value: any) {
+      return make({
+        ...options,
+        annotations: ServiceMap.add(self.annotations, tag, value)
+      })
+    },
+    annotateMerge(context: ServiceMap.ServiceMap<any>) {
+      return make({
+        ...options,
+        annotations: ServiceMap.merge(self.annotations, context)
+      })
+    },
     execute: executeWithoutInterrupt,
     executeEncoded: Effect.matchEffect(executeWithoutInterrupt, {
       onFailure: (error) => Effect.flatMap(Effect.orDie(Schema.encodeEffect(errorSchemaJson)(error)), Effect.fail),