effect 4.0.0-rc.111 → 4.0.0-rc.113

package/src/unstable/rpc/RpcSerialization.ts

@@ -3,19 +3,45 @@
  *
  * `RpcSerialization` is the boundary between `RpcMessage` envelopes and the
  * bytes or strings carried by a transport. This module provides built-in
- * serializers for JSON, newline-delimited JSON, JSON-RPC 2.0, and MessagePack,
- * including framed formats that can decode multiple messages from streaming
- * chunks.
+ * serializers for JSON, newline-delimited JSON, JSON-RPC 2.0, and SchemaBinary,
+ * including framed formats that can decode multiple messages
+ * from streaming chunks.
  *
  * @since 4.0.0
  */
-import * as Msgpackr from "msgpackr"
 import * as Context from "../../Context.ts"
 import * as Data from "../../Data.ts"
 import * as Layer from "../../Layer.ts"
 import * as Predicate from "../../Predicate.ts"
 import { hasProperty } from "../../Predicate.ts"
-import type * as RpcMessage from "./RpcMessage.ts"
+import * as Schema from "../../Schema.ts"
+import * as SchemaBinary from "../encoding/SchemaBinary.ts"
+import * as RpcMessage from "./RpcMessage.ts"
+
+/**
+ * Builds the codec used to fill the `unknown` holes of RPC protocol messages,
+ * such as request payloads, stream chunks, exits, and defects.
+ *
+ * **Details**
+ *
+ * The envelope around the hole is framed by {@link Parser}, which never sees the
+ * schema. JSON based serializations return `Schema.toCodecJson`, so the encoded
+ * hole is JSON. Other serializations may encode the hole as bytes; both fit the
+ * `unknown` hole, so `RpcSerialization` has no type parameter.
+ *
+ * @category serialization
+ * @since 4.0.0
+ */
+export type CodecFor = <S extends Schema.Top>(
+  schema: S
+) => Schema.Codec<S["Type"], unknown, S["DecodingServices"], S["EncodingServices"]>
+
+const codecForJson = Schema.toCodecJson as CodecFor
+
+// shared by every parser that decodes whole frames, so creating a parser does
+// not pay for a native TextDecoder
+let sharedTextDecoder: TextDecoder | undefined
+const decodeText = (bytes: Uint8Array): string => (sharedTextDecoder ??= new TextDecoder()).decode(bytes)
 
 /**
  * Service that describes how RPC protocol messages are encoded and decoded,
@@ -34,6 +60,7 @@ export class RpcSerialization extends Context.Service<RpcSerialization, {
   makeUnsafe(): Parser
   readonly contentType: string
   readonly includesFraming: boolean
+  readonly codecFor: CodecFor
 }>()("effect/rpc/RpcSerialization") {}
 
 /**
@@ -95,16 +122,14 @@ const isBufferSizeExceeded = (
 export const json: RpcSerialization["Service"] = RpcSerialization.of({
   contentType: "application/json",
   includesFraming: false,
-  makeUnsafe: () => {
-    const decoder = new TextDecoder()
-    return {
-      decode: (bytes) => {
-        const decoded = JSON.parse(typeof bytes === "string" ? bytes : decoder.decode(bytes))
-        return Array.isArray(decoded) ? decoded : [decoded]
-      },
-      encode: (response) => JSON.stringify(response)
-    }
-  }
+  codecFor: codecForJson,
+  makeUnsafe: () => ({
+    decode: (bytes) => {
+      const decoded = JSON.parse(typeof bytes === "string" ? bytes : decodeText(bytes))
+      return Array.isArray(decoded) ? decoded : [decoded]
+    },
+    encode: (response) => JSON.stringify(response)
+  })
 })
 
 /**
@@ -119,8 +144,10 @@ export const makeNdjson = (options?: StreamOptions): RpcSerialization["Service"]
   return RpcSerialization.of({
     contentType: "application/ndjson",
     includesFraming: true,
+    codecFor: codecForJson,
     makeUnsafe: () => {
-      const decoder = new TextDecoder()
+      // lazily created: string transports never need byte decoding
+      let decoder: TextDecoder | undefined
       let buffer = ""
       const failMaxBufferSize = (maxBufferSize: number): never => {
         buffer = ""
@@ -128,7 +155,7 @@ export const makeNdjson = (options?: StreamOptions): RpcSerialization["Service"]
       }
       return ({
         decode: (bytes) => {
-          buffer += typeof bytes === "string" ? bytes : decoder.decode(bytes, { stream: true })
+          buffer += typeof bytes === "string" ? bytes : (decoder ??= new TextDecoder()).decode(bytes, { stream: true })
           let position = 0
           let nlIndex = buffer.indexOf("\n", position)
           const items: Array<unknown> = []
@@ -184,8 +211,8 @@ export const jsonRpc = (options?: {
   RpcSerialization.of({
     contentType: options?.contentType ?? "application/json",
     includesFraming: false,
+    codecFor: codecForJson,
     makeUnsafe: () => {
-      const decoder = new TextDecoder()
       const batches = new Map<string | number, {
         readonly size: number
         readonly responses: Map<string | number, RpcMessage.FromServerEncoded>
@@ -193,7 +220,7 @@ export const jsonRpc = (options?: {
       return {
         decode: (bytes) => {
           const decoded: JsonRpcMessage | Array<JsonRpcMessage> = JSON.parse(
-            typeof bytes === "string" ? bytes : decoder.decode(bytes)
+            typeof bytes === "string" ? bytes : decodeText(bytes)
           )
           return decodeJsonRpcRaw(decoded, batches)
         },
@@ -219,6 +246,7 @@ export const ndJsonRpc = (options?: {
   RpcSerialization.of({
     contentType: options?.contentType ?? "application/json-rpc",
     includesFraming: true,
+    codecFor: codecForJson,
     makeUnsafe: () => {
       const parser = makeNdjson({ maxBufferSize: options?.maxBufferSize }).makeUnsafe()
       const batches = new Map<string, {
@@ -278,7 +306,7 @@ function decodeJsonRpcMessage(decoded: JsonRpcMessage): RpcMessage.FromClientEnc
         | RpcMessage.FromServerEncoded["_tag"]
         | Exclude<RpcMessage.FromClientEncoded["_tag"], "Request">
       const requestId = (request as any).params?.requestId
-      return requestId ?
+      return requestId !== undefined ?
         {
           _tag: tag,
           requestId
@@ -498,71 +526,42 @@ interface JsonRpcResponse {
 
 type JsonRpcMessage = JsonRpcRequest | JsonRpcResponse
 
-/**
- * Create a MessagePack serialization with custom msgpackr options.
- *
- * @category serialization
- * @since 4.0.0
- */
-export const makeMsgPack = (
-  options?: (Msgpackr.Options & StreamOptions) | undefined
-): RpcSerialization["Service"] => {
-  const { maxBufferSize = defaultMaxBufferSize, ...msgpackOptions } = options ?? {}
+const defaultSchemaBinaryMaxFrameSize = 16 * 1024 * 1024
+
+const schemaBinaryTextEncoder = new TextEncoder()
+
+const makeSchemaBinary = (options?: {
+  readonly maxFrameSize?: number | "unbounded" | undefined
+  readonly fingerprintPayloads?: boolean | undefined
+}): RpcSerialization["Service"] => {
+  const maxFrameSize = options?.maxFrameSize === "unbounded"
+    ? undefined
+    : options?.maxFrameSize ?? defaultSchemaBinaryMaxFrameSize
+  const codecFor: CodecFor = options?.fingerprintPayloads === true
+    ? (schema) => SchemaBinary.toCodecDirect(schema, { fingerprint: true })
+    : SchemaBinary.toCodecDirect
+  const envelopeOptions = { fingerprint: true } as const
   return RpcSerialization.of({
-    contentType: "application/msgpack",
+    contentType: "application/vnd.effect.rpc+schema-binary",
     includesFraming: true,
+    codecFor,
     makeUnsafe: () => {
-      const unpackr = new Msgpackr.Unpackr(msgpackOptions)
-      const packr = new Msgpackr.Packr(msgpackOptions)
-      const encoder = new TextEncoder()
-      let incomplete: Uint8Array | undefined = undefined
-      const failMaxBufferSize = (maxBufferSize: number): never => {
-        incomplete = undefined
-        throw new MaxBufferSizeExceeded({ maxBufferSize })
-      }
+      const parser = SchemaBinary.parser(RpcMessage.EncodedSchema, { ...envelopeOptions, maxFrameSize })
+      const encoder = SchemaBinary.encoder(RpcMessage.EncodedSchema, envelopeOptions)
       return {
-        decode(bytes) {
-          let buf = typeof bytes === "string" ? encoder.encode(bytes) : bytes
-          if (incomplete !== undefined) {
-            if (isBufferSizeExceeded(incomplete.length + buf.length, maxBufferSize)) {
-              failMaxBufferSize(maxBufferSize)
-            }
-            const prev = buf
-            bytes = new Uint8Array(incomplete.length + buf.length)
-            bytes.set(incomplete)
-            bytes.set(prev, incomplete.length)
-            buf = bytes
-            incomplete = undefined
-          }
-          try {
-            return unpackr.unpackMultiple(buf)
-          } catch (error_) {
-            const error = error_ as any
-            if (error.incomplete) {
-              incomplete = buf.subarray(error.lastPosition)
-              if (isBufferSizeExceeded(incomplete.length, maxBufferSize)) {
-                failMaxBufferSize(maxBufferSize)
-              }
-              return error.values ?? []
-            }
-            throw error_
+        decode: (data) => parser.feedSync(typeof data === "string" ? schemaBinaryTextEncoder.encode(data) : data),
+        encode: (response) => {
+          if (!Array.isArray(response)) {
+            return encoder.encode(response)
           }
-        },
-        encode: (response) => packr.pack(response)
+          if (response.length === 0) return undefined
+          return encoder.encodeMany(response)
+        }
       }
     }
   })
 }
 
-/**
- * Default MessagePack RPC serialization using record support and built-in
- * message framing.
- *
- * @category serialization
- * @since 4.0.0
- */
-export const msgPack: RpcSerialization["Service"] = makeMsgPack({ useRecords: true })
-
 /**
  * RPC serialization layer that uses JSON for serialization.
  *
@@ -623,24 +622,15 @@ export const layerNdJsonRpc = (options?: {
 }): Layer.Layer<RpcSerialization> => Layer.succeed(RpcSerialization)(ndJsonRpc(options))
 
 /**
- * RPC serialization layer that uses MessagePack for serialization.
- *
- * **Details**
- *
- * MessagePack has a more compact binary format compared to JSON and NDJSON. It
- * also has better support for binary data.
- *
- * @category layers
- * @since 4.0.0
- */
-export const layerMsgPack: Layer.Layer<RpcSerialization> = Layer.succeed(RpcSerialization)(msgPack)
-
-/**
- * RPC serialization layer that uses MessagePack with custom options.
+ * RPC serialization layer that uses SchemaBinary with fingerprinted RPC
+ * envelopes. Payload fingerprints are disabled by default to support compatible
+ * schema evolution. Frames default to a 16 MiB maximum size. Use `"unbounded"`
+ * to disable the frame-size limit.
  *
  * @category layers
  * @since 4.0.0
  */
-export const layerMsgPackWith = (
-  options?: (Msgpackr.Options & StreamOptions) | undefined
-): Layer.Layer<RpcSerialization> => Layer.succeed(RpcSerialization)(makeMsgPack(options))
+export const layerSchemaBinary = (options?: {
+  readonly maxFrameSize?: number | "unbounded" | undefined
+  readonly fingerprintPayloads?: boolean | undefined
+}): Layer.Layer<RpcSerialization> => Layer.sync(RpcSerialization)(() => makeSchemaBinary(options))