effect 4.0.5 → 4.0.6

package/src/encoding/Ndjson.ts

@@ -9,17 +9,20 @@
  * @since 4.0.0
  */
 import * as Arr from "../Array.ts"
+import * as Cause from "../Cause.ts"
 import * as Channel from "../Channel.ts"
 import * as ChannelSchema from "../ChannelSchema.ts"
 import * as Data from "../Data.ts"
 import * as Effect from "../Effect.ts"
-import { dual, identity } from "../Function.ts"
+import { dual } from "../Function.ts"
 import * as Predicate from "../Predicate.ts"
+import * as Pull from "../Pull.ts"
 import type * as Schema from "../Schema.ts"
 
 const NdjsonErrorTypeId = "~effect/encoding/Ndjson/NdjsonError"
 
 const encoder = new TextEncoder()
+const utf8Decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true })
 
 /**
  * Error raised when NDJSON encoding or decoding fails.
@@ -29,12 +32,18 @@ const encoder = new TextEncoder()
  * The `kind` field identifies whether the failure happened while packing or
  * unpacking, and `cause` preserves the original error.
  *
+ * Decoders also set `reason` to say why a line was rejected, and
+ * `unterminated` when that line is the final line of the input and has no
+ * line terminator.
+ *
  * @stability unstable
  * @category errors
  * @since 4.0.0
  */
 export class NdjsonError extends Data.TaggedError("NdjsonError")<{
   readonly kind: "Pack" | "Unpack"
+  readonly reason?: NdjsonErrorReason | undefined
+  readonly unterminated?: boolean | undefined
   readonly cause: unknown
 }> {
   /**
@@ -56,6 +65,21 @@ export class NdjsonError extends Data.TaggedError("NdjsonError")<{
   }
 }
 
+/**
+ * Why a decoder rejected a line.
+ *
+ * **Details**
+ *
+ * - `InvalidJson`: the line is not valid JSON.
+ * - `InvalidUtf8`: the line is not valid UTF-8 (byte decoders only).
+ * - `LineTooLarge`: the line is longer than `maxLineBytes` (byte decoders only).
+ *
+ * @stability unstable
+ * @category errors
+ * @since 4.0.6
+ */
+export type NdjsonErrorReason = "InvalidJson" | "InvalidUtf8" | "LineTooLarge"
+
 /**
  * Checks whether a value is an `NdjsonError`.
  *
@@ -169,6 +193,59 @@ export const encodeSchemaString = <S extends Schema.Constraint>(
   S["EncodingServices"]
 > => Channel.pipeTo(ChannelSchema.encode(schema)(), encodeString())
 
+// Collects the values of one input chunk into `out`. A failure stops the
+// chunk, and the values collected before it are still emitted.
+type DecodeStep<In> = (input: In, out: Array<unknown>) => NdjsonError | undefined
+
+const makeDecodePull = <In, IE, Done>(
+  upstream: Pull.Pull<In, IE, Done>,
+  step: DecodeStep<In>,
+  flush: (out: Array<unknown>) => NdjsonError | undefined
+): Pull.Pull<Arr.NonEmptyReadonlyArray<unknown>, IE | NdjsonError, Done> => {
+  let failure: NdjsonError | undefined
+  let done: { readonly leftover: Done } | undefined
+
+  const emit = (out: Array<unknown>, error: NdjsonError | undefined) => {
+    failure = error
+    if (Arr.isArrayNonEmpty(out)) return Effect.succeed(out)
+    return error !== undefined ? Effect.fail(error) : undefined
+  }
+
+  const pull: Pull.Pull<Arr.NonEmptyReadonlyArray<unknown>, IE | NdjsonError, Done> = Effect.suspend(() => {
+    if (failure !== undefined) return Effect.fail(failure)
+    if (done !== undefined) return Cause.done(done.leftover)
+    return Pull.matchEffect(upstream, {
+      onSuccess: (input) => {
+        const out: Array<unknown> = []
+        return emit(out, step(input, out)) ?? pull
+      },
+      onFailure: Effect.failCause,
+      onDone: (leftover): Pull.Pull<Arr.NonEmptyReadonlyArray<unknown>, NdjsonError, Done> => {
+        done = { leftover }
+        const out: Array<unknown> = []
+        return emit(out, flush(out)) ?? Cause.done(leftover)
+      }
+    })
+  })
+  return pull
+}
+
+const parseLine = (
+  line: string,
+  out: Array<unknown>,
+  ignoreEmptyLines: boolean,
+  unterminated: boolean
+): NdjsonError | undefined => {
+  if (ignoreEmptyLines && line.length === 0) return
+  try {
+    out.push(JSON.parse(line))
+  } catch (cause) {
+    return new NdjsonError({ kind: "Unpack", reason: "InvalidJson", unterminated, cause })
+  }
+}
+
+const flushNothing = () => undefined
+
 /**
  * Creates a channel that parses NDJSON string chunks into values.
  *
@@ -179,7 +256,9 @@ export const encodeSchemaString = <S extends Schema.Constraint>(
  *
  * **Details**
  *
- * Lines may span input chunks.
+ * Lines may span input chunks. `\n`, `\r\n` and a bare `\r` end a line. The
+ * first line that fails to parse fails the channel with
+ * `reason: "InvalidJson"`, after the lines before it have been emitted.
  *
  * **Gotchas**
  *
@@ -199,28 +278,53 @@ export const decodeString = <IE = never, Done = unknown>(options?: {
   Arr.NonEmptyReadonlyArray<string>,
   IE,
   Done
-> => {
-  const lines = Channel.splitLines<IE, Done>().pipe(
-    options?.ignoreEmptyLines === true ?
-      Channel.filterArray((line) => line.length > 0) :
-      identity
-  )
-  return Channel.mapEffect(lines, (chunk) => {
-    try {
-      return Effect.succeed(Arr.map(chunk, (line) => JSON.parse(line)))
-    } catch (cause) {
-      return Effect.fail(new NdjsonError({ kind: "Unpack", cause }))
-    }
+> =>
+  Channel.fromTransform((upstream, scope) => {
+    const ignoreEmptyLines = options?.ignoreEmptyLines === true
+    // splitLines only emits a line after its input has ended when that line
+    // has no terminator
+    let ended = false
+    const input = Pull.matchEffect(upstream, {
+      onSuccess: Effect.succeed,
+      onFailure: Effect.failCause,
+      onDone: (leftover) => {
+        ended = true
+        return Cause.done(leftover)
+      }
+    })
+    return Effect.map(
+      Channel.toTransform(Channel.splitLines<IE, Done>())(input, scope),
+      (lines) =>
+        makeDecodePull(lines, (chunk, out) => {
+          for (const line of chunk) {
+            const error = parseLine(line, out, ignoreEmptyLines, ended)
+            if (error !== undefined) return error
+          }
+        }, flushNothing)
+    )
   })
-}
+
+const LF = 0x0a
+const CR = 0x0d
+const emptyBytes = new Uint8Array(0)
 
 /**
  * Creates a channel that decodes UTF-8 byte chunks and parses them as NDJSON.
  *
  * **Details**
  *
- * Lines may span input chunks, and `ignoreEmptyLines` controls whether blank
- * lines are skipped before JSON parsing.
+ * Lines may span input chunks. `\n`, `\r\n` and a bare `\r` end a line, and a
+ * leading UTF-8 BOM is skipped. `ignoreEmptyLines` controls whether blank lines
+ * are skipped before JSON parsing.
+ *
+ * The first bad line fails the channel after the lines before it have been
+ * emitted. A line that is not valid UTF-8 fails with `reason: "InvalidUtf8"`
+ * and a line that is not valid JSON fails with `reason: "InvalidJson"`.
+ *
+ * `maxLineBytes` limits the UTF-8 bytes of a line, not counting its
+ * terminator. A longer line fails with `reason: "LineTooLarge"` as soon as the
+ * limit is passed, without reading the rest of the line. Lines are unbounded
+ * by default.
  *
  * @stability unstable
  * @category constructors
@@ -228,6 +332,7 @@ export const decodeString = <IE = never, Done = unknown>(options?: {
  */
 export const decode = <IE = never, Done = unknown>(options?: {
   readonly ignoreEmptyLines?: boolean | undefined
+  readonly maxLineBytes?: number | undefined
 }): Channel.Channel<
   Arr.NonEmptyReadonlyArray<unknown>,
   IE | NdjsonError,
@@ -235,9 +340,88 @@ export const decode = <IE = never, Done = unknown>(options?: {
   Arr.NonEmptyReadonlyArray<Uint8Array>,
   IE,
   Done
-> => {
-  return Channel.pipeTo(Channel.decodeText(), decodeString(options))
-}
+> =>
+  Channel.fromTransform((upstream, _scope) =>
+    Effect.sync(() => {
+      const ignoreEmptyLines = options?.ignoreEmptyLines === true
+      const maxLineBytes = options?.maxLineBytes ?? Number.POSITIVE_INFINITY
+      // The first decoder strips a leading BOM. `\n` and `\r` never occur
+      // inside a multi-byte character, so each line is decoded on its own.
+      let decoder = new TextDecoder("utf-8", { fatal: true })
+      // Bytes of the current line from earlier chunks
+      let pending: Array<Uint8Array> = []
+      let pendingBytes = 0
+      // The previous byte was `\r`, so a `\n` right after it ends nothing
+      let afterCR = false
+
+      const takeLine = (chunk: Uint8Array, start: number, end: number): Uint8Array => {
+        if (pendingBytes === 0) return chunk.subarray(start, end)
+        const line = new Uint8Array(pendingBytes + end - start)
+        let offset = 0
+        for (const bytes of pending) {
+          line.set(bytes, offset)
+          offset += bytes.length
+        }
+        line.set(chunk.subarray(start, end), offset)
+        pending = []
+        pendingBytes = 0
+        return line
+      }
+
+      const decodeLine = (bytes: Uint8Array, out: Array<unknown>, unterminated: boolean) => {
+        let line: string
+        try {
+          line = decoder.decode(bytes)
+        } catch (cause) {
+          return new NdjsonError({ kind: "Unpack", reason: "InvalidUtf8", unterminated, cause })
+        }
+        const firstLine = decoder !== utf8Decoder
+        decoder = utf8Decoder
+        // A stripped BOM alone is not an unterminated record.
+        if (firstLine && unterminated && line.length === 0) return
+        return parseLine(line, out, ignoreEmptyLines, unterminated)
+      }
+
+      const step: DecodeStep<Arr.NonEmptyReadonlyArray<Uint8Array>> = (chunks, out) => {
+        for (const chunk of chunks) {
+          let start = 0
+          for (let i = 0; i < chunk.length; i++) {
+            const byte = chunk[i]
+            if (byte === LF || byte === CR) {
+              if (byte === LF && afterCR) {
+                start = i + 1
+              } else {
+                const error = decodeLine(takeLine(chunk, start, i), out, false)
+                if (error !== undefined) return error
+                start = i + 1
+              }
+              afterCR = byte === CR
+            } else {
+              afterCR = false
+              if (pendingBytes + i + 1 - start > maxLineBytes) {
+                return new NdjsonError({
+                  kind: "Unpack",
+                  reason: "LineTooLarge",
+                  unterminated: false,
+                  cause: new RangeError(`NDJSON line exceeds maxLineBytes of ${maxLineBytes}`)
+                })
+              }
+            }
+          }
+          if (start < chunk.length) {
+            // Copy, so a reused upstream buffer cannot change the line
+            pending.push(chunk.slice(start))
+            pendingBytes += chunk.length - start
+          }
+        }
+      }
+
+      const flush = (out: Array<unknown>) =>
+        pendingBytes === 0 ? undefined : decodeLine(takeLine(emptyBytes, 0, 0), out, true)
+
+      return makeDecodePull(upstream, step, flush)
+    })
+  )
 
 /**
  * Creates an NDJSON byte decoder channel for values of a schema.
@@ -256,6 +440,7 @@ export const decodeSchema = <S extends Schema.Constraint>(
 ) =>
 <IE = never, Done = unknown>(options?: {
   readonly ignoreEmptyLines?: boolean | undefined
+  readonly maxLineBytes?: number | undefined
 }): Channel.Channel<
   Arr.NonEmptyReadonlyArray<S["Type"]>,
   Schema.SchemaError | NdjsonError | IE,
@@ -321,6 +506,7 @@ export const duplex: {
   (
     options?: {
       readonly ignoreEmptyLines?: boolean | undefined
+      readonly maxLineBytes?: number | undefined
     }
   ): <R, IE, OE, OutDone, InDone>(
     self: Channel.Channel<
@@ -365,6 +551,7 @@ export const duplex: {
     >,
     options?: {
       readonly ignoreEmptyLines?: boolean | undefined
+      readonly maxLineBytes?: number | undefined
     }
   ): Channel.Channel<
     Arr.NonEmptyReadonlyArray<unknown>,
@@ -387,6 +574,7 @@ export const duplex: {
   >,
   options?: {
     readonly ignoreEmptyLines?: boolean | undefined
+    readonly maxLineBytes?: number | undefined
   }
 ): Channel.Channel<
   Arr.NonEmptyReadonlyArray<unknown>,
@@ -543,6 +731,7 @@ export const duplexSchema: {
       readonly inputSchema: In
       readonly outputSchema: Out
       readonly ignoreEmptyLines?: boolean | undefined
+      readonly maxLineBytes?: number | undefined
     }
   ): <OutErr, OutDone, InErr, InDone, R>(
     self: Channel.Channel<
@@ -590,6 +779,7 @@ export const duplexSchema: {
       readonly inputSchema: In
       readonly outputSchema: Out
       readonly ignoreEmptyLines?: boolean | undefined
+      readonly maxLineBytes?: number | undefined
     }
   ): Channel.Channel<
     Arr.NonEmptyReadonlyArray<Out["Type"]>,
@@ -614,6 +804,7 @@ export const duplexSchema: {
     readonly inputSchema: In
     readonly outputSchema: Out
     readonly ignoreEmptyLines?: boolean | undefined
+    readonly maxLineBytes?: number | undefined
   }
 ): Channel.Channel<
   Arr.NonEmptyReadonlyArray<Out["Type"]>,

package/src/encoding/SchemaBinary.ts

@@ -13,6 +13,7 @@
  */
 import * as Arr from "../Array.ts"
 import * as BigDecimal from "../BigDecimal.ts"
+import * as Bytes from "../Bytes.ts"
 import * as Cause from "../Cause.ts"
 import * as Channel from "../Channel.ts"
 import * as Chunk from "../Chunk.ts"
@@ -43,7 +44,7 @@ import * as SchemaTransformation from "../SchemaTransformation.ts"
  *
  * The default mode supports compatible schema evolution. `fingerprint: true`
  * uses positional layouts and an 8-byte layout hash for smaller frames, but
- * requires peers to use the same schema definition.
+ * requires peers to use an identical encoded layout.
  *
  * @stability unstable
  * @category models
@@ -92,6 +93,14 @@ const directFingerprintCodecCache = new WeakMap<Schema.Constraint, toCodec<any>>
  * Encoded results are arena-backed views and may share a larger buffer. Use
  * `bytes.slice()` when independent ownership is required.
  *
+ * **Gotchas**
+ *
+ * Bytes carry the layout of the schema that wrote them. In the default mode,
+ * decode them with that schema or a compatible evolution of it; fingerprint
+ * mode requires an identical encoded layout. Most union codecs prefix values
+ * with a member selector, so they cannot read bytes written by a member's own
+ * codec.
+ *
  * **Example** (Deriving a binary codec)
  *
  * ```ts
@@ -248,16 +257,7 @@ export function encodeManyUnknownSync<S extends Schema.Constraint>(
 }
 
 function concatFrames(frames: ReadonlyArray<Uint8Array<ArrayBuffer>>): Uint8Array<ArrayBuffer> {
-  if (frames.length === 1) return frames[0]
-  let length = 0
-  for (const frame of frames) length += frame.length
-  const out = new Uint8Array(length)
-  let offset = 0
-  for (const frame of frames) {
-    out.set(frame, offset)
-    offset += frame.length
-  }
-  return out
+  return frames.length === 1 ? frames[0] : Bytes.concat(frames)
 }
 
 /**
@@ -907,33 +907,7 @@ export const duplex: {
   InDone,
   R | In["EncodingServices"] | Out["DecodingServices"]
 > =>
-  encode(options.inputSchema, options)</**
-   * Wraps a bidirectional byte channel with schema-driven binary encoding and
-   * decoding.
-   *
-   * **Details**
-   *
-   * Values sent to the wrapped channel are encoded with `inputSchema` as binary
-   * frames; bytes received from it are decoded with `outputSchema`.
-   *
-   * @stability unstable
-   * @category channels
-   * @since 4.0.0
-   */
-  InErr, /**
-   * Wraps a bidirectional byte channel with schema-driven binary encoding and
-   * decoding.
-   *
-   * **Details**
-   *
-   * Values sent to the wrapped channel are encoded with `inputSchema` as binary
-   * frames; bytes received from it are decoded with `outputSchema`.
-   *
-   * @stability unstable
-   * @category channels
-   * @since 4.0.0
-   */
-  InDone>().pipe(
+  encode(options.inputSchema, options)<InErr, InDone>().pipe(
     Channel.pipeTo(self),
     Channel.pipeTo(decode(options.outputSchema, options)())
   ))
@@ -1126,14 +1100,6 @@ function pushU64(out: Array<number>, n: bigint) {
   }
 }
 
-function compareBytes(a: Uint8Array, b: Uint8Array): number {
-  const len = Math.min(a.length, b.length)
-  for (let i = 0; i < len; i++) {
-    if (a[i] !== b[i]) return a[i] - b[i]
-  }
-  return a.length - b.length
-}
-
 class IssueError extends Error {
   readonly issue: SchemaIssue.Issue
   constructor(issue: SchemaIssue.Issue) {
@@ -2102,7 +2068,7 @@ function sentinelSetHash(sentinels: ReadonlyArray<SchemaAST.Sentinel>): number {
     const bn = typeof b.key === "number"
     if (an !== bn) return an ? -1 : 1
     if (an) return (a.key as number) - (b.key as number)
-    return compareBytes(utf8Encode.encode(a.key as string), utf8Encode.encode(b.key as string))
+    return Bytes.Order(utf8Encode.encode(a.key as string), utf8Encode.encode(b.key as string))
   })
   const out: Array<number> = []
   for (const sentinel of sorted) {
@@ -3223,7 +3189,7 @@ function extraPairs(ctx: EncodeContext, layout: StructLayout, obj: Record<string
     pairs.sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
   } else {
     for (const pair of pairs) pair[2] = utf8Encode.encode(pair[0])
-    pairs.sort((a, b) => compareBytes(a[2]!, b[2]!))
+    pairs.sort((a, b) => Bytes.Order(a[2]!, b[2]!))
   }
   if (persistent) {
     for (const pair of pairs) pair[2] ??= utf8Encode.encode(pair[0])