effect 4.0.7 → 4.1.0

package/src/encoding/Hex.ts

@@ -46,8 +46,9 @@ export const encode: (input: Uint8Array | string) => string = (input) =>
  * coercion rules.
  *
  * This function uses `Math.random()` and is not cryptographically secure. For
- * security-sensitive values, use the `Crypto.Crypto` service's `randomBytes`
- * method and encode the result with {@link encode}.
+ * security-sensitive values, draw bytes with the `randomBytes` method of the
+ * `SecureRandom` service from `effect/crypto/SecureRandom`, and encode them with
+ * {@link encode}.
  *
  * @stability stable
  * @category encoding

package/src/encoding/SchemaBinary.ts

@@ -80,6 +80,7 @@ const codecCache = new WeakMap<Schema.Constraint, toCodec<any>>()
 const fingerprintCodecCache = new WeakMap<Schema.Constraint, toCodec<any>>()
 const directCodecCache = new WeakMap<Schema.Constraint, toCodec<any>>()
 const directFingerprintCodecCache = new WeakMap<Schema.Constraint, toCodec<any>>()
+const strictCodecCache = new WeakMap<Schema.Constraint, toCodec<any>>()
 
 /**
  * Derives a compact binary codec from a schema.
@@ -135,6 +136,27 @@ export function toCodec<S extends Schema.Constraint>(schema: S, options?: Option
   return codec
 }
 
+/**
+ * Derives the default-mode {@link toCodec} codec, except that decoding rejects
+ * struct fields the schema does not know instead of skipping them.
+ *
+ * @internal
+ */
+export function toCodecStrict<S extends Schema.Constraint>(schema: S): toCodec<S> {
+  const cached = strictCodecCache.get(schema)
+  if (cached !== undefined) return cached
+  const { exact, layout, recursive, target } = compileTarget(schema)
+  const trusted: Trusted | undefined = exact ? { value: undefined } : undefined
+  const codec = assembleCodec<S>(
+    trusted === undefined ? target : withTrustedDecode(target, trusted, !recursive),
+    layout,
+    strictMode,
+    trusted
+  )
+  strictCodecCache.set(schema, codec)
+  return codec
+}
+
 /**
  * Derives the {@link toCodec} codec without the schema pass around the binary
  * layer.
@@ -571,7 +593,7 @@ function makeParser<T>(
           issuePathLen = 0
           const bodyStart = pos + headerLen
           indexSignatures.beginFrame()
-          body.reset(buf, bodyStart, bodyStart + frameLen, parseOptions, indexSignatures, mode.positional, dict)
+          body.reset(buf, bodyStart, bodyStart + frameLen, parseOptions, indexSignatures, mode, dict)
           const value = decodeFrameBody(layout, body, mode)
           out.push((decodeEncoded === undefined ? value : decodeEncoded(value)) as T)
         } catch (e) {
@@ -1541,6 +1563,7 @@ class Reader {
   options: SchemaAST.ParseOptions = SchemaAST.defaultParseOptions
   indexSignatures: IndexSignatureCache | undefined
   positional = false
+  rejectUnknownFields = false
   // Handed to the row-run field's immediate consumer, then cleared.
   intern: Array<unknown> | undefined
   // String tables shared by every frame on one connection; `undefined` for a
@@ -1552,7 +1575,7 @@ class Reader {
     end: number,
     options: SchemaAST.ParseOptions,
     indexSignatures: IndexSignatureCache | undefined,
-    positional: boolean,
+    mode: Mode,
     dict: DictRead | undefined
   ) {
     this.nesting = 0
@@ -1562,7 +1585,8 @@ class Reader {
     this.end = end
     this.options = options
     this.indexSignatures = indexSignatures
-    this.positional = positional
+    this.positional = mode.positional
+    this.rejectUnknownFields = mode.rejectUnknownFields
     this.intern = undefined
     this.dict = dict
   }
@@ -1576,6 +1600,7 @@ class Reader {
     this.options = SchemaAST.defaultParseOptions
     this.indexSignatures = undefined
     this.positional = false
+    this.rejectUnknownFields = false
     this.intern = undefined
     this.dict = undefined
   }
@@ -2908,6 +2933,7 @@ function fingerprintBytes(lo: number, hi: number): Uint8Array {
 
 interface Mode {
   readonly positional: boolean
+  readonly rejectUnknownFields: boolean
   readonly envelope: number
   readonly expectedEnvelope: string
   readonly fingerprint: Uint8Array
@@ -2915,15 +2941,19 @@ interface Mode {
 
 const defaultMode: Mode = {
   positional: false,
+  rejectUnknownFields: false,
   envelope: ENVELOPE,
   expectedEnvelope: "version 2 envelope, flags 0",
   fingerprint: new Uint8Array(8)
 }
 
+const strictMode: Mode = { ...defaultMode, rejectUnknownFields: true }
+
 function fingerprintMode(layout: Layout): Mode {
   const fingerprint = layoutFingerprint(layout)
   return {
     positional: true,
+    rejectUnknownFields: false,
     envelope: ENVELOPE_FINGERPRINT,
     expectedEnvelope: "version 2 envelope, flags 1",
     fingerprint: fingerprintBytes(
@@ -4275,6 +4305,7 @@ function decodeExtraPair(
   const signature = layout.extraAll ??
     (r.indexSignatures ??= new IndexSignatureCache(r.options)).find(layout, key)
   if (signature === undefined) {
+    if (r.rejectUnknownFields) invalid("a key matching an index signature", key, r.options)
     skipFieldPayload(kind, r)
     return
   }
@@ -4371,6 +4402,7 @@ function decodeScalarField(kind: number, r: Reader): unknown {
 
 function decodeFieldPayload(layout: Layout, wireMask: number, kind: number, r: Reader): unknown {
   if ((wireMask & (1 << kind)) === 0) {
+    if (r.rejectUnknownFields) invalid("matching field wire kind", kind, r.options)
     skipFieldPayload(kind, r)
     return ABSENT
   }
@@ -4427,6 +4459,7 @@ function decodeStruct(layout: StructLayout, r: Reader): unknown {
       if (field !== undefined) cursor = field.index + 1
     }
     if (field === undefined) {
+      if (r.rejectUnknownFields) invalid("a known field id", id, r.options)
       if (seenUnknown === undefined) seenUnknown = new Set()
       else if (seenUnknown.has(id)) invalid("unique field ids", undefined, r.options)
       seenUnknown.add(id)
@@ -4646,6 +4679,7 @@ function decodeRunRow(
       } else {
         slot = struct.byId.get(id)
         if (slot === undefined) {
+          if (r.rejectUnknownFields) invalid("a known field id", id, r.options)
           if (seenIds === undefined) seenIds = new Set([id])
           else if (seenIds.has(id)) invalid("unique field ids", undefined, r.options)
           else seenIds.add(id)
@@ -4843,6 +4877,7 @@ function decodeUnion(layout: UnionLayout, r: Reader): unknown {
     const tag = r.u32le()
     const variant = layout.byTag.get(tag)
     if (variant === undefined) {
+      if (r.rejectUnknownFields) invalid("known union member", tag, r.options)
       r.take(r.remaining)
       return ABSENT
     }
@@ -4857,6 +4892,7 @@ function decodeUnion(layout: UnionLayout, r: Reader): unknown {
   }
   const member = layout.byKind.get(kind)
   if (member === undefined) {
+    if (r.rejectUnknownFields) invalid("known union member", kind, r.options)
     r.take(r.remaining)
     return ABSENT
   }
@@ -5090,7 +5126,7 @@ function decodeOneShot(
   const r = pooledReader ?? new Reader()
   const pooled = r === pooledReader
   if (pooled) pooledReader = undefined
-  r.reset(bytes, 0, bytes.length, options, undefined, mode.positional, undefined)
+  r.reset(bytes, 0, bytes.length, options, undefined, mode, undefined)
   const savedPathLen = issuePathLen
   issuePathLen = 0
   try {

package/src/encoding/Toml.ts

@@ -32,12 +32,34 @@
  * SOFTWARE.
  */
 
+import * as Result from "../Result.ts"
+import { EncodingError } from "./EncodingError.ts"
+
 type Table = Record<string, unknown>
 
 const hasOwn = Object.prototype.hasOwnProperty
 const makeTable = (): Table => Object.create(null)
 const isTable = (value: unknown): value is Table =>
   typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Date)
+const localDatePattern = /^(\d{4})-(\d{2})-(\d{2})$/
+const localTimePattern = /^(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?$/
+const numericOffsetPattern = /^[+-](\d{2}):(\d{2})$/
+const dateTimePattern = /^(\d{4}-\d{2}-\d{2})[Tt](\d{2}:\d{2}:\d{2}(?:\.\d+)?)([Zz]|[+-]\d{2}:\d{2})?$/
+const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+const isValidDate = (date: string): boolean => {
+  const [, year, month, day] = localDatePattern.exec(date)!.map(Number)
+  const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0)
+  const days = month === 2 && leap ? 29 : daysInMonth[month - 1]
+  return month >= 1 && month <= 12 && day >= 1 && day <= days
+}
+const isValidTime = (time: string): boolean => {
+  const [, hour, minute, second] = localTimePattern.exec(time)!.map(Number)
+  return hour <= 23 && minute <= 59 && second <= 60
+}
+const isValidOffset = (offset: string): boolean => {
+  const numeric = numericOffsetPattern.exec(offset)
+  return numeric === null || (Number(numeric[1]) <= 23 && Number(numeric[2]) <= 59)
+}
 
 class TomlParser {
   readonly root = makeTable()
@@ -49,6 +71,8 @@ class TomlParser {
   private readonly explicitTables = new Set<Table>()
   // Tables defined by dotted keys; a header cannot define them again.
   private readonly dottedTables = new Set<Table>()
+  // Values parsed from `[...]` or `{...}` are complete; headers and dotted keys cannot extend them.
+  private readonly inlineValues = new Set<unknown>()
 
   constructor(input: string) {
     this.input = input
@@ -104,6 +128,9 @@ class TomlParser {
       const key = path[index]
       const last = index === path.length - 1
       let value = table[key]
+      if (this.inlineValues.has(value)) {
+        this.fail(`Cannot redefine existing key '${path.slice(0, index + 1).join(".")}'`)
+      }
 
       if (last && array) {
         if (value === undefined) {
@@ -142,7 +169,7 @@ class TomlParser {
         const child = makeTable()
         table[key] = child
         table = child
-      } else if (isTable(existing) && !this.explicitTables.has(existing)) {
+      } else if (isTable(existing) && !this.explicitTables.has(existing) && !this.inlineValues.has(existing)) {
         table = existing
       } else {
         this.fail(`Cannot redefine existing key '${keys.slice(0, index + 1).join(".")}'`)
@@ -198,10 +225,14 @@ class TomlParser {
       return this.parseLiteralString(this.input.startsWith("'''", this.index))
     }
     if (character === "[") {
-      return this.parseArray()
+      const array = this.parseArray()
+      this.inlineValues.add(array)
+      return array
     }
     if (character === "{") {
-      return this.parseInlineTable()
+      const table = this.parseInlineTable()
+      this.inlineValues.add(table)
+      return table
     }
 
     const spaceSeparatedDateTime = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})?)/
@@ -233,21 +264,21 @@ class TomlParser {
   }
 
   private parseDate(token: string): Date | string | undefined {
-    const date = "\\d{4}-\\d{2}-\\d{2}"
-    const time = "\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?"
-    if (new RegExp(`^${date}[Tt]${time}(?:[Zz]|[+-]\\d{2}:\\d{2})$`).test(token)) {
-      const value = new Date(token.replace("t", "T").replace("z", "Z"))
-      if (Number.isNaN(value.getTime())) {
+    const dateTime = dateTimePattern.exec(token)
+    if (dateTime !== null) {
+      const [, date, time, offset] = dateTime
+      // `Date` cannot represent a leap second, so offset date-times reject `:60`
+      const offsetValid = offset === undefined || (isValidOffset(offset) && time.slice(6, 8) !== "60")
+      if (!isValidDate(date) || !isValidTime(time) || !offsetValid) {
         this.fail(`Invalid date-time '${token}'`)
       }
-      return value
+      return offset === undefined ? `${date}T${time}` : new Date(`${date}T${time}${offset.toUpperCase()}`)
     }
-    if (
-      new RegExp(`^${date}[Tt]${time}$`).test(token) ||
-      new RegExp(`^${date}$`).test(token) ||
-      new RegExp(`^${time}$`).test(token)
-    ) {
-      return token.replace("t", "T")
+    if (localDatePattern.test(token)) {
+      return isValidDate(token) ? token : this.fail(`Invalid date-time '${token}'`)
+    }
+    if (localTimePattern.test(token)) {
+      return isValidTime(token) ? token : this.fail(`Invalid date-time '${token}'`)
     }
     return undefined
   }
@@ -292,7 +323,7 @@ class TomlParser {
     while (!this.done) {
       if (multiline && this.input.startsWith("\"\"\"", this.index)) {
         this.advance(3)
-        return output
+        return output + this.closingQuotes("\"")
       }
       const character = this.peek()
       if (!multiline && character === "\"") {
@@ -362,7 +393,7 @@ class TomlParser {
       if (multiline && this.input.startsWith("'''", this.index)) {
         const output = this.input.slice(start, this.index)
         this.advance(3)
-        return output
+        return output + this.closingQuotes("'")
       }
       if (!multiline && this.peek() === "'") {
         const output = this.input.slice(start, this.index)
@@ -377,6 +408,15 @@ class TomlParser {
     this.fail("Unterminated literal string")
   }
 
+  private closingQuotes(quote: string): string {
+    let quotes = ""
+    while (quotes.length < 2 && this.peek() === quote) {
+      quotes += quote
+      this.advance()
+    }
+    return quotes
+  }
+
   private parseArray(): Array<unknown> {
     this.expect("[")
     const output: Array<unknown> = []
@@ -515,11 +555,36 @@ class TomlParser {
  *
  * **Details**
  *
- * Offset date-times are represented by `Date`, matching the package this
- * parser replaces. Local dates and times remain strings.
+ * Returns `Result.succeed` with the parsed record, or `Result.fail` with an
+ * `EncodingError` when the input is not valid TOML. Offset date-times are
+ * represented by `Date`, matching the package this parser replaces. Local
+ * dates and times remain strings.
+ *
+ * **Example** (Parsing a TOML document)
+ *
+ * ```ts import.meta.vitest
+ * import { Result } from "effect"
+ * import { Toml } from "effect/encoding"
+ *
+ * Toml.parse("port = 8080") // => Result.succeed({ port: 8080 })
+ * Result.isFailure(Toml.parse("port = ")) // => true
+ * ```
  *
  * @stability unstable
  * @category decoding
  * @since 4.0.0
  */
-export const parse = (input: string): Record<string, unknown> => new TomlParser(input.replace(/^\uFEFF/, "")).parse()
+export const parse = (input: string): Result.Result<Record<string, unknown>, EncodingError> => {
+  try {
+    return Result.succeed(new TomlParser(input.replace(/^\uFEFF/, "")).parse());
+  } catch (e) {
+    return Result.fail(
+      new EncodingError({
+        module: "Toml",
+        kind: "Decode",
+        input,
+        message: e instanceof Error ? e.message : "Invalid input"
+      })
+    )
+  }
+}

package/src/encoding/Yaml.ts

@@ -2,7 +2,8 @@
  * Parses YAML configuration files.
  *
  * Supports YAML 1.2 block and flow collections, indentless sequence values,
- * multiline plain and quoted scalars, block scalars, anchors, and aliases.
+ * multiline plain and quoted scalars, block scalars, anchors, aliases, and
+ * YAML 1.1 merge keys (`<<`).
  *
  * Unlike YAML 1.2, the closing bracket of a multiline flow collection may sit
  * at its parent's indentation.
@@ -55,6 +56,34 @@ const setProperty = (record: YamlRecord, key: string, value: unknown): void => {
   })
 }
 
+const isMapping = (value: unknown): value is YamlRecord =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+
+// Explicit keys take precedence over merged keys, and earlier merge sources over later ones.
+class MappingBuilder {
+  readonly output: YamlRecord = {}
+  // Keys added by the merge key, which an explicit key may still override.
+  private merged: Set<string> | undefined
+
+  set(key: string, value: unknown, fail: (message: string) => never): void {
+    if (hasOwn.call(this.output, key) && !this.merged?.delete(key)) fail(`Duplicate key '${key}'`)
+    setProperty(this.output, key, value)
+  }
+
+  merge(value: unknown, fail: (message: string) => never): void {
+    if (this.merged !== undefined) fail("Duplicate key '<<'")
+    this.merged = new Set()
+    for (const source of Array.isArray(value) ? value : [value]) {
+      if (!isMapping(source)) fail("Merge key values must be mappings")
+      for (const key of Object.keys(source)) {
+        if (hasOwn.call(this.output, key)) continue
+        setProperty(this.output, key, source[key])
+        this.merged.add(key)
+      }
+    }
+  }
+}
+
 const closingQuote = (input: string, quote: string, start: number): number => {
   for (let index = start; index < input.length; index++) {
     const character = input[index]
@@ -293,28 +322,31 @@ class FlowParser {
 
   private parseMapping(): YamlRecord {
     this.expect("{")
-    const output: YamlRecord = {}
+    const mapping = new MappingBuilder()
+    const fail = (message: string) => this.fail(message)
     this.skipWhitespace()
     if (this.peek() === "}") {
       this.index++
-      return output
+      return mapping.output
     }
     while (true) {
+      this.skipWhitespace()
+      const quoted = this.peek() === "\"" || this.peek() === "'"
       const key = this.parseKey()
       this.skipWhitespace()
       this.expect(":")
-      if (hasOwn.call(output, key)) this.fail(`Duplicate key '${key}'`)
-      setProperty(output, key, this.parseValue())
+      if (!quoted && key === "<<") mapping.merge(this.parseValue(), fail)
+      else mapping.set(key, this.parseValue(), fail)
       this.skipWhitespace()
       if (this.peek() === "}") {
         this.index++
-        return output
+        return mapping.output
       }
       this.expect(",")
       this.skipWhitespace()
       if (this.peek() === "}") {
         this.index++
-        return output
+        return mapping.output
       }
     }
   }
@@ -398,31 +430,32 @@ class YamlParser {
     return this.parseNodeValue(line.text, parentIndent)
   }
 
-  private parseMapping(indent: number, output: YamlRecord = {}): YamlRecord {
+  private parseMapping(indent: number, mapping = new MappingBuilder()): YamlRecord {
     while (true) {
       this.skipIgnored()
       const line = this.lines[this.index]
-      if (line === undefined || line.indent < indent) return output
+      if (line === undefined || line.indent < indent) return mapping.output
       if (line.indent > indent) this.fail(line, `Unexpected indentation of ${line.indent} spaces`)
       const separator = mappingSeparator(line.text)
-      if (separator === -1) return output
+      if (separator === -1) return mapping.output
       this.index++
-      this.parseMappingEntry(output, line.text, separator, indent, line)
+      this.parseMappingEntry(mapping, line.text, separator, indent, line)
     }
   }
 
   private parseMappingEntry(
-    output: YamlRecord,
+    mapping: MappingBuilder,
     text: string,
     separator: number,
     indent: number,
     line: Line
   ): void {
-    const key = parseKey(text.slice(0, separator).trim())
+    const rawKey = text.slice(0, separator).trim()
     const rawValue = text.slice(separator + 1).trimStart()
     const value = this.parseNodeValue(rawValue, indent, true)
-    if (hasOwn.call(output, key)) this.fail(line, `Duplicate key '${key}'`)
-    setProperty(output, key, value)
+    const fail = (message: string) => this.fail(line, message)
+    if (rawKey === "<<") mapping.merge(value, fail)
+    else mapping.set(parseKey(rawKey), value, fail)
   }
 
   private parseSequence(indent: number): Array<unknown> {
@@ -440,9 +473,13 @@ class YamlParser {
       if (separator === -1) {
         output.push(this.parseNodeValue(item, indent))
       } else {
-        const mapping: YamlRecord = {}
-        this.parseMappingEntry(mapping, item, separator, indent + 2, line)
-        output.push(this.parseMapping(indent + 2, mapping))
+        // The mapping is indented to its first key, which may sit more than one space after the dash.
+        const keyOffset = line.text.length - item.length
+        if (line.text.slice(1, keyOffset).includes("\t")) this.fail(line, "Tabs cannot be used for YAML indentation")
+        const mappingIndent = indent + keyOffset
+        const mapping = new MappingBuilder()
+        this.parseMappingEntry(mapping, item, separator, mappingIndent, line)
+        output.push(this.parseMapping(mappingIndent, mapping))
       }
     }
   }
@@ -668,6 +705,12 @@ class YamlParser {
  * The core YAML 1.2 scalar schema is used, so booleans, nulls, and numbers are
  * decoded while date-like values remain strings.
  *
+ * YAML 1.1 merge keys are always enabled. An unquoted `<<` key merges a
+ * mapping, or a sequence of mappings, into the enclosing mapping. Explicit
+ * keys take precedence over merged keys, earlier merge sources over later
+ * ones, and a mapping may contain only one merge key. Quote the key (`"<<"`)
+ * to keep it as an ordinary key.
+ *
  * @stability unstable
  * @category decoding
  * @since 4.0.0

package/src/encoding/index.ts

@@ -17,6 +17,12 @@ export * as Base64 from "./Base64.ts"
  */
 export * as Base64Url from "./Base64Url.ts"
 
+/**
+ * @stability unstable
+ * @since 4.1.0
+ */
+export * as Csv from "./Csv.ts"
+
 /**
  * @stability stable
  * @since 4.0.0

package/src/eventlog/EventJournal.ts

@@ -16,6 +16,7 @@ import * as Arr from "../Array.ts"
 import type { Brand } from "../Brand.ts"
 import * as Clock from "../Clock.ts"
 import * as Context from "../Context.ts"
+import * as SecureRandom from "../crypto/SecureRandom.ts"
 import * as Data from "../Data.ts"
 import * as DateTime from "../DateTime.ts"
 import * as Effect from "../Effect.ts"
@@ -250,7 +251,7 @@ export const RemoteId = Schema.Uint8Array.pipe(Schema.brand(RemoteIdTypeId))
  * @category unsafe
  * @since 4.0.0
  */
-export const makeRemoteIdUnsafe = (): RemoteId => Uuid.v4Bytes() as RemoteId
+export const makeRemoteIdUnsafe = (): RemoteId => SecureRandom.uuidV4Unsafe({ format: "bytes" }) as RemoteId
 
 /**
  * Runtime brand identifier used for `EntryId` values.
@@ -321,7 +322,7 @@ const nextEntryId = (sequence: EntryIdSequence, now: number): EntryId => {
   if (sequence.millis > maxEntryIdMillis) {
     throw new RangeError("The entry id sequence has passed the last UUID v7 millisecond")
   }
-  const bytes = Uuid.v7Bytes(sequence.millis)
+  const bytes = Uuid.v7Bytes(sequence.millis, SecureRandom.randomBytesUnsafe(16))
   // The counter replaces the 12 random `rand_a` bits after the version.
   bytes[6] = 0x70 | (sequence.counter >> 8)
   bytes[7] = sequence.counter & 0xff
@@ -387,7 +388,7 @@ const maxEntryId = (ids: Iterable<EntryId>): EntryId | undefined => {
  */
 export const makeEntryIdUnsafe = (options: { msecs?: number } = {}): EntryId =>
   options.msecs !== undefined
-    ? Uuid.v7Bytes(options.msecs) as EntryId
+    ? Uuid.v7Bytes(options.msecs, SecureRandom.randomBytesUnsafe(16)) as EntryId
     : nextEntryId(processEntryIds, Date.now())
 
 // Imported ids further ahead of the local clock than this are logged.