@thomasfosterau/effect-parse 0.0.0-stage → 0.1.0

package/src/Binary.ts

@@ -0,0 +1,228 @@
+/**
+ * Parsers for binary formats: fixed-width integers and floats in either
+ * byte order, and length-prefixed data (winnow's `binary` module, nom's
+ * `number` and `multi::length_*` parsers).
+ *
+ * Integers up to 32 bits are returned as `number`, wider integers as
+ * `bigint`.
+ *
+ * @example
+ * ```ts
+ * import { Binary, Parser } from "@thomasfosterau/effect-parse"
+ *
+ * // A length-prefixed (u16 big-endian) payload
+ * const frame = Binary.lengthTake(Binary.beU16)
+ * Parser.parse(frame, new Uint8Array([0, 2, 0xca, 0xfe])) // Uint8Array [0xca, 0xfe]
+ * ```
+ *
+ * @since 0.1.0
+ */
+import type { Input, Slice, State } from "./Input.js"
+import { Fail } from "./Input.js"
+import type { Parser } from "./Parser.js"
+import * as P from "./Parser.js"
+import * as Token from "./Token.js"
+
+export {
+  /**
+   * Runs a bit-level parser on byte input. See the `Bits` module.
+   *
+   * @since 0.1.0
+   * @category bits
+   */
+  bits
+} from "./Bits.js"
+
+/**
+ * A byte order. `"native"` is the byte order of the current platform.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type Endianness = "big" | "little" | "native"
+
+const nativeLittle = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1
+
+const fixed = <A>(size: number, name: string, read: (view: DataView, offset: number) => A): Parser<A, never, never, Uint8Array> => {
+  const expected = [`${size} byte${size === 1 ? "" : "s"}`]
+  return P.make((state: State<Uint8Array>) => {
+    const pos = state.pos
+    const avail = state.end - pos
+    if (avail < size) return state.eoi("Token", expected, size - avail)
+    const bytes = state.input
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+    state.pos = pos + size
+    return read(view, pos)
+  }, name)
+}
+
+const readU24 = (little: boolean) => (v: DataView, o: number) =>
+  little
+    ? v.getUint8(o) | (v.getUint8(o + 1) << 8) | (v.getUint8(o + 2) << 16)
+    : (v.getUint8(o) << 16) | (v.getUint8(o + 1) << 8) | v.getUint8(o + 2)
+
+const readI24 = (little: boolean) => {
+  const u = readU24(little)
+  return (v: DataView, o: number) => {
+    const x = u(v, o)
+    return x & 0x800000 ? x - 0x1000000 : x
+  }
+}
+
+const readU128 = (little: boolean) => (v: DataView, o: number) => {
+  const a = v.getBigUint64(o, little)
+  const b = v.getBigUint64(o + 8, little)
+  return little ? (b << 64n) | a : (a << 64n) | b
+}
+
+const readI128 = (little: boolean) => {
+  const u = readU128(little)
+  return (v: DataView, o: number) => BigInt.asIntN(128, u(v, o))
+}
+
+/**
+ * An unsigned byte.
+ *
+ * @since 0.1.0
+ * @category integers
+ */
+export const u8: Parser<number, never, never, Uint8Array> = fixed(1, "u8", (v, o) => v.getUint8(o))
+
+/**
+ * A signed byte.
+ *
+ * @since 0.1.0
+ * @category integers
+ */
+export const i8: Parser<number, never, never, Uint8Array> = fixed(1, "i8", (v, o) => v.getInt8(o))
+
+/** @since 0.1.0 @category integers */
+export const beU16: Parser<number, never, never, Uint8Array> = fixed(2, "beU16", (v, o) => v.getUint16(o, false))
+/** @since 0.1.0 @category integers */
+export const leU16: Parser<number, never, never, Uint8Array> = fixed(2, "leU16", (v, o) => v.getUint16(o, true))
+/** @since 0.1.0 @category integers */
+export const beI16: Parser<number, never, never, Uint8Array> = fixed(2, "beI16", (v, o) => v.getInt16(o, false))
+/** @since 0.1.0 @category integers */
+export const leI16: Parser<number, never, never, Uint8Array> = fixed(2, "leI16", (v, o) => v.getInt16(o, true))
+/** @since 0.1.0 @category integers */
+export const beU24: Parser<number, never, never, Uint8Array> = fixed(3, "beU24", readU24(false))
+/** @since 0.1.0 @category integers */
+export const leU24: Parser<number, never, never, Uint8Array> = fixed(3, "leU24", readU24(true))
+/** @since 0.1.0 @category integers */
+export const beI24: Parser<number, never, never, Uint8Array> = fixed(3, "beI24", readI24(false))
+/** @since 0.1.0 @category integers */
+export const leI24: Parser<number, never, never, Uint8Array> = fixed(3, "leI24", readI24(true))
+/** @since 0.1.0 @category integers */
+export const beU32: Parser<number, never, never, Uint8Array> = fixed(4, "beU32", (v, o) => v.getUint32(o, false))
+/** @since 0.1.0 @category integers */
+export const leU32: Parser<number, never, never, Uint8Array> = fixed(4, "leU32", (v, o) => v.getUint32(o, true))
+/** @since 0.1.0 @category integers */
+export const beI32: Parser<number, never, never, Uint8Array> = fixed(4, "beI32", (v, o) => v.getInt32(o, false))
+/** @since 0.1.0 @category integers */
+export const leI32: Parser<number, never, never, Uint8Array> = fixed(4, "leI32", (v, o) => v.getInt32(o, true))
+/** @since 0.1.0 @category integers */
+export const beU64: Parser<bigint, never, never, Uint8Array> = fixed(8, "beU64", (v, o) => v.getBigUint64(o, false))
+/** @since 0.1.0 @category integers */
+export const leU64: Parser<bigint, never, never, Uint8Array> = fixed(8, "leU64", (v, o) => v.getBigUint64(o, true))
+/** @since 0.1.0 @category integers */
+export const beI64: Parser<bigint, never, never, Uint8Array> = fixed(8, "beI64", (v, o) => v.getBigInt64(o, false))
+/** @since 0.1.0 @category integers */
+export const leI64: Parser<bigint, never, never, Uint8Array> = fixed(8, "leI64", (v, o) => v.getBigInt64(o, true))
+/** @since 0.1.0 @category integers */
+export const beU128: Parser<bigint, never, never, Uint8Array> = fixed(16, "beU128", readU128(false))
+/** @since 0.1.0 @category integers */
+export const leU128: Parser<bigint, never, never, Uint8Array> = fixed(16, "leU128", readU128(true))
+/** @since 0.1.0 @category integers */
+export const beI128: Parser<bigint, never, never, Uint8Array> = fixed(16, "beI128", readI128(false))
+/** @since 0.1.0 @category integers */
+export const leI128: Parser<bigint, never, never, Uint8Array> = fixed(16, "leI128", readI128(true))
+
+/** @since 0.1.0 @category floats */
+export const beF32: Parser<number, never, never, Uint8Array> = fixed(4, "beF32", (v, o) => v.getFloat32(o, false))
+/** @since 0.1.0 @category floats */
+export const leF32: Parser<number, never, never, Uint8Array> = fixed(4, "leF32", (v, o) => v.getFloat32(o, true))
+/** @since 0.1.0 @category floats */
+export const beF64: Parser<number, never, never, Uint8Array> = fixed(8, "beF64", (v, o) => v.getFloat64(o, false))
+/** @since 0.1.0 @category floats */
+export const leF64: Parser<number, never, never, Uint8Array> = fixed(8, "leF64", (v, o) => v.getFloat64(o, true))
+
+const byEndian = <A>(big: A, little: A) => (endian: Endianness): A =>
+  endian === "big" ? big : endian === "little" ? little : nativeLittle ? little : big
+
+/**
+ * An unsigned 16-bit integer in the given byte order (winnow's `u16`).
+ *
+ * @since 0.1.0
+ * @category integers
+ */
+export const u16: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beU16, leU16)
+/** @since 0.1.0 @category integers */
+export const i16: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beI16, leI16)
+/** @since 0.1.0 @category integers */
+export const u24: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beU24, leU24)
+/** @since 0.1.0 @category integers */
+export const i24: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beI24, leI24)
+/** @since 0.1.0 @category integers */
+export const u32: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beU32, leU32)
+/** @since 0.1.0 @category integers */
+export const i32: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beI32, leI32)
+/** @since 0.1.0 @category integers */
+export const u64: (endian: Endianness) => Parser<bigint, never, never, Uint8Array> = byEndian(beU64, leU64)
+/** @since 0.1.0 @category integers */
+export const i64: (endian: Endianness) => Parser<bigint, never, never, Uint8Array> = byEndian(beI64, leI64)
+/** @since 0.1.0 @category integers */
+export const u128: (endian: Endianness) => Parser<bigint, never, never, Uint8Array> = byEndian(beU128, leU128)
+/** @since 0.1.0 @category integers */
+export const i128: (endian: Endianness) => Parser<bigint, never, never, Uint8Array> = byEndian(beI128, leI128)
+/** @since 0.1.0 @category floats */
+export const f32: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beF32, leF32)
+/** @since 0.1.0 @category floats */
+export const f64: (endian: Endianness) => Parser<number, never, never, Uint8Array> = byEndian(beF64, leF64)
+
+
+const toCount = (n: number | bigint): number => typeof n === "bigint" ? Number(n) : n
+
+/**
+ * Parses a length with `count`, then takes that many tokens (winnow's
+ * `length_take`, nom's `length_data`).
+ *
+ * @since 0.1.0
+ * @category length-prefixed
+ */
+export const lengthTake = <E, R, I extends Input = Uint8Array>(
+  count: Parser<number | bigint, E, R, I>
+): Parser<Slice<I>, E, R, I> => {
+  const countStep = count.step
+  return P.make((state: State<I>) => {
+    const n = countStep(state)
+    if (n === Fail) return Fail
+    return (Token.take<I>(toCount(n)).step as any)(state)
+  }, "lengthTake")
+}
+
+/**
+ * Parses a length with `count`, then applies `parser` that many times
+ * (winnow's `length_repeat`, nom's `length_count`).
+ *
+ * @since 0.1.0
+ * @category length-prefixed
+ */
+export const lengthRepeat = <E, R, I extends Input, A, E2, R2, I2 extends Input>(
+  count: Parser<number | bigint, E, R, I>,
+  parser: Parser<A, E2, R2, I2>
+): Parser<Array<A>, E | E2, R | R2, I & I2> =>
+  P.flatMap(count, (n) => P.repeat(parser, toCount(n)) as Parser<Array<A>, E2, R2, I2>)
+
+/**
+ * Parses a length with `count`, takes that many tokens, and applies
+ * `parser` to them (winnow's `length_and_then`, nom's `length_value`).
+ * `parser` does not need to consume the whole slice.
+ *
+ * @since 0.1.0
+ * @category length-prefixed
+ */
+export const lengthAndThen = <E, R, I extends Input, A, E2, R2>(
+  count: Parser<number | bigint, E, R, I>,
+  parser: Parser<A, E2, R2, Slice<I> & Input>
+): Parser<A, E | E2, R | R2, I> =>
+  P.mapParser(lengthTake(count) as Parser<Slice<I> & Input, E, R, I>, P.complete(parser))

package/src/Bits.ts

@@ -0,0 +1,192 @@
+/**
+ * Bit-level parsing (winnow's `binary::bits`, nom's `bits` module).
+ *
+ * Bit parsers run inside {@link bits}, which switches a byte parser into
+ * bit mode. Bits are read most-significant first. When `bits` finishes, any
+ * remaining bits of a partly consumed byte are skipped.
+ *
+ * Inside `bits`, offsets count bits, so the general combinators (`alt`,
+ * `repeat`, `opt`, ...) backtrack at bit granularity.
+ *
+ * @example
+ * ```ts
+ * import { Binary, Bits, Parser } from "@thomasfosterau/effect-parse"
+ *
+ * // An IPv4 header starts with a 4-bit version and a 4-bit header length
+ * const versionAndLength = Bits.bits(Parser.seq(Bits.take(4), Bits.take(4)))
+ *
+ * Parser.parse(versionAndLength, new Uint8Array([0x45])) // Result.succeed([4, 5])
+ * ```
+ *
+ * @since 0.1.0
+ */
+import type { State } from "./Input.js"
+import { Fail, Failure } from "./Input.js"
+import type { Parser } from "./Parser.js"
+import { make } from "./Parser.js"
+
+declare const BitInputBrand: unique symbol
+
+/**
+ * The input type of bit-level parsers. A `Parser<A, E, R, BitInput>` can
+ * only be run inside {@link bits}.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface BitInput extends ReadonlyArray<never> {
+  readonly [BitInputBrand]: true
+}
+
+const scale = (f: Failure, factor: number): Failure =>
+  new Failure(
+    factor < 1 ? Math.floor(f.offset * factor) : f.offset * factor,
+    f.kind,
+    f.expected,
+    f.context,
+    f.hasCause,
+    f.cause,
+    f.message,
+    factor < 1 ? Math.ceil(f.needed * factor) : f.needed * factor
+  )
+
+/**
+ * Runs `step` with offsets measured in different units (bits vs bytes),
+ * converting failure offsets back to the outer units afterwards.
+ */
+const inUnits = <A>(state: State<any>, factor: number, step: (state: State<any>) => A | Fail): A | Fail => {
+  const end = state.end
+  const deepest = state.deepest
+  state.deepest = undefined
+  state.pos = factor > 1 ? state.pos * factor : Math.ceil(state.pos * factor)
+  state.end = end * factor
+  const a = step(state)
+  state.end = end
+  const inner = state.deepest
+  state.deepest = deepest
+  if (a === Fail) {
+    state.pos = factor > 1 ? Math.floor(state.pos / factor) : state.pos / factor
+    const f = state.failure
+    state.failure = scale(f, 1 / factor)
+    if (inner !== undefined) {
+      const d = inner === f ? state.failure : scale(inner, 1 / factor)
+      if (deepest === undefined || d.offset >= deepest.offset) state.deepest = d
+    }
+    return Fail
+  }
+  state.pos = factor > 1 ? Math.ceil(state.pos / factor) : state.pos / factor
+  if (inner !== undefined) {
+    const d = scale(inner, 1 / factor)
+    if (deepest === undefined || d.offset >= deepest.offset) state.deepest = d
+  }
+  return a
+}
+
+/**
+ * Runs a bit-level parser on byte input (winnow's and nom's `bits`). After
+ * success, the input advances to the next byte boundary.
+ *
+ * @since 0.1.0
+ * @category combinators
+ */
+export const bits = <A, E, R>(self: Parser<A, E, R, BitInput>): Parser<A, E, R, Uint8Array> => {
+  const step = self.step as unknown as (state: State<any>) => A | Fail
+  return make((state) => inUnits(state, 8, step), "bits")
+}
+
+/**
+ * Runs a byte-level parser from within a bit-level parser (winnow's and
+ * nom's `bytes`). The partly consumed byte, if any, is skipped first.
+ *
+ * @since 0.1.0
+ * @category combinators
+ */
+export const bytes = <A, E, R>(self: Parser<A, E, R, Uint8Array>): Parser<A, E, R, BitInput> => {
+  const step = self.step as (state: State<any>) => A | Fail
+  return make((state: State<any>) => inUnits(state, 1 / 8, step), "bytes")
+}
+
+const readBits = (state: State<any>, count: number): number => {
+  const input = state.input as Uint8Array
+  let pos = state.pos
+  let value = 0
+  for (let i = 0; i < count; i++) {
+    const byte = input[pos >>> 3]!
+    value = value * 2 + ((byte >>> (7 - (pos & 7))) & 1)
+    pos++
+  }
+  state.pos = pos
+  return value
+}
+
+/**
+ * Takes `count` bits (at most 53) as an unsigned number (winnow's and nom's
+ * bit-level `take`).
+ *
+ * @since 0.1.0
+ * @category parsers
+ */
+export const take = (count: number): Parser<number, never, never, BitInput> => {
+  if (count > 53) throw new RangeError("Bits.take: use Bits.takeBig for more than 53 bits")
+  const expected = [`${count} bits`]
+  return make((state: State<any>) => {
+    const avail = state.end - state.pos
+    if (avail < count) return state.eoi("Bits", expected, count - avail)
+    return readBits(state, count)
+  }, "take")
+}
+
+/**
+ * Takes `count` bits as an unsigned `bigint`.
+ *
+ * @since 0.1.0
+ * @category parsers
+ */
+export const takeBig = (count: number): Parser<bigint, never, never, BitInput> => {
+  const expected = [`${count} bits`]
+  return make((state: State<any>) => {
+    const avail = state.end - state.pos
+    if (avail < count) return state.eoi("Bits", expected, count - avail)
+    let value = 0n
+    let remaining = count
+    while (remaining > 0) {
+      const n = Math.min(remaining, 32)
+      value = (value << BigInt(n)) | BigInt(readBits(state, n))
+      remaining -= n
+    }
+    return value
+  }, "takeBig")
+}
+
+/**
+ * Matches `count` bits equal to `pattern` (winnow's `pattern`, nom's bit
+ * `tag`).
+ *
+ * @since 0.1.0
+ * @category parsers
+ */
+export const pattern = (pattern: number, count: number): Parser<number, never, never, BitInput> => {
+  const expected = [`bit pattern ${pattern.toString(2).padStart(count, "0")}`]
+  return make((state: State<any>) => {
+    const start = state.pos
+    const avail = state.end - start
+    if (avail < count) return state.eoi("Bits", expected, count - avail)
+    const value = readBits(state, count)
+    if (value !== pattern) {
+      state.pos = start
+      return state.backtrack("Bits", expected)
+    }
+    return value
+  }, "pattern")
+}
+
+/**
+ * Takes one bit as a boolean (winnow's and nom's `bool`).
+ *
+ * @since 0.1.0
+ * @category parsers
+ */
+export const bool: Parser<boolean, never, never, BitInput> = make((state: State<any>) => {
+  if (state.pos >= state.end) return state.eoi("Bits", ["1 bit"], 1)
+  return readBits(state, 1) === 1
+}, "bool")

package/src/Encoding.ts

@@ -0,0 +1,128 @@
+/**
+ * Text encodings for the `Text` module.
+ *
+ * An {@link Encoding} tells the `Text` parsers how to read Unicode code
+ * points out of an input. JavaScript strings are read natively, without any
+ * conversion. Byte inputs are decoded as they are parsed, so offsets stay
+ * byte offsets, slices that are kept as bytes (`Parser.taken`) are
+ * zero-copy views, and partial input that ends in the middle of a character
+ * reports `Incomplete`.
+ *
+ * ```ts
+ * import { Encoding, Text } from "@thomasfosterau/effect-parse"
+ *
+ * Text.utf8.encoding === Encoding.utf8
+ * Text.make(Encoding.latin1) // the Text parsers for any Encoding, including your own
+ * ```
+ *
+ * @since 0.1.0
+ */
+import * as Option from "effect/Option"
+import * as internal from "./internal/text.js"
+
+/**
+ * A text encoding: how to read Unicode code points out of an input. The
+ * `Text` module is generic over encodings, so one grammar can parse
+ * JavaScript strings, UTF-8 bytes, UTF-16 bytes and so on.
+ *
+ * Decoding is strict: malformed sequences (invalid UTF-8, unpaired UTF-16
+ * surrogates) are reported as invalid, never replaced.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Encoding<I extends string | Uint8Array = string | Uint8Array> {
+  /** The name of the encoding, e.g. `"UTF-8"`. Used in error messages. */
+  readonly name: string
+  /**
+   * The size of a code unit, in input units: 1 for strings, UTF-8 and
+   * Latin-1, 2 for UTF-16 bytes. Code points start at multiples of it.
+   */
+  readonly unit: number
+  /**
+   * Whether every input unit below `0x80` encodes the ASCII character with
+   * that value, and is never part of a longer sequence. Enables fast paths.
+   */
+  readonly asciiCompatible: boolean
+  /**
+   * Decodes the code point starting at `offset` (with `offset < end`).
+   * Returns the code point, `-1` if the input is malformed there, or `-2` if
+   * the input ends inside a sequence (more input could complete it).
+   */
+  readonly decode: (input: I, offset: number, end: number) => number
+  /** The number of input units that encode a code point. */
+  readonly width: (codePoint: number) => number
+  /** Encodes a string. Throws if the string can't be represented. */
+  readonly encode: (text: string) => I
+  /**
+   * Decodes the units `[start, end)` to a string. The parsers only call it
+   * on ranges they have checked are well formed.
+   */
+  readonly text: (input: I, start: number, end: number) => string
+}
+
+/**
+ * JavaScript strings, read as UTF-16 code units. Offsets are code-unit
+ * indices and slices are plain `String.prototype.slice`s. Unpaired
+ * surrogates are malformed.
+ *
+ * @since 0.1.0
+ * @category encodings
+ */
+export const string: Encoding<string> = internal.string
+
+/**
+ * UTF-8 bytes. Overlong encodings, surrogates and code points above
+ * U+10FFFF are malformed.
+ *
+ * @since 0.1.0
+ * @category encodings
+ */
+export const utf8: Encoding<Uint8Array> = internal.utf8
+
+/**
+ * UTF-16 bytes, little endian. Unpaired surrogates are malformed.
+ *
+ * @since 0.1.0
+ * @category encodings
+ */
+export const utf16le: Encoding<Uint8Array> = internal.utf16le
+
+/**
+ * UTF-16 bytes, big endian. Unpaired surrogates are malformed.
+ *
+ * @since 0.1.0
+ * @category encodings
+ */
+export const utf16be: Encoding<Uint8Array> = internal.utf16be
+
+/**
+ * ISO-8859-1 (Latin-1) bytes: each byte is the code point U+0000 to U+00FF
+ * of the same value.
+ *
+ * @since 0.1.0
+ * @category encodings
+ */
+export const latin1: Encoding<Uint8Array> = internal.latin1
+
+/**
+ * Detects the encoding of bytes from a byte order mark: UTF-8
+ * (`EF BB BF`), UTF-16LE (`FF FE`) or UTF-16BE (`FE FF`). The mark itself
+ * is not skipped; parse it with `Text.bom`.
+ *
+ * @example
+ * ```ts
+ * import { Encoding } from "@thomasfosterau/effect-parse"
+ *
+ * Encoding.sniff(new Uint8Array([0xff, 0xfe, 0x41, 0x00])) // Option.some(Encoding.utf16le)
+ * ```
+ *
+ * @since 0.1.0
+ * @category utils
+ */
+export const sniff = (bytes: Uint8Array): Option.Option<Encoding<Uint8Array>> => {
+  if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return Option.some(utf8)
+  if (bytes[0] === 0xff && bytes[1] === 0xfe) return Option.some(utf16le)
+  if (bytes[0] === 0xfe && bytes[1] === 0xff) return Option.some(utf16be)
+  return Option.none()
+}