@thomasfosterau/effect-parse 0.4.0 → 0.5.0

package/src/Binary.ts

@@ -44,45 +44,61 @@ export type Endianness = "big" | "little" | "native"
 
 const nativeLittle = /*#__PURE__*/ new Uint8Array(/*#__PURE__*/ new Uint16Array([1]).buffer)[0] === 1
 
-const fixed = <A>(
-  size: number,
-  name: string,
-  read: (view: DataView, offset: number) => A
-): Parser<A, never, never, Uint8Array> => {
+/** Reads a number at an offset of a state's bytes. */
+type Read<A> = (bytes: Uint8Array, offset: number, state: State<Uint8Array>) => A
+
+/**
+ * A `DataView` of a state's bytes, for the numbers integer arithmetic can't
+ * read: created once per state (whose input never changes), as one per
+ * read cost most of a number's time.
+ */
+const viewOf = (state: State<Uint8Array>): DataView => {
+  const b = state.input
+  return (state.view ??= new DataView(b.buffer, b.byteOffset, b.byteLength))
+}
+
+/** A {@link Read} through the state's view. */
+const viewed =
+  <A>(read: (view: DataView, offset: number) => A): Read<A> =>
+  (_, o, s) =>
+    read(viewOf(s), o)
+
+const fixed = <A>(size: number, name: string, read: Read<A>): Parser<A, never, never, Uint8Array> => {
   const expected = [`${size} byte${size === 1 ? "" : "s"}`]
-  return P.make((state: State<Uint8Array>) => {
-    const pos = state.pos
-    const avail = state.end - pos
-    if (avail < size) return state.eoi("Token", expected, size - avail)
-    const bytes = state.input
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
-    state.pos = pos + size
-    return read(view, pos)
-  }, name)
+  return P.described(
+    (state: State<Uint8Array>) => {
+      const pos = state.pos
+      const avail = state.end - pos
+      if (avail < size) return state.eoi("Token", expected, size - avail)
+      state.pos = pos + size
+      return read(state.input, pos, state)
+    },
+    name,
+    { _tag: "Fixed", size, read }
+  )
 }
 
-const readU24 = (little: boolean) => (v: DataView, o: number) =>
-  little
-    ? v.getUint8(o) | (v.getUint8(o + 1) << 8) | (v.getUint8(o + 2) << 16)
-    : (v.getUint8(o) << 16) | (v.getUint8(o + 1) << 8) | v.getUint8(o + 2)
+// Integers of up to 32 bits are read from the bytes; the rest through a view
+const readU24 =
+  (little: boolean): Read<number> =>
+  (b, o) =>
+    little ? b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) : (b[o] << 16) | (b[o + 1] << 8) | b[o + 2]
 
-const readI24 = (little: boolean) => {
+const readI24 = (little: boolean): Read<number> => {
   const u = readU24(little)
-  return (v: DataView, o: number) => {
-    const x = u(v, o)
-    return x & 0x800000 ? x - 0x1000000 : x
-  }
+  return (b, o, s) => (u(b, o, s) << 8) >> 8
 }
 
-const readU128 = (little: boolean) => (v: DataView, o: number) => {
-  const a = v.getBigUint64(o, little)
-  const b = v.getBigUint64(o + 8, little)
-  return little ? (b << 64n) | a : (a << 64n) | b
-}
+const readU128 = (little: boolean): Read<bigint> =>
+  viewed((v, o) => {
+    const a = v.getBigUint64(o, little)
+    const b = v.getBigUint64(o + 8, little)
+    return little ? (b << 64n) | a : (a << 64n) | b
+  })
 
-const readI128 = (little: boolean) => {
+const readI128 = (little: boolean): Read<bigint> => {
   const u = readU128(little)
-  return (v: DataView, o: number) => BigInt.asIntN(128, u(v, o))
+  return (b, o, s) => BigInt.asIntN(128, u(b, o, s))
 }
 
 /**
@@ -91,7 +107,7 @@ const readI128 = (little: boolean) => {
  * @since 0.1.0
  * @category integers
  */
-export const u8: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(1, "u8", (v, o) => v.getUint8(o))
+export const u8: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(1, "u8", (b, o) => b[o])
 
 /**
  * A signed byte.
@@ -99,23 +115,31 @@ export const u8: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
  * @since 0.1.0
  * @category integers
  */
-export const i8: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(1, "i8", (v, o) => v.getInt8(o))
+export const i8: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(1, "i8", (b, o) => (b[o] << 24) >> 24)
 
 /** @since 0.1.0 @category integers */
-export const beU16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(2, "beU16", (v, o) =>
-  v.getUint16(o, false)
+export const beU16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  2,
+  "beU16",
+  (b, o) => (b[o] << 8) | b[o + 1]
 )
 /** @since 0.1.0 @category integers */
-export const leU16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(2, "leU16", (v, o) =>
-  v.getUint16(o, true)
+export const leU16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  2,
+  "leU16",
+  (b, o) => b[o] | (b[o + 1] << 8)
 )
 /** @since 0.1.0 @category integers */
-export const beI16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(2, "beI16", (v, o) =>
-  v.getInt16(o, false)
+export const beI16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  2,
+  "beI16",
+  (b, o) => (((b[o] << 8) | b[o + 1]) << 16) >> 16
 )
 /** @since 0.1.0 @category integers */
-export const leI16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(2, "leI16", (v, o) =>
-  v.getInt16(o, true)
+export const leI16: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  2,
+  "leI16",
+  (b, o) => ((b[o] | (b[o + 1] << 8)) << 16) >> 16
 )
 /** @since 0.1.0 @category integers */
 export const beU24: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
@@ -142,36 +166,52 @@ export const leI24: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fix
   /*#__PURE__*/ readI24(true)
 )
 /** @since 0.1.0 @category integers */
-export const beU32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "beU32", (v, o) =>
-  v.getUint32(o, false)
+export const beU32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "beU32",
+  (b, o) => ((b[o] << 24) | (b[o + 1] << 16) | (b[o + 2] << 8) | b[o + 3]) >>> 0
 )
 /** @since 0.1.0 @category integers */
-export const leU32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "leU32", (v, o) =>
-  v.getUint32(o, true)
+export const leU32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "leU32",
+  (b, o) => (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0
 )
 /** @since 0.1.0 @category integers */
-export const beI32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "beI32", (v, o) =>
-  v.getInt32(o, false)
+export const beI32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "beI32",
+  (b, o) => (b[o] << 24) | (b[o + 1] << 16) | (b[o + 2] << 8) | b[o + 3]
 )
 /** @since 0.1.0 @category integers */
-export const leI32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "leI32", (v, o) =>
-  v.getInt32(o, true)
+export const leI32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "leI32",
+  (b, o) => b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)
 )
 /** @since 0.1.0 @category integers */
-export const beU64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "beU64", (v, o) =>
-  v.getBigUint64(o, false)
+export const beU64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "beU64",
+  /*#__PURE__*/ viewed((v, o) => v.getBigUint64(o, false))
 )
 /** @since 0.1.0 @category integers */
-export const leU64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "leU64", (v, o) =>
-  v.getBigUint64(o, true)
+export const leU64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "leU64",
+  /*#__PURE__*/ viewed((v, o) => v.getBigUint64(o, true))
 )
 /** @since 0.1.0 @category integers */
-export const beI64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "beI64", (v, o) =>
-  v.getBigInt64(o, false)
+export const beI64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "beI64",
+  /*#__PURE__*/ viewed((v, o) => v.getBigInt64(o, false))
 )
 /** @since 0.1.0 @category integers */
-export const leI64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "leI64", (v, o) =>
-  v.getBigInt64(o, true)
+export const leI64: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "leI64",
+  /*#__PURE__*/ viewed((v, o) => v.getBigInt64(o, true))
 )
 /** @since 0.1.0 @category integers */
 export const beU128: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fixed(
@@ -199,20 +239,28 @@ export const leI128: Parser<bigint, never, never, Uint8Array> = /*#__PURE__*/ fi
 )
 
 /** @since 0.1.0 @category floats */
-export const beF32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "beF32", (v, o) =>
-  v.getFloat32(o, false)
+export const beF32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "beF32",
+  /*#__PURE__*/ viewed((v, o) => v.getFloat32(o, false))
 )
 /** @since 0.1.0 @category floats */
-export const leF32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(4, "leF32", (v, o) =>
-  v.getFloat32(o, true)
+export const leF32: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  4,
+  "leF32",
+  /*#__PURE__*/ viewed((v, o) => v.getFloat32(o, true))
 )
 /** @since 0.1.0 @category floats */
-export const beF64: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "beF64", (v, o) =>
-  v.getFloat64(o, false)
+export const beF64: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "beF64",
+  /*#__PURE__*/ viewed((v, o) => v.getFloat64(o, false))
 )
 /** @since 0.1.0 @category floats */
-export const leF64: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(8, "leF64", (v, o) =>
-  v.getFloat64(o, true)
+export const leF64: Parser<number, never, never, Uint8Array> = /*#__PURE__*/ fixed(
+  8,
+  "leF64",
+  /*#__PURE__*/ viewed((v, o) => v.getFloat64(o, true))
 )
 
 const byEndian =
@@ -288,6 +336,27 @@ export const f64: (endian: Endianness) => Parser<number, never, never, Uint8Arra
 
 const toCount = (n: number | bigint): number => (Predicate.isBigInt(n) ? Number(n) : n)
 
+/** How many counts {@link byCount} keeps the parsers of. */
+const CountCache = 256
+
+/**
+ * `build`, keeping the parsers it built for the first {@link CountCache}
+ * counts seen: length-prefixed data repeats a few lengths, and building a
+ * parser per value cost several times the parse.
+ */
+const byCount = <A>(build: (n: number) => A): ((n: number | bigint) => A) => {
+  const built = new Map<number, A>()
+  return (n) => {
+    const count = toCount(n)
+    let a = built.get(count)
+    if (a === undefined) {
+      a = build(count)
+      if (built.size < CountCache) built.set(count, a)
+    }
+    return a
+  }
+}
+
 /**
  * Parses a length with `count`, then takes that many tokens (winnow's
  * `length_take`, nom's `length_data`).
@@ -298,12 +367,19 @@ const toCount = (n: number | bigint): number => (Predicate.isBigInt(n) ? Number(
 export const lengthTake = <E, R, I extends Input = Uint8Array>(
   count: Parser<number | bigint, E, R, I>
 ): Parser<Slice<I>, E, R, I> => {
-  const countStep = count.step
-  return P.make((state: State<I>) => {
-    const n = countStep(state)
-    if (n === Fail) return Fail
-    return (Token.take<I>(toCount(n)).step as any)(state)
-  }, "lengthTake")
+  const take = byCount((n) => Token.take<I>(n).step)
+  // The count compiles with the grammar: the step is built from its (see `Parser.combinator`)
+  return P.combinator(
+    [count],
+    ([countStep]) =>
+      (state: State<I>) => {
+        // SAFETY: the count parser's step, interpreted or compiled, returns its output (a number or bigint) or `Fail`
+        const n = countStep(state) as number | bigint | Fail
+        if (n === Fail) return Fail
+        return take(n)(state)
+      },
+    "lengthTake"
+  )
 }
 
 /**
@@ -317,7 +393,10 @@ export const lengthRepeat = <E, R, I extends Input, A, E2, R2, I2 extends Input>
   count: Parser<number | bigint, E, R, I>,
   parser: Parser<A, E2, R2, I2>
 ): Parser<Array<A>, E | E2, R | R2, I & I2> =>
-  P.flatMap(count, (n) => P.repeat(parser, toCount(n)) as Parser<Array<A>, E2, R2, I2>)
+  P.flatMap(
+    count,
+    byCount((n) => P.repeat(parser, n))
+  )
 
 /**
  * Parses a length with `count`, takes that many tokens, and applies
@@ -331,4 +410,5 @@ export const lengthAndThen = <E, R, I extends Input, A, E2, R2>(
   count: Parser<number | bigint, E, R, I>,
   parser: Parser<A, E2, R2, Slice<I> & Input>
 ): Parser<A, E | E2, R | R2, I> =>
+  // SAFETY: a slice of an input is itself an input (a string, a `Uint8Array` or an array), so `Slice<I>` is an `Input`
   P.mapParser(lengthTake(count) as Parser<Slice<I> & Input, E, R, I>, P.complete(parser))

package/src/Bits.ts

@@ -93,7 +93,7 @@ const inUnits = <A>(state: State<any>, factor: number, step: (state: State<any>)
  * @category combinators
  */
 export const bits = <A, E, R>(self: Parser<A, E, R, BitInput>): Parser<A, E, R, Uint8Array> => {
-  const step = self.step as unknown as (state: State<any>) => A | Fail
+  const step = self.step
   return make((state) => inUnits(state, 8, step), "bits")
 }
 
@@ -105,18 +105,24 @@ export const bits = <A, E, R>(self: Parser<A, E, R, BitInput>): Parser<A, E, R,
  * @category combinators
  */
 export const bytes = <A, E, R>(self: Parser<A, E, R, Uint8Array>): Parser<A, E, R, BitInput> => {
-  const step = self.step as (state: State<any>) => A | Fail
+  const step = self.step
   return make((state: State<any>) => inUnits(state, 1 / 8, step), "bytes")
 }
 
 const readBits = (state: State<any>, count: number): number => {
+  // SAFETY: bit-level parsers run inside `bits`, on the `Uint8Array` it was given (`BitInput` only re-indexes it in bits)
   const input = state.input as Uint8Array
   let pos = state.pos
   let value = 0
-  for (let i = 0; i < count; i++) {
-    const byte = input[pos >>> 3]
-    value = value * 2 + ((byte >>> (7 - (pos & 7))) & 1)
-    pos++
+  let left = count
+  // As many bits as the current byte holds at once, not one at a time
+  while (left > 0) {
+    const avail = 8 - (pos & 7)
+    const n = left < avail ? left : avail
+    // Multiplying, not shifting: the value can have up to 53 bits
+    value = value * (1 << n) + ((input[pos >>> 3] >>> (avail - n)) & ((1 << n) - 1))
+    pos += n
+    left -= n
   }
   state.pos = pos
   return value

package/src/Compiler.ts

@@ -4,11 +4,15 @@
  * `Parser.parseSync`, `Parser.parsePeek`, `Parser.parsePartial` and
  * `Parser.parseEffect` faster, from any `offset`.
  *
- * There are two compilers, one function each, so a program bundles only the
- * one it uses, and a program that compiles nothing bundles neither:
+ * There are three compilers, one function each, so a program bundles only
+ * the ones it uses, and a program that compiles nothing bundles none:
  *
+ * - {@link wasm} scans the input in WebAssembly and builds the value with
+ *   generated JavaScript. It is the fastest on strings and bytes, and uses
+ *   the JIT for array input and streaming, and where WebAssembly or
+ *   `new Function` is unavailable.
  * - {@link jit} generates code with `new Function` where it is allowed, and
- *   otherwise uses the closure compiler. It is the fastest.
+ *   otherwise uses the closure compiler.
  * - {@link closure} builds specialized closures only. It never generates
  *   code, and a bundle that uses only it contains no `new Function`, so it
  *   is the choice under a Content Security Policy, where even probing for
@@ -32,7 +36,9 @@
  * **Results and errors are unchanged.** Compiled code records no
  * diagnostics. When it fails, the parser runs again, interpreted, from the
  * same offset, so every `ParseError` (offset, expected, context, cause and
- * message) is exactly the one the interpreter reports.
+ * message) is exactly the one the interpreter reports. Running out of
+ * partial input is not a failure that needs it: compiled code reports
+ * `Incomplete` where the interpreter does, with the same `needed`.
  *
  * **Callbacks should be pure.** On an input that fails, the callbacks given
  * to `map`, `filter`, `tryMap`, `tap`, `fold`, ... may run twice, once
@@ -41,19 +47,57 @@
  * nor those of a branch that is certain to fail, and token-set predicates
  * may be called once per character code and cached.
  *
- * `Parser.parsePartial` and `ParserStream.parseAll` run a build that
- * handles partial input. `ParserStream.parseMany`, `Parser.parseRecoverable`
- * and runs with the `trace` option are always interpreted. Parsers that
+ * `Parser.parsePartial`, `ParserStream.parseAll` and
+ * `ParserStream.parseMany` run a build that handles partial input;
+ * `parseMany` runs it once per value and parses a value again, interpreted,
+ * only when it fails. `Parser.parseRecoverable`, `Syntax`
+ * parses and runs with the `trace` option are always interpreted. Parsers that
  * can't be described, such as `Parser.make` and `Parser.gen`, run
- * interpreted inside compiled code.
+ * interpreted inside compiled code; the parsers given to a
+ * `Parser.combinator` compile, and its step is built from theirs.
  *
  * @since 0.2.0
  */
-import type { Input } from "./Input.js"
-import type { Parser } from "./Parser.js"
+import type { Fail, Input, State } from "./Input.js"
+import type { Parser, RunOptions } from "./Parser.js"
 import { backend as closureBackend } from "./internal/compiler/closure.js"
 import { backend as jitBackend } from "./internal/compiler/jit.js"
 import { register } from "./internal/compiler/registry.js"
+import { backend as wasmBackend } from "./internal/compiler/wasm.js"
+import { prepare } from "./internal/run.js"
+
+/**
+ * Compiles a parser with the WebAssembly compiler. Returns the same parser.
+ *
+ * The input is copied into WebAssembly memory and scanned there, by code
+ * generated from the grammar; a JavaScript function generated with
+ * `new Function` then builds the value from what the scan recorded, with
+ * the same callbacks and the same objects as the JIT. Parts of the grammar
+ * with no WebAssembly form (such as `Parser.make`, `flatMap`, or `filter`
+ * on a value) run their JIT-compiled code from inside the scan.
+ *
+ * It uses the JIT (see {@link jit}) for array input, for the streaming build
+ * of `Parser.parsePartial` and `ParserStream`, and where `WebAssembly` or
+ * `new Function` is unavailable: a Content Security Policy needs both
+ * `'wasm-unsafe-eval'` and `'unsafe-eval'` for it. Short strings with the
+ * same characters may be returned as one string, which no program can tell
+ * apart from two.
+ *
+ * @example
+ * ```ts
+ * import { Ascii, Compiler, Parser, Token } from "@thomasfosterau/effect-parse"
+ *
+ * const list = Compiler.wasm(Parser.separated(Ascii.decInt, Token.literal(",")))
+ * Parser.parse(list, "1,2,3") // Result.succeed([1, 2, 3])
+ * ```
+ *
+ * @since 0.5.0
+ * @category compilation
+ */
+export const wasm = <A, E, R, I extends Input>(self: Parser<A, E, R, I>): Parser<A, E, R, I> => {
+  register(self, wasmBackend)
+  return self
+}
 
 /**
  * Compiles a parser with the JIT: generated code where `new Function` is
@@ -96,3 +140,50 @@ export const closure = <A, E, R, I extends Input>(self: Parser<A, E, R, I>): Par
   register(self, closureBackend)
   return self
 }
+
+/**
+ * Prepares a parser for running many times on states you own, as
+ * `Parser.run` does: for drivers that make one small run per token, such as
+ * a lexer over a buffer. `Parser.run` looks the parser's compiled form up
+ * on every run, which costs about a quarter of a small token's time; the
+ * function returned here looks it up once (again only when the input kind
+ * changes, or between complete and partial states) and calls the compiled
+ * code directly.
+ *
+ * It returns exactly what `Parser.run(self, state, options)` returns, and
+ * leaves the state as it would: the output with the cursor where the
+ * parser stopped, or `Fail` with the interpreter's failure (or, with
+ * `diagnostics: false`, only the mode). Compile the parser first with
+ * {@link jit} or {@link closure}; a parser that isn't compiled runs
+ * interpreted. The compiled form is looked up on the first run, so compile
+ * the parser before using the runner: compiling it again later doesn't
+ * reach a runner that has already run.
+ *
+ * @example
+ * ```ts
+ * import { Ascii, Compiler, Input, Parser, Token } from "@thomasfosterau/effect-parse"
+ *
+ * const token = Compiler.jit(Parser.preceded(Ascii.space0, Parser.alt(Ascii.alpha1, Token.oneOf("+-"))))
+ * const next = Compiler.runner(token)
+ *
+ * const tokens = (text: string): Array<string> => {
+ *   const state = new Input.State(text)
+ *   const out: Array<string> = []
+ *   while (state.pos < state.end) {
+ *     const t = next(state.reset(state.pos))
+ *     if (t === Input.Fail) break
+ *     out.push(t)
+ *   }
+ *   return out
+ * }
+ *
+ * tokens("a + b") // ["a", "+", "b"]
+ * ```
+ *
+ * @since 0.5.0
+ * @category running
+ */
+export const runner = <A, E, R, I extends Input>(
+  self: Parser<A, E, R, I>,
+  options?: RunOptions
+): ((state: State<I>) => A | Fail) => prepare(self, self.step, options?.diagnostics !== false)