@thomasfosterau/effect-parse 0.1.0 → 0.3.0

package/src/Token.ts

@@ -20,12 +20,14 @@
  *
  * @since 0.1.0
  */
+import * as Predicate from "effect/Predicate"
 import type { Input, Slice, State, Token } from "./Input.js"
 import {
   advance,
+  codePointPredicate,
   compileAny,
   countTokens,
-  describeBytes,
+  describeLiteral,
   End,
   indexOf,
   matchLiteral,
@@ -67,9 +69,65 @@ export interface TokenRange<T> {
  */
 export const range = <T extends string | number>(from: T, to: T): TokenRange<T> => ({ _tag: "TokenRange", from, to })
 
+declare const CodePointsBrand: unique symbol
+
+/**
+ * A predicate over code points, made with {@link codePoints}. In a token set
+ * it receives each character as a number, so testing a character allocates
+ * nothing.
+ *
+ * @since 0.2.0
+ * @category models
+ */
+export interface CodePoints {
+  (codePoint: number): boolean
+  readonly [CodePointsBrand]: true
+}
+
+/**
+ * Declares a character class as a predicate over code points, for token sets
+ * on string, `Text` and byte inputs. `name` describes it in error messages.
+ *
+ * A plain `(char: string) => boolean` predicate receives each character as a
+ * string, which costs an allocation per character tested; a `codePoints`
+ * predicate receives the code point itself (a byte, on byte input).
+ *
+ * `ranges` optionally lists the code points the predicate accepts, as
+ * inclusive `[from, to]` pairs. Compiled parsers then test the ranges
+ * instead of calling the predicate, so they must agree with it exactly.
+ *
+ * @example
+ * ```ts
+ * import { Parser, Token } from "@thomasfosterau/effect-parse"
+ *
+ * const isNameStart = Token.codePoints(
+ *   "name start",
+ *   (c) => (c >= 0x61 && c <= 0x7a) || c === 0x5f || c >= 0xc0,
+ *   [[0x5f, 0x5f], [0x61, 0x7a], [0xc0, 0x10ffff]]
+ * )
+ * const name = Token.takeWhile([isNameStart, Token.range("0", "9")], { min: 1 })
+ * ```
+ *
+ * @since 0.2.0
+ * @category constructors
+ */
+export const codePoints = (
+  name: string,
+  predicate: (codePoint: number) => boolean,
+  ranges?: ReadonlyArray<readonly [number, number]>
+): CodePoints =>
+  // Wrapped, so the caller's function is left unmarked
+  codePointPredicate(
+    name,
+    (c: number): boolean => predicate(c),
+    ranges?.map(([from, to]) => [from, to] as const)
+  ) as unknown as CodePoints
+
 /**
  * A set of string tokens: the characters of a string, a range, a predicate,
- * or an array of these.
+ * or an array of these. Strings, ranges, {@link codePoints} predicates and
+ * the `Ascii` and `Text` predicates test characters without allocating; a
+ * `(char: string) => boolean` predicate allocates a string per character.
  *
  * @since 0.1.0
  * @category models
@@ -77,6 +135,7 @@ export const range = <T extends string | number>(from: T, to: T): TokenRange<T>
 export type StringTokenSet =
   | string
   | TokenRange<string>
+  | CodePoints
   | ((char: string) => boolean)
   | ReadonlyArray<StringTokenSet>
 
@@ -92,6 +151,7 @@ export type ByteTokenSet =
   | string
   | Uint8Array
   | TokenRange<number>
+  | CodePoints
   | ((byte: number) => boolean)
   | ReadonlyArray<ByteTokenSet>
 
@@ -102,10 +162,13 @@ export type ByteTokenSet =
  * @since 0.1.0
  * @category models
  */
-export type TokenSet<I> = I extends string ? StringTokenSet
-  : I extends Uint8Array ? ByteTokenSet
-  : I extends ReadonlyArray<infer T> ? ((token: T) => boolean) | ReadonlyArray<T>
-  : never
+export type TokenSet<I> = I extends string
+  ? StringTokenSet
+  : I extends Uint8Array
+    ? ByteTokenSet
+    : I extends ReadonlyArray<infer T>
+      ? ((token: T) => boolean) | ReadonlyArray<T>
+      : never
 
 const expectedAny = ["any token"]
 
@@ -140,7 +203,7 @@ const anyStep = (state: State<Input>): unknown => {
  * @since 0.1.0
  * @category tokens
  */
-export const any: Parser<string> = described(anyStep as any, "any", { _tag: "Any", encoding: undefined })
+export const any: Parser<string> = /*#__PURE__*/ described(anyStep as any, "any", { _tag: "Any", encoding: undefined })
 
 /**
  * Matches any single token of an input of type `I`.
@@ -182,28 +245,28 @@ export const literal: {
 
 /** @internal */
 export const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, never, any> => {
-  const description = typeof lit === "string"
-    ? JSON.stringify(lit)
-    : lit instanceof Uint8Array
-    ? describeBytes(lit)
-    : `[${lit.map((t) => JSON.stringify(t)).join(", ")}]`
+  const description = describeLiteral(lit)
   const expected = [caseless ? `${description} (case insensitive)` : description]
   // A string literal may also be matched against byte input (as UTF-8)
   let asBytes: Uint8Array | undefined
-  const literalKind = typeof lit === "string" ? 0 : lit instanceof Uint8Array ? 1 : 2
-  return described((state: State<Input>) => {
-    const pos = state.pos
-    const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit as string)) : lit
-    const len = l.length
-    const r = matchLiteral(state, l, caseless)
-    if (r === -1) return state.backtrack("Literal", expected)
-    if (r === -2) {
-      if (state.partial) return state.incomplete(len - (state.end - pos), expected)
-      return state.backtrack("Literal", expected)
-    }
-    state.pos = pos + len
-    return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit
-  }, description, { _tag: "Literal", literal: lit, caseless, encoding: undefined })
+  const literalKind = Predicate.isString(lit) ? 0 : lit instanceof Uint8Array ? 1 : 2
+  return described(
+    (state: State<Input>) => {
+      const pos = state.pos
+      const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit as string)) : lit
+      const len = l.length
+      const r = matchLiteral(state, l, caseless)
+      if (r === -1) return state.backtrack("Literal", expected)
+      if (r === -2) {
+        if (state.partial) return state.incomplete(len - (state.end - pos), expected)
+        return state.backtrack("Literal", expected)
+      }
+      state.pos = pos + len
+      return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit
+    },
+    description,
+    { _tag: "Literal", literal: lit, caseless, encoding: undefined }
+  )
 }
 
 /**
@@ -222,15 +285,21 @@ export const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, n
  * @since 0.1.0
  * @category tokens
  */
-export const oneOf = <I extends Input = string>(set: TokenSet<I>): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {
+export const oneOf = <I extends Input = string>(
+  set: TokenSet<I>
+): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {
   const m = compileAny(set)
   const expected = [m.description]
-  return described((state) => {
-    const t = nextIf(state, m, false)
-    if (t === End) return state.eoi("OneOf", expected, 1)
-    if (t === NoMatch) return state.backtrack("OneOf", expected)
-    return t as Token<I>
-  }, m.description, { _tag: "OneOf", set, negate: false, encoding: undefined })
+  return described(
+    (state) => {
+      const t = nextIf(state, m, false)
+      if (t === End) return state.eoi("OneOf", expected, 1)
+      if (t === NoMatch) return state.backtrack("OneOf", expected)
+      return t as Token<I>
+    },
+    m.description,
+    { _tag: "OneOf", set, negate: false, encoding: undefined }
+  )
 }
 
 /**
@@ -240,15 +309,21 @@ export const oneOf = <I extends Input = string>(set: TokenSet<I>): Parser<Token<
  * @since 0.1.0
  * @category tokens
  */
-export const noneOf = <I extends Input = string>(set: TokenSet<I>): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {
+export const noneOf = <I extends Input = string>(
+  set: TokenSet<I>
+): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {
   const m = compileAny(set)
   const expected = [`anything but ${m.description}`]
-  return described((state) => {
-    const t = nextIf(state, m, true)
-    if (t === End) return state.eoi("NoneOf", expected, 1)
-    if (t === NoMatch) return state.backtrack("NoneOf", expected)
-    return t as Token<I>
-  }, expected[0], { _tag: "OneOf", set, negate: true, encoding: undefined })
+  return described(
+    (state) => {
+      const t = nextIf(state, m, true)
+      if (t === End) return state.eoi("NoneOf", expected, 1)
+      if (t === NoMatch) return state.backtrack("NoneOf", expected)
+      return t as Token<I>
+    },
+    expected[0],
+    { _tag: "OneOf", set, negate: true, encoding: undefined }
+  )
 }
 
 /**
@@ -259,18 +334,22 @@ export const noneOf = <I extends Input = string>(set: TokenSet<I>): Parser<Token
  */
 export const take = <I extends Input = string>(count: number): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {
   const expected = [count === 1 ? "1 token" : `${count} tokens`]
-  return described((state) => {
-    const start = state.pos
-    const { count: got, offset } = advance(state, start, count)
-    if (got < count) return state.eoi("Take", expected, count - got)
-    state.pos = offset
-    return state.slice(start, offset)
-  }, "take", { _tag: "Take", count, encoding: undefined })
+  return described(
+    (state) => {
+      const start = state.pos
+      const { count: got, offset } = advance(state, start, count)
+      if (got < count) return state.eoi("Take", expected, count - got)
+      state.pos = offset
+      return state.slice(start, offset)
+    },
+    "take",
+    { _tag: "Take", count, encoding: undefined }
+  )
 }
 
 const normalize = (r: Range | undefined): [number, number] => {
   if (r === undefined) return [0, Infinity]
-  if (typeof r === "number") return [r, r]
+  if (Predicate.isNumber(r)) return [r, r]
   return [r.min ?? 0, r.max ?? Infinity]
 }
 
@@ -287,16 +366,20 @@ export const takeWhileImpl = <I extends Input>(
   const m = compileAny(set)
   const [min, max] = normalize(r)
   const expected = [description ?? (negate ? `anything but ${m.description}` : m.description)]
-  return described((state) => {
-    const start = state.pos
-    const { count, hitEnd, offset } = scan(state, m, negate, max)
-    if (hitEnd && state.partial) return state.incomplete(1, expected)
-    if (count < min) return state.backtrack(kind, expected, offset)
-    // More matching tokens would have been accepted where we stopped
-    if (hint && count < max) state.setHint(offset, expected)
-    state.pos = offset
-    return state.slice(start, offset)
-  }, name, { _tag: "TakeWhile", set, negate, min, max, encoding: undefined })
+  return described(
+    (state) => {
+      const start = state.pos
+      const { count, hitEnd, offset } = scan(state, m, negate, max)
+      if (hitEnd && state.partial) return state.incomplete(1, expected)
+      if (count < min) return state.backtrack(kind, expected, offset)
+      // More matching tokens would have been accepted where we stopped
+      if (hint && count < max) state.setHint(offset, expected)
+      state.pos = offset
+      return state.slice(start, offset)
+    },
+    name,
+    { _tag: "TakeWhile", set, negate, min, max, encoding: undefined }
+  )
 }
 
 /**
@@ -349,28 +432,30 @@ export const takeUntil: {
   <T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>
 } = (lit: Input, r?: Range): Parser<any, never, never, any> => {
   const [min, max] = normalize(r)
-  const description = typeof lit === "string"
-    ? JSON.stringify(lit)
-    : lit instanceof Uint8Array
-    ? describeBytes(lit)
-    : `[${lit.map((t) => JSON.stringify(t)).join(", ")}]`
+  const description = describeLiteral(lit)
   const expected = [description]
-  return described((state: State<Input>) => {
-    const start = state.pos
-    const idx = indexOf(state, lit, start)
-    if (idx < 0) {
-      if (state.partial) return state.incomplete(0, expected)
-      return state.backtrack("TakeUntil", expected, state.end)
-    }
-    // As in winnow, the first occurrence must lie within the range
-    if (min > 0 || max !== Infinity) {
-      const count = countTokens(state, start, idx)
-      if (count < min) return state.backtrack("TakeUntil", expected, idx)
-      if (count > max) return state.backtrack("TakeUntil", expected)
-    }
-    state.pos = idx
-    return state.slice(start, idx)
-  }, "takeUntil", { _tag: "TakeUntil", literal: lit, ...bounds(r), encoding: undefined })
+  return described(
+    (state: State<Input>) => {
+      const start = state.pos
+      const idx = indexOf(state, lit, start)
+      if (idx < 0) {
+        if (state.partial) return state.incomplete(0, expected)
+        return state.backtrack("TakeUntil", expected, state.end)
+      }
+      // The literal was read but is not consumed (a string literal is up to three bytes per unit of UTF-8)
+      state.syntax?.read(idx + lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1))
+      // As in winnow, the first occurrence must lie within the range
+      if (min > 0 || max !== Infinity) {
+        const count = countTokens(state, start, idx)
+        if (count < min) return state.backtrack("TakeUntil", expected, idx)
+        if (count > max) return state.backtrack("TakeUntil", expected)
+      }
+      state.pos = idx
+      return state.slice(start, idx)
+    },
+    "takeUntil",
+    { _tag: "TakeUntil", literal: lit, ...bounds(r), encoding: undefined }
+  )
 }
 
 /**
@@ -379,11 +464,15 @@ export const takeUntil: {
  * @since 0.1.0
  * @category slices
  */
-export const rest: Parser<string> = described((state: State<any>) => {
-  const start = state.pos
-  state.pos = state.end
-  return state.slice(start, state.end) as string
-}, "rest", { _tag: "Rest", encoding: undefined })
+export const rest: Parser<string> = /*#__PURE__*/ described(
+  (state: State<any>) => {
+    const start = state.pos
+    state.pos = state.end
+    return state.slice(start, state.end) as string
+  },
+  "rest",
+  { _tag: "Rest", encoding: undefined }
+)
 
 /**
  * Takes the rest of an input of type `I`.
@@ -400,9 +489,13 @@ export const restOf = <I extends Input = string>(): Parser<Slice<NoInfer<I>>, ne
  * @since 0.1.0
  * @category slices
  */
-export const restLen: Parser<number, never, never, Input> = described((state) => state.end - state.pos, "restLen", {
-  _tag: "RestLen"
-})
+export const restLen: Parser<number, never, never, Input> = /*#__PURE__*/ described(
+  (state) => state.end - state.pos,
+  "restLen",
+  {
+    _tag: "RestLen"
+  }
+)
 
 /**
  * Matches a token satisfying a predicate (nom's `satisfy`). Equivalent to

package/src/index.ts

@@ -3,6 +3,7 @@
  */
 export * as Input from "./Input.js"
 export * as Parser from "./Parser.js"
+export * as Compiler from "./Compiler.js"
 export * as ParseError from "./ParseError.js"
 export * as Token from "./Token.js"
 export * as Ascii from "./Ascii.js"
@@ -14,3 +15,4 @@ export * as Expression from "./Expression.js"
 export * as Nom from "./Nom.js"
 export * as ParserSchema from "./ParserSchema.js"
 export * as ParserStream from "./ParserStream.js"
+export * as Syntax from "./Syntax.js"