@thomasfosterau/effect-parse 0.4.0 → 0.5.0

package/src/Token.ts

@@ -30,6 +30,7 @@ import {
   describeLiteral,
   End,
   indexOf,
+  indexOfAny,
   matchLiteral,
   nextIf,
   NoMatch,
@@ -116,12 +117,13 @@ export const codePoints = (
   predicate: (codePoint: number) => boolean,
   ranges?: ReadonlyArray<readonly [number, number]>
 ): CodePoints =>
-  // Wrapped, so the caller's function is left unmarked
+  // Wrapped, so the caller's function is left unmarked.
+  // SAFETY: `CodePointsBrand` is a declared, phantom key, so the marked function is all a `CodePoints` is at runtime
   codePointPredicate(
     name,
     (c: number): boolean => predicate(c),
     ranges?.map(([from, to]) => [from, to] as const)
-  ) as unknown as CodePoints
+  ) as CodePoints
 
 /**
  * A set of string tokens: the characters of a string, a range, a predicate,
@@ -177,6 +179,7 @@ const anyStep = (state: State<Input>): unknown => {
   if (pos >= state.end) return state.eoi("Token", expectedAny, 1)
   switch (state.inputKind) {
     case 0: {
+      // SAFETY: `inputKind` 0 is set only for a string input
       const s = state.input as string
       if (splitSurrogate(state, pos)) return state.incomplete(1, expectedAny)
       const c = s.charCodeAt(pos)
@@ -189,13 +192,21 @@ const anyStep = (state: State<Input>): unknown => {
     }
     case 1:
       state.pos = pos + 1
+      // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input
       return (state.input as Uint8Array)[pos]
     default:
       state.pos = pos + 1
+      // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array
       return (state.input as ReadonlyArray<unknown>)[pos]
   }
 }
 
+/** The parser behind `any` and `anyToken`, on every input kind. */
+const anyInput: Parser<unknown, never, never, Input> = /*#__PURE__*/ described(anyStep, "any", {
+  _tag: "Any",
+  encoding: undefined
+})
+
 /**
  * Matches any single token: a character of a string (winnow's `any`, nom's
  * `anychar`).
@@ -203,7 +214,9 @@ const anyStep = (state: State<Input>): unknown => {
  * @since 0.1.0
  * @category tokens
  */
-export const any: Parser<string> = /*#__PURE__*/ described(anyStep as any, "any", { _tag: "Any", encoding: undefined })
+export const any: Parser<string> =
+  // SAFETY: on string input, `anyStep` returns the character (one or two code units) at the position, a string
+  anyInput as Parser<string>
 
 /**
  * Matches any single token of an input of type `I`.
@@ -219,7 +232,21 @@ export const any: Parser<string> = /*#__PURE__*/ described(anyStep as any, "any"
  * @since 0.1.0
  * @category tokens
  */
-export const anyToken = <I extends Input = string>(): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => any as any
+export const anyToken = <I extends Input = string>(): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> =>
+  // SAFETY: `anyStep` handles every input kind, returning the token at the position (a character, byte or element)
+  anyInput as Parser<Token<NoInfer<I>>, never, never, NoInfer<I>>
+
+/**
+ * The overloads of {@link literal}.
+ *
+ * @since 0.5.0
+ * @category tokens
+ */
+export interface LiteralFn {
+  <const L extends string>(literal: L): Parser<L, never, never, string>
+  (literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>
+  <const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>
+}
 
 /**
  * Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).
@@ -237,11 +264,7 @@ export const anyToken = <I extends Input = string>(): Parser<Token<NoInfer<I>>,
  * @since 0.1.0
  * @category tokens
  */
-export const literal: {
-  <const L extends string>(literal: L): Parser<L, never, never, string>
-  (literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>
-  <const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>
-} = (lit: Input): Parser<any, never, never, any> => literalImpl(lit, false)
+export const literal: LiteralFn = (lit: Input): Parser<any, never, never, any> => literalImpl(lit, false)
 
 /** @internal */
 export const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, never, any> => {
@@ -253,6 +276,7 @@ export const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, n
   return described(
     (state: State<Input>) => {
       const pos = state.pos
+      // SAFETY: `literalKind` 0 is set only for a string literal
       const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit as string)) : lit
       const len = l.length
       const r = matchLiteral(state, l, caseless)
@@ -295,6 +319,7 @@ export const oneOf = <I extends Input = string>(
       const t = nextIf(state, m, false)
       if (t === End) return state.eoi("OneOf", expected, 1)
       if (t === NoMatch) return state.backtrack("OneOf", expected)
+      // SAFETY: past `End` and `NoMatch`, `nextIf` returns the token it consumed from the parser's input `I`
       return t as Token<I>
     },
     m.description,
@@ -319,6 +344,7 @@ export const noneOf = <I extends Input = string>(
       const t = nextIf(state, m, true)
       if (t === End) return state.eoi("NoneOf", expected, 1)
       if (t === NoMatch) return state.backtrack("NoneOf", expected)
+      // SAFETY: past `End` and `NoMatch`, `nextIf` returns the token it consumed from the parser's input `I`
       return t as Token<I>
     },
     expected[0],
@@ -418,6 +444,18 @@ export const takeTill = <I extends Input = string>(
   range?: Range
 ): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => takeWhileImpl(set, range, true, "takeTill")
 
+/**
+ * The overloads of {@link takeUntil}.
+ *
+ * @since 0.5.0
+ * @category slices
+ */
+export interface TakeUntilFn {
+  (literal: string, range?: Range): Parser<string, never, never, string>
+  (literal: Uint8Array, range?: Range): Parser<Uint8Array, never, never, Uint8Array>
+  <T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>
+}
+
 /**
  * Takes tokens up to (not including) the first occurrence of a literal
  * (winnow's `take_until`, nom's `take_until`). The literal is not
@@ -426,25 +464,40 @@ export const takeTill = <I extends Input = string>(
  * @since 0.1.0
  * @category slices
  */
-export const takeUntil: {
-  (literal: string, range?: Range): Parser<string, never, never, string>
-  (literal: Uint8Array, range?: Range): Parser<Uint8Array, never, never, Uint8Array>
-  <T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>
-} = (lit: Input, r?: Range): Parser<any, never, never, any> => {
+export const takeUntil: TakeUntilFn = (lit: Input, r?: Range): Parser<any, never, never, any> =>
+  untilParser(
+    (state, from) => indexOf(state, lit, from),
+    // A string literal is up to three bytes per unit of UTF-8
+    (state) => lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1),
+    [describeLiteral(lit)],
+    r,
+    "takeUntil",
+    { _tag: "TakeUntil", literal: lit, ...bounds(r), encoding: undefined }
+  )
+
+/**
+ * The parser of `takeUntil` and `takeUntilAny`: `search` finds the first
+ * occurrence, which is not consumed (`read` is how far past it the search
+ * read). As in winnow, the first occurrence must lie within the range.
+ */
+const untilParser = (
+  search: (state: State<Input>, from: number) => number,
+  read: (state: State<Input>) => number,
+  expected: ReadonlyArray<string>,
+  r: Range | undefined,
+  name: string,
+  node: Parameters<typeof described>[2]
+): Parser<any, never, never, any> => {
   const [min, max] = normalize(r)
-  const description = describeLiteral(lit)
-  const expected = [description]
   return described(
     (state: State<Input>) => {
       const start = state.pos
-      const idx = indexOf(state, lit, start)
+      const idx = search(state, start)
       if (idx < 0) {
         if (state.partial) return state.incomplete(0, expected)
         return state.backtrack("TakeUntil", expected, state.end)
       }
-      // The literal was read but is not consumed (a string literal is up to three bytes per unit of UTF-8)
-      state.syntax?.read(idx + lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1))
-      // As in winnow, the first occurrence must lie within the range
+      state.syntax?.read(idx + read(state))
       if (min > 0 || max !== Infinity) {
         const count = countTokens(state, start, idx)
         if (count < min) return state.backtrack("TakeUntil", expected, idx)
@@ -453,34 +506,88 @@ export const takeUntil: {
       state.pos = idx
       return state.slice(start, idx)
     },
-    "takeUntil",
-    { _tag: "TakeUntil", literal: lit, ...bounds(r), encoding: undefined }
+    name,
+    node
   )
 }
 
 /**
- * Takes the rest of the input (winnow's and nom's `rest`).
+ * The overloads of {@link takeUntilAny}.
  *
- * @since 0.1.0
+ * @since 0.5.0
+ * @category slices
+ */
+export interface TakeUntilAnyFn {
+  (literals: ReadonlyArray<string>, range?: Range): Parser<string, never, never, string>
+  (literals: ReadonlyArray<Uint8Array>, range?: Range): Parser<Uint8Array, never, never, Uint8Array>
+}
+
+/**
+ * Takes tokens up to (not including) the first occurrence of any of
+ * several literals: {@link takeUntil} for a set of them, the search a
+ * multi-pattern prefilter makes. No literal is consumed. `range` constrains
+ * the number of tokens taken. On bytes, `Compiler.wasm` searches with SIMD
+ * (Teddy, from ripgrep), sixteen bytes at a time.
+ *
+ * @example
+ * ```ts
+ * import { Parser, Token } from "@thomasfosterau/effect-parse"
+ * import * as Result from "effect/Result"
+ *
+ * const upTo = Parser.seq(Token.takeUntilAny(["Holmes", "Watson"]), Token.rest)
+ * console.log(Result.getOrThrow(Parser.parseSync(upTo, "Dr. Watson and Holmes")))
+ * // ["Dr. ", "Watson and Holmes"]
+ * ```
+ *
+ * @since 0.5.0
  * @category slices
  */
-export const rest: Parser<string> = /*#__PURE__*/ described(
-  (state: State<any>) => {
+export const takeUntilAny: TakeUntilAnyFn = (
+  literals: ReadonlyArray<Input>,
+  r?: Range
+): Parser<any, never, never, any> => {
+  const longest = Math.max(0, ...literals.map((lit) => lit.length))
+  return untilParser(
+    (state, from) => indexOfAny(state, literals, from),
+    // Every literal was read up to where it would end (a string literal is up to three bytes per unit of UTF-8)
+    (state) => longest * (state.inputKind === 1 ? 3 : 1),
+    literals.map(describeLiteral),
+    r,
+    "takeUntilAny",
+    { _tag: "TakeUntilAny", literals, ...bounds(r), encoding: undefined }
+  )
+}
+
+/** The parser behind `rest` and `restOf`, on every input kind. */
+const restInput: Parser<unknown, never, never, Input> = /*#__PURE__*/ described(
+  (state) => {
     const start = state.pos
     state.pos = state.end
-    return state.slice(start, state.end) as string
+    return state.slice(start, state.end)
   },
   "rest",
   { _tag: "Rest", encoding: undefined }
 )
 
+/**
+ * Takes the rest of the input (winnow's and nom's `rest`).
+ *
+ * @since 0.1.0
+ * @category slices
+ */
+export const rest: Parser<string> =
+  // SAFETY: on string input, `State.slice` returns a string
+  restInput as Parser<string>
+
 /**
  * Takes the rest of an input of type `I`.
  *
  * @since 0.1.0
  * @category slices
  */
-export const restOf = <I extends Input = string>(): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => rest as any
+export const restOf = <I extends Input = string>(): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> =>
+  // SAFETY: `State.slice` returns the input's own slice type for every input kind, so `restInput` returns `Slice<I>`
+  restInput as Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>>
 
 /**
  * Returns the length of the remaining input without consuming it (winnow's
@@ -506,4 +613,6 @@ export const restLen: Parser<number, never, never, Input> = /*#__PURE__*/ descri
  */
 export const satisfy = <I extends Input = string>(
   predicate: (token: Token<I>) => boolean
-): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => oneOf<I>(predicate as any)
+): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> =>
+  // SAFETY: a predicate over the input's tokens is a member of `TokenSet<I>` for each input kind
+  oneOf<I>(predicate as TokenSet<I>)