@thomasfosterau/effect-parse 0.4.1 → 0.6.0

package/src/internal/tokens.ts

@@ -1,5 +1,3 @@
-// fallow-ignore-file duplicate-export -- `End` and `scan` here are the token-set versions of the text
-// helpers in ./text.ts; both are internal modules, imported by path, never re-exported from one barrel.
 import * as Predicate from "effect/Predicate"
 import * as Result from "effect/Result"
 import type { Input, State } from "../Input.js"
@@ -22,7 +20,7 @@ export interface TokenRangeLike {
   readonly to: string | number
 }
 
-const isRange = (u: unknown): u is TokenRangeLike => Predicate.isObjectOrArray(u) && (u as any)._tag === "TokenRange"
+const isRange = (u: unknown): u is TokenRangeLike => Predicate.isObjectOrArray(u) && Predicate.isTagged(u, "TokenRange")
 
 const toCode = (u: string | number): number => (Predicate.isNumber(u) ? u : firstCodePoint(u))
 
@@ -67,7 +65,10 @@ const compileCode = (u: unknown): ((code: number) => boolean) => {
     return bitmap(codes)
   }
   if (Predicate.isNumber(u)) return (x) => x === u
-  if (Predicate.isFunction(u)) return u as (code: number) => boolean
+  if (Predicate.isFunction(u)) {
+    // SAFETY: a predicate compiled for codes is a byte, code point or `codePoints` predicate (`compile` wraps string predicates first)
+    return u as (code: number) => boolean
+  }
   if (u instanceof Uint8Array) return bitmap(Array.from(u))
   if (isRange(u)) {
     const from = toCode(u.from)
@@ -98,7 +99,10 @@ const bitmap = (codes: ReadonlyArray<number>): ((code: number) => boolean) => {
 }
 
 const compileToken = (u: unknown): ((token: unknown) => boolean) => {
-  if (Predicate.isFunction(u)) return u as (token: unknown) => boolean
+  if (Predicate.isFunction(u)) {
+    // SAFETY: a predicate in an array input's token set takes the array's elements, the tokens tested here
+    return u as (token: unknown) => boolean
+  }
   if (Array.isArray(u)) {
     const set = new Set(u)
     return (t) => set.has(t)
@@ -130,8 +134,13 @@ export const codePointPredicate = <F extends (c: never) => boolean>(
   ranges?: ReadonlyArray<readonly [number, number]>
 ): F => {
   Object.defineProperty(f, "name", { value: name })
-  ;(f as any)[CodePointPredicate] = true
-  if (ranges !== undefined) (f as any)[PredicateRanges] = ranges
+  // SAFETY: `f` is a function object, which takes new properties; the intersection only names the two markers set here
+  const marked = f as F & {
+    [CodePointPredicate]?: true
+    [PredicateRanges]?: ReadonlyArray<readonly [number, number]>
+  }
+  marked[CodePointPredicate] = true
+  if (ranges !== undefined) marked[PredicateRanges] = ranges
   return f
 }
 
@@ -144,6 +153,7 @@ export const compile = (set: unknown, kind: "string" | "code" = "code"): Matcher
   const description = describeSet(set)
   let code: (code: number) => boolean
   if (Predicate.isFunction(set) && kind === "string" && !(CodePointPredicate in set)) {
+    // SAFETY: an unmarked predicate in a string token set takes each character as a string (`StringTokenSet`)
     const pred = set as (c: string) => boolean
     code = (x) => pred(charOf(x))
   } else {
@@ -227,6 +237,7 @@ const compileAnyImpl = (set: unknown): AnyMatcher => {
  */
 export const splitSurrogate = (state: State<Input>, i: number): boolean => {
   if (!state.partial || i !== state.end - 1) return false
+  // SAFETY: called only on string input (the `inputKind === 0` paths of `nextIf`, `scan`, `advance` and `Token.any`)
   const c = (state.input as string).charCodeAt(i)
   return c >= 0xd800 && c <= 0xdbff
 }
@@ -253,6 +264,7 @@ export const nextIf = (state: State<Input>, m: AnyMatcher, negate: boolean): unk
   if (pos >= state.end) return End
   switch (state.inputKind) {
     case 0: {
+      // SAFETY: `inputKind` 0 is set only for a string input
       const s = state.input as string
       if (splitSurrogate(state, pos)) return End
       const cp = codePointAt(s, pos)
@@ -262,12 +274,14 @@ export const nextIf = (state: State<Input>, m: AnyMatcher, negate: boolean): unk
       return len === 1 ? s[pos] : s.slice(pos, pos + 2)
     }
     case 1: {
+      // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input
       const b = (state.input as Uint8Array)[pos]
       if (m.forCode.code(b) === negate) return NoMatch
       state.pos = pos + 1
       return b
     }
     default: {
+      // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array
       const t = (state.input as ReadonlyArray<unknown>)[pos]
       if (m.forCode.token(t) === negate) return NoMatch
       state.pos = pos + 1
@@ -282,10 +296,32 @@ export interface Scanned extends Advanced {
   readonly hitEnd: boolean
 }
 
+/** The one record {@link scan} fills; see there. */
+class ScanResult implements Scanned {
+  offset = 0
+  count = 0
+  hitEnd = false
+}
+
+const scanned = /*#__PURE__*/ new ScanResult()
+
+/** Fills {@link scanned} and returns it. */
+const scannedAt = (offset: number, count: number, hitEnd: boolean): Scanned => {
+  scanned.offset = offset
+  scanned.count = count
+  scanned.hitEnd = hitEnd
+  return scanned
+}
+
 /**
  * Scans forward while tokens satisfy the matcher (xor `negate`), up to `max`
  * tokens. Returns the offset reached, the number of tokens matched, and
  * whether the end of input was hit before `max` tokens.
+ *
+ * The result is one shared record, overwritten by the next scan: read it
+ * before running anything else. Allocating one per call cost the
+ * interpreter a record per token, since `scan` (shared by every `takeWhile`)
+ * is too polymorphic for V8 to inline and remove it.
  */
 export const scan = (state: State<Input>, m: AnyMatcher, negate: boolean, max: number): Scanned => {
   let i = state.pos
@@ -293,45 +329,48 @@ export const scan = (state: State<Input>, m: AnyMatcher, negate: boolean, max: n
   let count = 0
   switch (state.inputKind) {
     case 0: {
+      // SAFETY: `inputKind` 0 is set only for a string input
       const s = state.input as string
       const test = m.forString.code
       while (count < max) {
-        if (i >= end) return { offset: i, count, hitEnd: true }
+        if (i >= end) return scannedAt(i, count, true)
         const c = s.charCodeAt(i)
         if (c < 0xd800 || c > 0xdbff) {
           if (test(c) === negate) break
           i++
         } else {
-          if (splitSurrogate(state, i)) return { offset: i, count, hitEnd: true }
+          if (splitSurrogate(state, i)) return scannedAt(i, count, true)
           const cp = codePointAt(s, i)
           if (test(cp) === negate) break
           i += cp > 0xffff ? 2 : 1
         }
         count++
       }
-      return { offset: i, count, hitEnd: false }
+      return scannedAt(i, count, false)
     }
     case 1: {
+      // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input
       const bytes = state.input as Uint8Array
       const test = m.forCode.code
       while (count < max) {
-        if (i >= end) return { offset: i, count, hitEnd: true }
+        if (i >= end) return scannedAt(i, count, true)
         if (test(bytes[i]) === negate) break
         i++
         count++
       }
-      return { offset: i, count, hitEnd: false }
+      return scannedAt(i, count, false)
     }
     default: {
+      // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array
       const arr = state.input as ReadonlyArray<unknown>
       const test = m.forCode.token
       while (count < max) {
-        if (i >= end) return { offset: i, count, hitEnd: true }
+        if (i >= end) return scannedAt(i, count, true)
         if (test(arr[i]) === negate) break
         i++
         count++
       }
-      return { offset: i, count, hitEnd: false }
+      return scannedAt(i, count, false)
     }
   }
 }
@@ -348,6 +387,7 @@ export const advance = (state: State<Input>, start: number, n: number): Advanced
     const offset = Math.min(state.end, start + n)
     return { offset, count: offset - start }
   }
+  // SAFETY: the other input kinds returned above, and `inputKind` 0 is set only for a string input
   const s = state.input as string
   let i = start
   let count = 0
@@ -363,6 +403,7 @@ export const advance = (state: State<Input>, start: number, n: number): Advanced
 /** Counts tokens between two offsets. */
 export const countTokens = (state: State<Input>, start: number, end: number): number => {
   if (state.inputKind !== 0) return end - start
+  // SAFETY: the other input kinds returned above, and `inputKind` 0 is set only for a string input
   const s = state.input as string
   let count = 0
   for (let i = start; i < end; i++) {
@@ -373,6 +414,46 @@ export const countTokens = (state: State<Input>, start: number, end: number): nu
   return count
 }
 
+/**
+ * The first index at or after `from` of any of `needles` within
+ * `[from, state.end)`, or -1. The state remembers, under `needles`, where
+ * each one next occurs (`[from, at, end]` at `3 * i`), so successive
+ * searches forwards over one input search each needle once per occurrence
+ * rather than once per call (as `nextStop` does for `takeTill`).
+ *
+ * A remembered answer is used only where it still holds: an occurrence, when
+ * the search starts between where it was searched from and where it was
+ * found, and it still fits before the end; "not found", when the search
+ * starts no earlier and the end is no further than the end that search ran
+ * to (a parser may narrow `state.end` for a while, and `reset` widens it
+ * again, so a miss within a narrower end says nothing about a wider one).
+ */
+export const indexOfAny = (state: State<Input>, needles: ReadonlyArray<Input>, from: number): number => {
+  const searches = (state.searches ??= new Map())
+  let memo = searches.get(needles)
+  if (memo === undefined) {
+    memo = needles.flatMap(() => [-1, -1, -1])
+    searches.set(needles, memo)
+  }
+  let best = -1
+  for (let i = 0; i < needles.length; i++) {
+    const known = memo[3 * i] ?? -1
+    let at = memo[3 * i + 1] ?? -1
+    const searchedTo = memo[3 * i + 2] ?? -1
+    const needle = needles[i] ?? ""
+    const stale =
+      known < 0 || from < known || (at >= 0 ? at < from || at > state.end - needle.length : state.end > searchedTo)
+    if (stale) {
+      at = indexOf(state, needle, from)
+      memo[3 * i] = from
+      memo[3 * i + 1] = at
+      memo[3 * i + 2] = state.end
+    }
+    if (at >= 0 && (best < 0 || at < best)) best = at
+  }
+  return best
+}
+
 /** Finds `needle` within `[from, state.end)`; returns -1 if absent. */
 export const indexOf = (state: State<Input>, needle: Input, from: number): number => {
   // The whole needle must lie before `state.end`, which may narrow the input
@@ -380,11 +461,14 @@ export const indexOf = (state: State<Input>, needle: Input, from: number): numbe
   if (from > last) return -1
   switch (state.inputKind) {
     case 0: {
+      // SAFETY: `inputKind` 0 is set only for a string input, and the callers' types give the needle the input's kind
       const i = (state.input as string).indexOf(needle as string, from)
       return i <= last ? i : -1
     }
     case 1: {
+      // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input
       const hay = state.input as Uint8Array
+      // SAFETY: the callers' types give the needle the input's kind (`Text` encodes it for byte input)
       const n = needle as Uint8Array
       if (n.byteLength === 0) return from
       const first = n[0]
@@ -398,7 +482,9 @@ export const indexOf = (state: State<Input>, needle: Input, from: number): numbe
       return -1
     }
     default: {
+      // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array
       const hay = state.input as ReadonlyArray<unknown>
+      // SAFETY: the callers' types give the needle the input's kind, an array of the same element type
       const n = needle as ReadonlyArray<unknown>
       outer: for (let i = from; i <= last; i++) {
         for (let j = 0; j < n.length; j++) if (hay[i + j] !== n[j]) continue outer
@@ -428,7 +514,9 @@ const match = (state: State<Input>, lit: Input, caseless: boolean): number => {
   const n = Math.min(len, avail)
   switch (state.inputKind) {
     case 0: {
+      // SAFETY: `inputKind` 0 is set only for a string input
       const s = state.input as string
+      // SAFETY: the callers' types give the literal the input's kind (`Token.literal` encodes a string literal for byte input)
       const l = lit as string
       if (!caseless) {
         if (avail >= len) return s.startsWith(l, pos) ? len : -1
@@ -440,7 +528,9 @@ const match = (state: State<Input>, lit: Input, caseless: boolean): number => {
       return n === len ? len : -2
     }
     case 1: {
+      // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input
       const b = state.input as Uint8Array
+      // SAFETY: the callers' types give the literal the input's kind (`Token.literal` and `Text` encode string literals for byte input)
       const l = lit as Uint8Array
       for (let i = 0; i < n; i++) {
         const x = b[pos + i]
@@ -450,7 +540,9 @@ const match = (state: State<Input>, lit: Input, caseless: boolean): number => {
       return n === len ? len : -2
     }
     default: {
+      // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array
       const a = state.input as ReadonlyArray<unknown>
+      // SAFETY: the callers' types give the literal the input's kind, an array of the same element type
       const l = lit as ReadonlyArray<unknown>
       for (let i = 0; i < n; i++) if (a[pos + i] !== l[i]) return -1
       return n === len ? len : -2

package/src/internal/tuple.ts

@@ -0,0 +1,141 @@
+/**
+ * Tuple steps: run steps in sequence and collect their outputs in an array.
+ *
+ * V8 pretenures by allocation site, and only array and object literals have
+ * one: an array from `new Array(n)` starts young, so tuples that a parse keeps
+ * (a `repeat(seq(...))`, a tree) are copied by a scavenge or two before they
+ * are promoted. A literal's site learns that its arrays survive, and V8 then
+ * allocates them straight into the old generation. So arities up to
+ * {@link maxLiteral} build their tuple with a literal of that arity, each its
+ * own site; longer tuples fall back to `new Array(n)` (#86).
+ */
+import { Fail } from "../Input.js"
+import type { State } from "../Input.js"
+
+type Step = (state: State<any>) => unknown
+
+/** The longest tuple built with a literal. */
+const maxLiteral = 8
+
+/** A step that runs `steps` in order and returns their outputs as an array, or {@link Fail}. */
+export const tupleStep = (steps: ReadonlyArray<Step>): ((state: State<any>) => Array<unknown> | Fail) => {
+  const n = steps.length
+  if (n <= maxLiteral) return literals[n](steps)
+  return (state) => {
+    const out = new Array<unknown>(n)
+    for (let i = 0; i < n; i++) {
+      const a = steps[i](state)
+      if (a === Fail) return Fail
+      out[i] = a
+    }
+    return out
+  }
+}
+
+const literals: ReadonlyArray<(steps: ReadonlyArray<Step>) => (state: State<any>) => Array<unknown> | Fail> = [
+  () => () => [],
+  ([s0]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      return [a0]
+    },
+  ([s0, s1]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      return [a0, a1]
+    },
+  ([s0, s1, s2]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      return [a0, a1, a2]
+    },
+  ([s0, s1, s2, s3]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      const a3 = s3(state)
+      if (a3 === Fail) return Fail
+      return [a0, a1, a2, a3]
+    },
+  ([s0, s1, s2, s3, s4]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      const a3 = s3(state)
+      if (a3 === Fail) return Fail
+      const a4 = s4(state)
+      if (a4 === Fail) return Fail
+      return [a0, a1, a2, a3, a4]
+    },
+  ([s0, s1, s2, s3, s4, s5]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      const a3 = s3(state)
+      if (a3 === Fail) return Fail
+      const a4 = s4(state)
+      if (a4 === Fail) return Fail
+      const a5 = s5(state)
+      if (a5 === Fail) return Fail
+      return [a0, a1, a2, a3, a4, a5]
+    },
+  ([s0, s1, s2, s3, s4, s5, s6]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      const a3 = s3(state)
+      if (a3 === Fail) return Fail
+      const a4 = s4(state)
+      if (a4 === Fail) return Fail
+      const a5 = s5(state)
+      if (a5 === Fail) return Fail
+      const a6 = s6(state)
+      if (a6 === Fail) return Fail
+      return [a0, a1, a2, a3, a4, a5, a6]
+    },
+  ([s0, s1, s2, s3, s4, s5, s6, s7]) =>
+    (state) => {
+      const a0 = s0(state)
+      if (a0 === Fail) return Fail
+      const a1 = s1(state)
+      if (a1 === Fail) return Fail
+      const a2 = s2(state)
+      if (a2 === Fail) return Fail
+      const a3 = s3(state)
+      if (a3 === Fail) return Fail
+      const a4 = s4(state)
+      if (a4 === Fail) return Fail
+      const a5 = s5(state)
+      if (a5 === Fail) return Fail
+      const a6 = s6(state)
+      if (a6 === Fail) return Fail
+      const a7 = s7(state)
+      if (a7 === Fail) return Fail
+      return [a0, a1, a2, a3, a4, a5, a6, a7]
+    }
+]