@thomasfosterau/effect-parse 0.1.0 → 0.2.0

package/src/internal/tokens.ts

@@ -1,4 +1,8 @@
+// fallow-ignore-file duplicate-export -- `End` and `scan` here are the token-set versions of the text
+// helpers in ./text.ts; both are internal modules, imported by path, never re-exported from one barrel.
+import * as Predicate from "effect/Predicate"
 import type { Input, State } from "../Input.js"
+import { charOf, firstCodePoint } from "./text.js"
 
 /** A compiled token set. */
 export interface Matcher {
@@ -17,21 +21,20 @@ export interface TokenRangeLike {
   readonly to: string | number
 }
 
-const isRange = (u: unknown): u is TokenRangeLike =>
-  typeof u === "object" && u !== null && (u as any)._tag === "TokenRange"
+const isRange = (u: unknown): u is TokenRangeLike => Predicate.isObjectOrArray(u) && (u as any)._tag === "TokenRange"
 
-const toCode = (u: string | number): number => typeof u === "number" ? u : u.codePointAt(0)!
+const toCode = (u: string | number): number => (Predicate.isNumber(u) ? u : firstCodePoint(u))
 
 /** @internal */
 export const describeChar = (s: string): string => JSON.stringify(s)
 
 const describeSet = (u: unknown): string => {
-  if (typeof u === "string") {
+  if (Predicate.isString(u)) {
     const chars = Array.from(u)
     return chars.length === 1 ? describeChar(u) : `one of ${JSON.stringify(u)}`
   }
-  if (typeof u === "number") return `0x${u.toString(16).padStart(2, "0")}`
-  if (typeof u === "function") return u.name ? u.name : "a matching token"
+  if (Predicate.isNumber(u)) return `0x${u.toString(16).padStart(2, "0")}`
+  if (Predicate.isFunction(u)) return u.name ? u.name : "a matching token"
   if (u instanceof Uint8Array) return `one of ${describeBytes(u)}`
   if (isRange(u)) return `${describeSet(u.from)}..=${describeSet(u.to)}`
   if (Array.isArray(u)) return u.map(describeSet).join(" or ")
@@ -39,10 +42,10 @@ const describeSet = (u: unknown): string => {
 }
 
 /** Renders bytes as a Rust-style byte string literal: b"GET\r\n". */
-export const describeBytes = (bytes: Uint8Array): string => {
-  let out = "b\""
+const describeBytes = (bytes: Uint8Array): string => {
+  let out = 'b"'
   for (const b of bytes) {
-    if (b === 0x22) out += "\\\""
+    if (b === 0x22) out += '\\"'
     else if (b === 0x5c) out += "\\\\"
     else if (b === 0x0a) out += "\\n"
     else if (b === 0x0d) out += "\\r"
@@ -50,20 +53,20 @@ export const describeBytes = (bytes: Uint8Array): string => {
     else if (b >= 0x20 && b < 0x7f) out += String.fromCharCode(b)
     else out += `\\x${b.toString(16).padStart(2, "0")}`
   }
-  return out + "\""
+  return out + '"'
 }
 
-const compileCode = (u: unknown): (code: number) => boolean => {
-  if (typeof u === "string") {
-    const codes = Array.from(u, (c) => c.codePointAt(0)!)
+const compileCode = (u: unknown): ((code: number) => boolean) => {
+  if (Predicate.isString(u)) {
+    const codes = Array.from(u, firstCodePoint)
     if (codes.length === 1) {
-      const c = codes[0]!
+      const c = codes[0]
       return (x) => x === c
     }
     return bitmap(codes)
   }
-  if (typeof u === "number") return (x) => x === u
-  if (typeof u === "function") return u as (code: number) => boolean
+  if (Predicate.isNumber(u)) return (x) => x === u
+  if (Predicate.isFunction(u)) return u as (code: number) => boolean
   if (u instanceof Uint8Array) return bitmap(Array.from(u))
   if (isRange(u)) {
     const from = toCode(u.from)
@@ -72,16 +75,16 @@ const compileCode = (u: unknown): (code: number) => boolean => {
   }
   if (Array.isArray(u)) {
     const tests = u.map(compileCode)
-    if (tests.length === 1) return tests[0]!
+    if (tests.length === 1) return tests[0]
     return (x) => {
-      for (let i = 0; i < tests.length; i++) if (tests[i]!(x)) return true
+      for (let i = 0; i < tests.length; i++) if (tests[i](x)) return true
       return false
     }
   }
   throw new TypeError(`invalid token set: ${String(u)}`)
 }
 
-const bitmap = (codes: ReadonlyArray<number>): (code: number) => boolean => {
+const bitmap = (codes: ReadonlyArray<number>): ((code: number) => boolean) => {
   const ascii = new Uint8Array(256)
   let other: Set<number> | undefined
   for (const c of codes) {
@@ -90,11 +93,11 @@ const bitmap = (codes: ReadonlyArray<number>): (code: number) => boolean => {
   }
   if (other === undefined) return (x) => x < 256 && ascii[x] === 1
   const set = other
-  return (x) => x < 256 ? ascii[x] === 1 : set.has(x)
+  return (x) => (x < 256 ? ascii[x] === 1 : set.has(x))
 }
 
-const compileToken = (u: unknown): (token: unknown) => boolean => {
-  if (typeof u === "function") return u as (token: unknown) => boolean
+const compileToken = (u: unknown): ((token: unknown) => boolean) => {
+  if (Predicate.isFunction(u)) return u as (token: unknown) => boolean
   if (Array.isArray(u)) {
     const set = new Set(u)
     return (t) => set.has(t)
@@ -107,14 +110,14 @@ const compileToken = (u: unknown): (token: unknown) => boolean => {
  * number), so string inputs can call it without allocating a string per
  * character.
  */
-export const CodePointPredicate: unique symbol = Symbol.for("effect-parse/CodePointPredicate")
+const CodePointPredicate: unique symbol = /*#__PURE__*/ Symbol.for("effect-parse/CodePointPredicate")
 
 /**
  * The code points accepted by a predicate, as sorted, disjoint, inclusive
  * ranges, when they are known. The compiler turns such predicates into
  * comparisons or tables.
  */
-export const PredicateRanges: unique symbol = Symbol.for("effect-parse/PredicateRanges")
+export const PredicateRanges: unique symbol = /*#__PURE__*/ Symbol.for("effect-parse/PredicateRanges")
 
 /**
  * Marks `f` as a {@link CodePointPredicate} named `name` (used in error
@@ -133,26 +136,27 @@ export const codePointPredicate = <F extends (c: never) => boolean>(
 
 /**
  * Compiles a token set. Predicates for string inputs receive the token as a
- * string, so they are wrapped here.
+ * string, so they are wrapped here; `charOf` avoids `String.fromCodePoint`,
+ * which is several times slower, for the Basic Multilingual Plane.
  */
 export const compile = (set: unknown, kind: "string" | "code" = "code"): Matcher => {
   const description = describeSet(set)
   let code: (code: number) => boolean
-  if (typeof set === "function" && kind === "string" && !(CodePointPredicate in set)) {
+  if (Predicate.isFunction(set) && kind === "string" && !(CodePointPredicate in set)) {
     const pred = set as (c: string) => boolean
-    code = (x) => pred(String.fromCodePoint(x))
+    code = (x) => pred(charOf(x))
   } else {
     code = compileCodeSafe(set, kind)
   }
   return { code, token: compileToken(set), description }
 }
 
-const compileCodeSafe = (set: unknown, kind: "string" | "code"): (code: number) => boolean => {
+const compileCodeSafe = (set: unknown, kind: "string" | "code"): ((code: number) => boolean) => {
   if (Array.isArray(set) && kind === "string") {
     // Predicates nested in arrays also take strings
     const tests = set.map((s) => compile(s, kind).code)
     return (x) => {
-      for (let i = 0; i < tests.length; i++) if (tests[i]!(x)) return true
+      for (let i = 0; i < tests.length; i++) if (tests[i](x)) return true
       return false
     }
   }
@@ -173,9 +177,42 @@ export interface AnyMatcher {
   readonly description: string
 }
 
-export const compileAny = (set: unknown): AnyMatcher => {
+/**
+ * A small cache keyed by value, for the parts of a parser that are costly to
+ * build and depend only on a string (compiled sets, descriptions). Parsers
+ * built inside `Parser.gen` bodies are rebuilt on every run, usually from the
+ * same few strings. Dynamic strings could grow it without bound, so it is
+ * cleared when full.
+ */
+const cached = <A>(cache: Map<string, A>, key: string, build: (key: string) => A): A => {
+  let a = cache.get(key)
+  if (a === undefined) {
+    if (cache.size >= 1024) cache.clear()
+    a = build(key)
+    cache.set(key, a)
+  }
+  return a
+}
+
+const literalDescriptions = /*#__PURE__*/ new Map<string, string>()
+
+/** Describes a literal for error messages: `"abc"`, `b"abc"` or `["a", 1]`. */
+export const describeLiteral = (lit: Input): string =>
+  Predicate.isString(lit)
+    ? cached(literalDescriptions, lit, describeChar)
+    : lit instanceof Uint8Array
+      ? describeBytes(lit)
+      : `[${lit.map((t) => JSON.stringify(t)).join(", ")}]`
+
+const stringMatchers = /*#__PURE__*/ new Map<string, AnyMatcher>()
+
+export const compileAny = (set: unknown): AnyMatcher =>
+  // Matchers for strings are pure functions of the string, so they're shared
+  Predicate.isString(set) ? cached(stringMatchers, set, compileAnyImpl) : compileAnyImpl(set)
+
+const compileAnyImpl = (set: unknown): AnyMatcher => {
   const forCode = compile(set, "code")
-  const needsString = typeof set === "function" || Array.isArray(set)
+  const needsString = Predicate.isFunction(set) || Array.isArray(set)
   return {
     forString: needsString ? compile(set, "string") : forCode,
     forCode,
@@ -195,7 +232,7 @@ export const splitSurrogate = (state: State<Input>, i: number): boolean => {
 }
 
 /** Reads the code point at `i` of a string. */
-export const codePointAt = (s: string, i: number): number => {
+const codePointAt = (s: string, i: number): number => {
   const c = s.charCodeAt(i)
   if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) {
     const d = s.charCodeAt(i + 1)
@@ -208,14 +245,10 @@ export const codePointAt = (s: string, i: number): number => {
  * Tests the token at the current position. Returns the token (as the input's
  * token type) and advances, or returns `NoMatch`/`End`.
  */
-export const NoMatch: unique symbol = Symbol("NoMatch")
-export const End: unique symbol = Symbol("End")
+export const NoMatch: unique symbol = /*#__PURE__*/ Symbol("NoMatch")
+export const End: unique symbol = /*#__PURE__*/ Symbol("End")
 
-export const nextIf = (
-  state: State<Input>,
-  m: AnyMatcher,
-  negate: boolean
-): unknown => {
+export const nextIf = (state: State<Input>, m: AnyMatcher, negate: boolean): unknown => {
   const pos = state.pos
   if (pos >= state.end) return End
   switch (state.inputKind) {
@@ -229,7 +262,7 @@ export const nextIf = (
       return len === 1 ? s[pos] : s.slice(pos, pos + 2)
     }
     case 1: {
-      const b = (state.input as Uint8Array)[pos]!
+      const b = (state.input as Uint8Array)[pos]
       if (m.forCode.code(b) === negate) return NoMatch
       state.pos = pos + 1
       return b
@@ -282,7 +315,7 @@ export const scan = (
       const test = m.forCode.code
       while (count < max) {
         if (i >= end) return { offset: i, count, hitEnd: true }
-        if (test(bytes[i]!) === negate) break
+        if (test(bytes[i]) === negate) break
         i++
         count++
       }
@@ -333,18 +366,23 @@ export const countTokens = (state: State<Input>, start: number, end: number): nu
   return count
 }
 
-/** Finds `needle` in the input at or after `from`; returns -1 if absent. */
+/** Finds `needle` within `[from, state.end)`; returns -1 if absent. */
 export const indexOf = (state: State<Input>, needle: Input, from: number): number => {
+  // The whole needle must lie before `state.end`, which may narrow the input
+  const last = state.end - needle.length
+  if (from > last) return -1
   switch (state.inputKind) {
-    case 0:
-      return (state.input as string).indexOf(needle as string, from)
+    case 0: {
+      const i = (state.input as string).indexOf(needle as string, from)
+      return i <= last ? i : -1
+    }
     case 1: {
       const hay = state.input as Uint8Array
       const n = needle as Uint8Array
-      if (n.length === 0) return from <= hay.length ? from : -1
-      const first = n[0]!
+      if (n.length === 0) return from
+      const first = n[0]
       let i = hay.indexOf(first, from)
-      while (i !== -1 && i + n.length <= hay.length) {
+      while (i !== -1 && i <= last) {
         let j = 1
         while (j < n.length && hay[i + j] === n[j]) j++
         if (j === n.length) return i
@@ -355,7 +393,7 @@ export const indexOf = (state: State<Input>, needle: Input, from: number): numbe
     default: {
       const hay = state.input as ReadonlyArray<unknown>
       const n = needle as ReadonlyArray<unknown>
-      outer: for (let i = from; i + n.length <= hay.length; i++) {
+      outer: for (let i = from; i <= last; i++) {
         for (let j = 0; j < n.length; j++) if (hay[i + j] !== n[j]) continue outer
         return i
       }
@@ -370,6 +408,13 @@ export const indexOf = (state: State<Input>, needle: Input, from: number): numbe
  * -1 on mismatch, or -2 when the input ends while matching a prefix.
  */
 export const matchLiteral = (state: State<Input>, lit: Input, caseless: boolean): number => {
+  const r = match(state, lit, caseless)
+  // A failed match may have read up to its length past where it fails (see `Syntax`)
+  if (r < 0 && state.syntax !== undefined) state.syntax.read(state.pos + Math.min(lit.length, state.end - state.pos))
+  return r
+}
+
+const match = (state: State<Input>, lit: Input, caseless: boolean): number => {
   const pos = state.pos
   const len = lit.length
   const avail = state.end - pos
@@ -391,8 +436,8 @@ export const matchLiteral = (state: State<Input>, lit: Input, caseless: boolean)
       const b = state.input as Uint8Array
       const l = lit as Uint8Array
       for (let i = 0; i < n; i++) {
-        const x = b[pos + i]!
-        const y = l[i]!
+        const x = b[pos + i]
+        const y = l[i]
         if (caseless ? lowerAscii(x) !== lowerAscii(y) : x !== y) return -1
       }
       return n === len ? len : -2
@@ -406,4 +451,4 @@ export const matchLiteral = (state: State<Input>, lit: Input, caseless: boolean)
   }
 }
 
-const lowerAscii = (c: number): number => c >= 65 && c <= 90 ? c + 32 : c
+const lowerAscii = (c: number): number => (c >= 65 && c <= 90 ? c + 32 : c)

package/dist/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA;AACjD,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA"}

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA;AACjD,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA"}