@thomasfosterau/effect-parse 0.0.0-stage → 0.1.0

package/src/internal/compiler/registry.ts

@@ -0,0 +1,79 @@
+/**
+ * The registry of compiled parsers, keyed by parser (see `Parser.compile`).
+ * Compiled steps are built lazily, once per input kind.
+ */
+import type { State } from "../../Input.js"
+import type { ParserLike } from "../node.js"
+import type { Fallback, Step } from "./closure.js"
+import { compileClosure } from "./closure.js"
+import { compileJit, jitSupported } from "./jit.js"
+
+/** Which compiler to use. */
+export type Backend = "auto" | "closure"
+
+/** What was compiled for a parser and an input kind. */
+export interface Compiled {
+  /** The compiler actually used. */
+  readonly backend: "jit" | "closure" | undefined
+  readonly step: Step | undefined
+  /** Parts of the grammar that run interpreted, and why. */
+  readonly fallbacks: ReadonlyArray<Fallback>
+  /** Why compilation failed, if it did (the parser then runs interpreted). */
+  readonly error: unknown
+  /** Why the JIT failed, if it did (the closure compiler is then used). */
+  readonly jitError?: unknown
+  /** The generated source, for the JIT. */
+  readonly source?: string
+}
+
+interface Entry {
+  readonly backend: Backend
+  readonly byKind: Array<Compiled | undefined>
+}
+
+const registry = new WeakMap<ParserLike, Entry>()
+
+export const register = (p: ParserLike, backend: Backend): void => {
+  const entry = registry.get(p)
+  if (entry === undefined || entry.backend !== backend) registry.set(p, { backend, byKind: [] })
+}
+
+export const isRegistered = (p: ParserLike): boolean => registry.has(p)
+
+const build = (p: ParserLike, kind: 0 | 1 | 2, backend: Backend, resumable: boolean): Compiled => {
+  if (backend === "auto" && jitSupported()) {
+    try {
+      const { fallbacks, source, step } = compileJit(p, kind, resumable)
+      return { backend: "jit", step, fallbacks, error: undefined, source }
+    } catch (error) {
+      // Fall back to the closure compiler, remembering why
+      return { ...buildClosure(p, kind, resumable), jitError: error }
+    }
+  }
+  return buildClosure(p, kind, resumable)
+}
+
+const buildClosure = (p: ParserLike, kind: 0 | 1 | 2, resumable: boolean): Compiled => {
+  try {
+    const { fallbacks, step } = compileClosure(p, kind, resumable)
+    return { backend: "closure", step, fallbacks, error: undefined }
+  } catch (error) {
+    return { backend: undefined, step: undefined, fallbacks: [], error }
+  }
+}
+
+/**
+ * The compiled form of a registered parser for an input kind, compiling it
+ * if needed. `resumable` selects the build used by streaming parses, which
+ * supports partial input and checkpoints.
+ */
+export const compiled = (p: ParserLike, kind: 0 | 1 | 2, resumable = false): Compiled | undefined => {
+  const entry = registry.get(p)
+  if (entry === undefined) return undefined
+  const index = resumable ? kind + 3 : kind
+  return entry.byKind[index] ??= build(p, kind, entry.backend, resumable)
+}
+
+/** The compiled step for running `p` on `state`, if it is registered. */
+export const stepFor = (p: ParserLike, state: State<any>, resumable = false): Step | undefined =>
+  compiled(p, state.inputKind, resumable)?.step

package/src/internal/compiler/sets.ts

@@ -0,0 +1,100 @@
+/**
+ * Token sets for compiled parsers: membership tables for small units, the
+ * exact test for the rest, and the code point ranges when they are known.
+ */
+import type { Encoding } from "../../Encoding.js"
+import { compile, compileAny, PredicateRanges } from "../tokens.js"
+
+/** Size of the membership table: every byte, and the Latin-1 code points. */
+export const TableSize = 256
+
+export interface CharSet {
+  /** Tests a code point (or byte, for `Token` on bytes). */
+  readonly test: (code: number) => boolean
+  /** `table[c] === 1` iff `test(c)`, for `c < TableSize`. */
+  readonly table: Uint8Array
+  /** Sorted inclusive ranges of accepted code points, when known. */
+  readonly ranges: ReadonlyArray<readonly [number, number]> | undefined
+}
+
+const isRange = (u: unknown): u is { readonly from: string | number; readonly to: string | number } =>
+  typeof u === "object" && u !== null && (u as any)._tag === "TokenRange"
+
+const toCode = (u: string | number): number => typeof u === "number" ? u : u.codePointAt(0)!
+
+/** The code point ranges of a token set, or `undefined` if it has an unknown predicate. */
+export const rangesOf = (set: unknown): Array<[number, number]> | undefined => {
+  const out: Array<[number, number]> = []
+  const add = (u: unknown): boolean => {
+    if (typeof u === "string") {
+      for (const c of u) {
+        const cp = c.codePointAt(0)!
+        out.push([cp, cp])
+      }
+      return true
+    }
+    if (typeof u === "number") {
+      out.push([u, u])
+      return true
+    }
+    if (u instanceof Uint8Array) {
+      for (const b of u) out.push([b, b])
+      return true
+    }
+    if (isRange(u)) {
+      out.push([toCode(u.from), toCode(u.to)])
+      return true
+    }
+    if (typeof u === "function") {
+      const r = (u as any)[PredicateRanges] as ReadonlyArray<readonly [number, number]> | undefined
+      if (r === undefined) return false
+      for (const [a, b] of r) out.push([a, b])
+      return true
+    }
+    if (Array.isArray(u)) return u.every(add)
+    return false
+  }
+  if (!add(set)) return undefined
+  out.sort((a, b) => a[0] - b[0])
+  const merged: Array<[number, number]> = []
+  for (const [a, b] of out) {
+    if (b < a) continue
+    const last = merged[merged.length - 1]
+    if (last !== undefined && a <= last[1] + 1) last[1] = Math.max(last[1], b)
+    else merged.push([a, b])
+  }
+  return merged
+}
+
+const makeSet = (test: (code: number) => boolean, set: unknown): CharSet => {
+  // The table calls the test once per small code: token-set predicates are
+  // assumed to be pure (see the design notes).
+  const table = new Uint8Array(TableSize)
+  for (let c = 0; c < TableSize; c++) table[c] = test(c) ? 1 : 0
+  return { test, table, ranges: rangesOf(set) }
+}
+
+/**
+ * The set as a `Token` parser sees it: string predicates receive strings on
+ * string input, everything receives numbers on byte input.
+ */
+export const tokenSet = (set: unknown, kind: 0 | 1): CharSet => {
+  const m = compileAny(set)
+  return makeSet(kind === 0 ? m.forString.code : m.forCode.code, set)
+}
+
+/** The set as a `Text` parser sees it: code points, for any encoding. */
+export const textSet = (set: unknown): CharSet => makeSet(compile(set, "string").code, set)
+
+/** Whether some accepted code point is at least `limit` (true when unknown). */
+export const reachesAbove = (s: CharSet, limit: number): boolean =>
+  s.ranges === undefined || s.ranges.some(([, b]) => b >= limit)
+
+/** The input kind a `Text` encoding reads: 0 for strings, 1 for bytes. */
+export const kindOfEncoding = (enc: Encoding<any>): 0 | 1 => {
+  let k = kinds.get(enc)
+  if (k === undefined) kinds.set(enc, k = typeof enc.encode("") === "string" ? 0 : 1)
+  return k
+}
+
+const kinds = new WeakMap<Encoding<any>, 0 | 1>()

package/src/internal/entries.ts

@@ -0,0 +1,51 @@
+/**
+ * Building objects as `Object.fromEntries` does, without its array of
+ * `[key, value]` pairs: the compilers use these when a parser maps a
+ * repetition of pairs with `Object.fromEntries` (see `Parser.fromEntries`).
+ */
+
+/** The built-in `Object.fromEntries`, which compiled parsers recognize. */
+export const fromEntries: <K extends PropertyKey, V>(entries: Iterable<readonly [K, V]>) => { [P in K]: V } =
+  Object.fromEntries as any
+
+const objectPrototype = Object.prototype
+const hasOwn = objectPrototype.hasOwnProperty
+
+/**
+ * Whether `o[key] = value` on a new plain object creates the same property
+ * as `Object.fromEntries` (a writable, enumerable, configurable data
+ * property): `key` is a string that `Object.prototype` has no property for,
+ * so there is no setter (`__proto__`) or read-only property to run into.
+ */
+export const isPlainKey = (key: unknown): key is string => typeof key === "string" && !hasOwn.call(objectPrototype, key)
+
+const define = (o: object, key: unknown, value: unknown): void => {
+  Object.defineProperty(o, key as PropertyKey, { value, writable: true, enumerable: true, configurable: true })
+}
+
+/**
+ * Adds an entry whose key is not plain to `o`, given the entries already
+ * deferred. Keys that are objects may run code when converted to property
+ * keys, which `Object.fromEntries` only does once every entry is parsed:
+ * from the first such key on, entries are deferred to {@link flushEntries}.
+ * Returns the deferred entries, if any.
+ */
+export const addEntry = (
+  o: object,
+  deferred: Array<unknown> | undefined,
+  key: unknown,
+  value: unknown
+): Array<unknown> | undefined => {
+  if (deferred === undefined && (typeof key !== "object" || key === null) && typeof key !== "function") {
+    define(o, key, value)
+    return undefined
+  }
+  if (deferred === undefined) deferred = []
+  deferred.push(key, value)
+  return deferred
+}
+
+/** Adds the deferred entries of {@link addEntry} to `o`, in order. */
+export const flushEntries = (o: object, deferred: ReadonlyArray<unknown>): void => {
+  for (let i = 0; i < deferred.length; i += 2) define(o, deferred[i], deferred[i + 1])
+}

package/src/internal/node.ts

@@ -0,0 +1,207 @@
+/**
+ * Descriptions of parsers, recorded next to their steps so that parsers can
+ * be compiled (see `docs/DESIGN.md`, "Compiled parsers").
+ *
+ * Nodes refer to their child parsers rather than to child nodes, so a
+ * child's own step is always at hand for fallback, and recursion through
+ * `Suspend` is just a reference.
+ */
+import type { Encoding } from "../Encoding.js"
+import type { ErrMode, Input, State } from "../Input.js"
+
+/** The parts of a parser the compiler needs. */
+export interface ParserLike {
+  readonly step: (state: State<any>) => unknown
+  readonly name: string | undefined
+}
+
+type P = ParserLike
+type Step = (state: State<any>) => unknown
+
+/** A repetition range, normalized: `max` may be `Infinity`. */
+interface Bounds {
+  readonly min: number
+  readonly max: number
+}
+
+/**
+ * For token nodes: `undefined` for the `Token`/`Ascii` parsers (code points
+ * of strings with lenient surrogates, bytes, or array elements), or the
+ * encoding of a `Text` parser.
+ */
+type Enc = Encoding<any> | undefined
+
+export type Node =
+  // Constants
+  | { readonly _tag: "Opaque"; readonly step: Step }
+  | { readonly _tag: "Succeed"; readonly value: unknown }
+  | { readonly _tag: "Sync"; readonly evaluate: () => unknown }
+  | { readonly _tag: "Fail"; readonly mode: ErrMode }
+  | { readonly _tag: "Eof" }
+  | { readonly _tag: "Location" }
+  | { readonly _tag: "Suspend"; readonly resolve: () => P }
+  // Tokens
+  | { readonly _tag: "Any"; readonly encoding: Enc }
+  | { readonly _tag: "Literal"; readonly literal: Input; readonly caseless: boolean; readonly encoding: Enc }
+  | { readonly _tag: "OneOf"; readonly set: unknown; readonly negate: boolean; readonly encoding: Enc }
+  | { readonly _tag: "Take"; readonly count: number; readonly encoding: Enc }
+  | ({ readonly _tag: "TakeWhile"; readonly set: unknown; readonly negate: boolean; readonly encoding: Enc } & Bounds)
+  | ({ readonly _tag: "TakeUntil"; readonly literal: Input; readonly encoding: Enc } & Bounds)
+  | { readonly _tag: "Rest"; readonly encoding: Enc }
+  | { readonly _tag: "RestLen" }
+  | { readonly _tag: "TillLineEnding"; readonly encoding: Enc }
+  | { readonly _tag: "Identifier"; readonly encoding: Enc }
+  | { readonly _tag: "Bom"; readonly encoding: Enc }
+  | {
+    readonly _tag: "Escaped"
+    readonly normal: P
+    /** The control character, as a code point (or a byte / unit for `Ascii`). */
+    readonly control: number
+    readonly escape: P
+    /** `takeEscaped`: return the raw slice instead of the unescaped text. */
+    readonly raw: boolean
+    readonly encoding: Enc
+  }
+  // Mapping
+  | { readonly _tag: "Map"; readonly self: P; readonly f: (a: any) => unknown }
+  | { readonly _tag: "As"; readonly self: P; readonly value: unknown }
+  | { readonly _tag: "Tap"; readonly self: P; readonly f: (a: any) => void }
+  | { readonly _tag: "TryMap"; readonly self: P; readonly f: (a: any) => unknown }
+  | { readonly _tag: "FilterMap"; readonly self: P; readonly f: (a: any) => unknown }
+  | { readonly _tag: "Filter"; readonly self: P; readonly predicate: (a: any) => boolean }
+  | { readonly _tag: "FlatMap"; readonly self: P; readonly f: (a: any) => P }
+  // Consumed input
+  | { readonly _tag: "Taken"; readonly self: P; readonly encoding: Enc }
+  | { readonly _tag: "WithTaken"; readonly self: P }
+  | { readonly _tag: "Span"; readonly self: P }
+  | { readonly _tag: "WithSpan"; readonly self: P }
+  // Errors
+  | { readonly _tag: "Mode"; readonly self: P; readonly from: ErrMode; readonly to: ErrMode }
+  | { readonly _tag: "Context"; readonly self: P; readonly label: string }
+  | { readonly _tag: "Expected"; readonly self: P; readonly description: string }
+  | { readonly _tag: "MapError"; readonly self: P }
+  | { readonly _tag: "OrFail"; readonly self: P }
+  // Control
+  | { readonly _tag: "Opt"; readonly self: P }
+  | { readonly _tag: "OrElseSucceed"; readonly self: P; readonly value: () => unknown }
+  | { readonly _tag: "Peek"; readonly self: P }
+  | { readonly _tag: "Not"; readonly self: P }
+  // Sequencing
+  | {
+    readonly _tag: "Seq"
+    readonly parsers: ReadonlyArray<P>
+    /** Which outputs are kept: all (a tuple), one index, or a pair of indices. */
+    readonly output: "tuple" | number | readonly [number, number]
+  }
+  | { readonly _tag: "Struct"; readonly keys: ReadonlyArray<string>; readonly parsers: ReadonlyArray<P> }
+  | { readonly _tag: "ZipWith"; readonly self: P; readonly that: P; readonly f: (a: any, b: any) => unknown }
+  // Choice
+  | { readonly _tag: "Alt"; readonly parsers: ReadonlyArray<P> }
+  | {
+    readonly _tag: "Dispatch"
+    readonly selector: P
+    /** A record of cases, when `dispatch` was given one. */
+    readonly cases: Readonly<Record<PropertyKey, P>> | undefined
+    readonly otherwise: P | undefined
+    readonly choose: (key: any) => P | undefined
+  }
+  // Repetition
+  | ({ readonly _tag: "Repeat"; readonly self: P } & Bounds)
+  | ({
+    readonly _tag: "Fold"
+    readonly self: P
+    readonly init: () => unknown
+    readonly f: (acc: any, a: any) => unknown
+    readonly name: string
+  } & Bounds)
+  | ({ readonly _tag: "RepeatTill"; readonly self: P; readonly end: P } & Bounds)
+  | ({ readonly _tag: "Separated"; readonly self: P; readonly separator: P } & Bounds)
+  | {
+    readonly _tag: "SeparatedFold"
+    readonly self: P
+    readonly separator: P
+    readonly op: (l: any, s: any, r: any) => unknown
+    readonly right: boolean
+  }
+  // Services, recovery, debugging
+  | { readonly _tag: "Service" }
+  | { readonly _tag: "Provide"; readonly self: P }
+  | { readonly _tag: "ResumeAfter"; readonly self: P; readonly recover: P }
+  | { readonly _tag: "RetryAfter"; readonly self: P; readonly recover: P }
+  | { readonly _tag: "Trace"; readonly self: P; readonly name: string }
+
+export type Tag = Node["_tag"]
+
+export type NodeOf<T extends Tag> = Extract<Node, { readonly _tag: T }>
+
+/** Where a parser's description is stored. */
+export const NodeId: unique symbol = Symbol.for("effect-parse/Parser/node")
+
+/** The description of a parser: `Opaque` when it has none. */
+export const nodeOf = (p: ParserLike): Node => (p as any)[NodeId] ?? { _tag: "Opaque", step: p.step }
+
+/** Normalizes a range, as `repeat`, `takeWhile`, ... do. */
+export const bounds = (range: number | { readonly min?: number | undefined; readonly max?: number | undefined } | undefined): Bounds => {
+  if (range === undefined) return { min: 0, max: Infinity }
+  if (typeof range === "number") return { min: range, max: range }
+  return { min: range.min ?? 0, max: range.max ?? Infinity }
+}
+
+/** The child parsers of a node, in evaluation order. */
+export const children = (node: Node): ReadonlyArray<P> => {
+  switch (node._tag) {
+    case "Escaped":
+      return [node.normal, node.escape]
+    case "Map":
+    case "As":
+    case "Tap":
+    case "TryMap":
+    case "FilterMap":
+    case "Filter":
+    case "FlatMap":
+    case "Taken":
+    case "WithTaken":
+    case "Span":
+    case "WithSpan":
+    case "Mode":
+    case "Context":
+    case "Expected":
+    case "MapError":
+    case "OrFail":
+    case "Opt":
+    case "OrElseSucceed":
+    case "Peek":
+    case "Not":
+    case "Repeat":
+    case "Fold":
+    case "Provide":
+    case "Trace":
+      return [node.self]
+    case "ResumeAfter":
+    case "RetryAfter":
+      return [node.self, node.recover]
+    case "Seq":
+    case "Struct":
+    case "Alt":
+      return node.parsers
+    case "ZipWith":
+      return [node.self, node.that]
+    case "Dispatch": {
+      const out: Array<P> = [node.selector]
+      if (node.cases !== undefined) {
+        for (const key of Reflect.ownKeys(node.cases)) out.push(node.cases[key as any]!)
+      }
+      if (node.otherwise !== undefined) out.push(node.otherwise)
+      return out
+    }
+    case "RepeatTill":
+      return [node.self, node.end]
+    case "Separated":
+    case "SeparatedFold":
+      return [node.self, node.separator]
+    case "Suspend":
+      return [node.resolve()]
+    default:
+      return []
+  }
+}