@thomasfosterau/effect-parse 0.2.0 → 0.4.0

package/src/internal/syntax.ts

@@ -14,23 +14,32 @@
  * completes and adopts them. Nodes built by an attempt that is later
  * abandoned must not end up in the tree. Every combinator that abandons an
  * attempt moves the cursor back to (or before) where the attempt started,
- * and every node has a positive width, so the nodes of an abandoned attempt
- * are exactly those that end after the position the cursor moves back to:
- * the `pos` setter of {@link SyntaxState} drops them. This also covers
- * lookahead (`peek`, `not`) and custom parsers written with `Parser.make`.
+ * with `State.rewind`, and every node has a positive width, so the nodes of
+ * an abandoned attempt are exactly those that end after the position the
+ * cursor moves back to: {@link SyntaxBuilder} drops them when told of the rewind. This also
+ * covers lookahead (`peek`, `not`) and custom parsers written with
+ * `Parser.make`, which rewind the same way.
+ *
+ * `pos` stays a plain field, not an accessor that would watch every move: a
+ * `pos` accessor on this subclass made every parser's cursor accesses
+ * polymorphic, slowing ordinary parses by about a fifth in any program that
+ * also built a syntax tree, and the syntax parse itself about twofold.
  *
  * For incremental parsing, every node records how far its parse looked: the
  * furthest position the cursor reached and the furthest failure recorded
- * while it was being parsed (its *lookahead*). A node from the previous tree
+ * while it was being parsed (its *lookahead*). The cursor only moves forward
+ * between rewinds, so the furthest position is where it was at a rewind, or
+ * where it is now. A node from the previous tree
  * is reused at a position when it was built by the same `Syntax.node`, and
  * the input it looked at, plus a margin for the tokens that read a little
  * past where they stop, is unchanged.
  */
+import * as Arr from "effect/Array"
 import * as Predicate from "effect/Predicate"
-import type * as Context from "effect/Context"
 import type { Encoding } from "../Encoding.js"
-import type { ErrMode, Input, Source } from "../Input.js"
+import type { ErrMode, Input } from "../Input.js"
 import { Fail, Failure, State } from "../Input.js"
+import type { Key } from "./key.js"
 
 /** What a failure inside an `ERROR` node was. Offsets are relative to the node. */
 export interface ErrorInfo {
@@ -44,7 +53,7 @@ export class Green {
     readonly kind: string,
     readonly named: boolean,
     /** The `Syntax.node` that built it; `undefined` for roots and errors, which are never reused alone. */
-    readonly key: object | undefined,
+    readonly key: Key | undefined,
     readonly width: number,
     /** How far its parse looked, relative to its start (at least `width`). */
     readonly lookahead: number,
@@ -103,7 +112,7 @@ export const absolute = (f: Failure, start: number): Failure => rebase(f, -start
 const build = (
   kind: string,
   named: boolean,
-  key: object | undefined,
+  key: Key | undefined,
   start: number,
   end: number,
   lookahead: number,
@@ -153,7 +162,7 @@ export interface Previous {
 
 /** The hooks that `Syntax` combinators and `Parser.resumeAfter`/`retryAfter` call. */
 export interface Builder {
-  node(state: State<any>, step: (state: State<any>) => unknown, key: object, kind: string, named: boolean): unknown
+  node(state: State<any>, step: (state: State<any>) => unknown, key: Key, kind: string, named: boolean): unknown
   field(state: State<any>, step: (state: State<any>) => unknown, name: string): unknown
   /** After error recovery: `[start, pos)` was skipped after `failure`. */
   skipped(start: number, failure: Failure, mode: ErrMode): void
@@ -162,74 +171,61 @@ export interface Builder {
    * where they leave the cursor or record a failure.
    */
   read(end: number): void
+  /** A failure or hint was recorded at `offset`. */
+  reach(offset: number): void
+  /** The cursor moved back from `from` to `to` (see `State.rewind`). */
+  rewound(from: number, to: number): void
 }
 
-/** The parse state of a `Syntax` parse. */
-export class SyntaxState<I extends Input = Input> extends State<I> implements Builder {
-  // Declared, not defined: `State`'s constructor sets `pos` through the
-  // accessor below before these could be initialized
-  declare cursor: number
-  /** The furthest position the cursor reached, or failure recorded, since the innermost open node started. */
-  declare highWater: number
+/**
+ * The tree builder of a `Syntax` parse, reached through `state.syntax`. It
+ * is not a `State` subclass: one class of state keeps every parser's cursor
+ * and failure accesses monomorphic, whether or not the program also builds
+ * syntax trees. `State` calls {@link reach} and {@link rewound} itself.
+ */
+export class SyntaxBuilder implements Builder {
+  /**
+   * The furthest position the cursor reached before a rewind, or failure
+   * recorded, since the innermost open node started; {@link looked} adds
+   * where the cursor is now.
+   */
+  highWater: number
   /** Completed nodes waiting for their parent, in input order. */
-  declare stack: Array<Child>
-  declare previous: Previous | undefined
+  readonly stack: Array<Child> = []
   /** How many nodes were reused from the previous tree. */
-  declare reused: number
+  reused = 0
   /** Where {@link find} found the last node it found. */
-  declare lastParent: Green | undefined
-  declare lastParentStart: number
-
-  static {
-    Object.defineProperty(this.prototype, "pos", {
-      get(this: SyntaxState) {
-        return this.cursor
-      },
-      set(this: SyntaxState, p: number) {
-        if (p < this.cursor) {
-          // An attempt was abandoned (or a lookahead finished): drop its nodes
-          const stack = this.stack
-          while (stack.length > 0 && stack[stack.length - 1].end > p) stack.pop()
-        }
-        if (p > this.highWater) this.highWater = p
-        this.cursor = p
-      },
-      configurable: true,
-      enumerable: true
-    })
-  }
+  lastParent: Green | undefined = undefined
+  lastParentStart = 0
 
   constructor(
-    source: I | Source<I>,
-    options: {
-      readonly context?: Context.Context<never> | undefined
-    },
-    previous: Previous | undefined
+    readonly state: State<any>,
+    readonly previous: Previous | undefined
   ) {
-    super(source, { context: options.context })
-    this.highWater = this.cursor
-    this.stack = []
-    this.previous = previous
-    this.reused = 0
-    this.lastParent = undefined
-    this.lastParentStart = 0
-    this.syntax = this
+    this.highWater = state.pos
+    state.syntax = this
     // Error recovery is always on: like Tree-sitter, a syntax parse always produces a tree
-    this.recovered = []
+    state.recovered = []
   }
 
-  override failWith(mode: ErrMode, failure: Failure): Fail {
-    if (failure.offset > this.highWater) this.highWater = failure.offset
-    return super.failWith(mode, failure)
+  reach(offset: number): void {
+    if (offset > this.highWater) this.highWater = offset
   }
 
-  override setHint(offset: number, expected: ReadonlyArray<string>): void {
-    if (offset > this.highWater) this.highWater = offset
-    super.setHint(offset, expected)
+  rewound(from: number, to: number): void {
+    if (from > this.highWater) this.highWater = from
+    // An attempt was abandoned (or a lookahead finished): drop its nodes
+    const stack = this.stack
+    while (Arr.isArrayNonEmpty(stack) && Arr.lastNonEmpty(stack).end > to) stack.pop()
+  }
+
+  /** How far the parse looked: {@link highWater}, or the cursor if it is further. */
+  private looked(): number {
+    return this.state.pos > this.highWater ? this.state.pos : this.highWater
   }
 
-  node(state: State<any>, step: (state: State<any>) => unknown, key: object, kind: string, named: boolean): unknown {
-    const start = this.cursor
+  node(state: State<any>, step: (state: State<any>) => unknown, key: Key, kind: string, named: boolean): unknown {
+    const start = this.state.pos
     const stack = this.stack
     if (this.previous !== undefined) {
       const green = this.reusable(this.previous, key, start)
@@ -242,7 +238,7 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
         const d = green.diagnostics
         if (d !== undefined) {
           if (d.deepest !== undefined) this.deepen(start === d.base ? d.deepest : rebase(d.deepest, d.base - start))
-          if (d.hintExpected !== undefined) this.setHint(start + d.hintOffset, d.hintExpected)
+          if (d.hintExpected !== undefined) this.state.setHint(start + d.hintOffset, d.hintExpected)
         }
         stack.push(new Child(green, start, end, undefined))
         return green.value
@@ -250,34 +246,34 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
     }
     const mark = stack.length
     const outer = this.highWater
-    const outerDeepest = this.deepest
-    const hintOffset = this.hintOffset
-    const hintExpected = this.hintExpected
+    const outerDeepest = this.state.deepest
+    const hintOffset = this.state.hintOffset
+    const hintExpected = this.state.hintExpected
     this.highWater = start
     // Collect the furthest failure of this node alone, then merge it as `failWith` would have
-    this.deepest = undefined
+    this.state.deepest = undefined
     const a = step(state)
-    const looked = this.highWater
-    if (outer > looked) this.highWater = outer
-    const deepest = this.deepest as Failure | undefined
-    this.deepest = outerDeepest
+    const looked = this.looked()
+    this.highWater = outer > looked ? outer : looked
+    const deepest = this.state.deepest as Failure | undefined
+    this.state.deepest = outerDeepest
     if (deepest !== undefined) this.deepen(deepest)
     if (a === Fail) {
       if (stack.length > mark) stack.length = mark
       return a
     }
-    const end = this.cursor
+    const end = this.state.pos
     // Like Tree-sitter's rules, nodes are never empty: an empty match adds nothing to the tree
     if (end === start) return a
-    const hinted = this.hintOffset !== hintOffset || this.hintExpected !== hintExpected
+    const hinted = this.state.hintOffset !== hintOffset || this.state.hintExpected !== hintExpected
     const diagnostics: Diagnostics | undefined =
       deepest === undefined && !hinted
         ? undefined
         : {
             deepest,
             base: start,
-            hintOffset: this.hintOffset - start,
-            hintExpected: hinted ? this.hintExpected : undefined
+            hintOffset: this.state.hintOffset - start,
+            hintExpected: hinted ? this.state.hintExpected : undefined
           }
     const entries = stack.splice(mark)
     stack.push(
@@ -293,7 +289,7 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
 
   /** Records `failure` as the furthest failure if it is, as `failWith` does. */
   private deepen(failure: Failure): void {
-    if (this.deepest === undefined || failure.offset >= this.deepest.offset) this.deepest = failure
+    if (this.state.deepest === undefined || failure.offset >= this.state.deepest.offset) this.state.deepest = failure
   }
 
   field(state: State<any>, step: (state: State<any>) => unknown, name: string): unknown {
@@ -308,8 +304,8 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
     const stack = this.stack
     let i = stack.length
     while (i > 0 && stack[i - 1].start >= start && stack[i - 1].end > start) i--
-    const end = this.cursor
-    const green = build(errorKind, true, undefined, start, end, this.highWater, stack.splice(i), undefined, {
+    const end = this.state.pos
+    const green = build(errorKind, true, undefined, start, end, this.looked(), stack.splice(i), undefined, {
       failure: rebase(failure, start),
       mode
     })
@@ -323,9 +319,9 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
    */
   finish(kind: string, value: unknown, fatal: ErrorInfo | undefined, errorStart: number): Green {
     const entries = this.stack.splice(0)
-    const end = this.end
+    const end = this.state.end
     if (fatal !== undefined) {
-      const last = entries.length > 0 ? entries[entries.length - 1].end : 0
+      const last = Arr.isArrayNonEmpty(entries) ? Arr.lastNonEmpty(entries).end : 0
       const start = Math.max(Math.min(errorStart, end), last)
       const error = build(errorKind, true, undefined, start, end, end, [], undefined, {
         failure: rebase(fatal.failure, start),
@@ -333,7 +329,7 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
       })
       entries.push(new Child(error, start, end, undefined))
     }
-    return build(kind, true, undefined, 0, end, Math.max(end, this.highWater), entries, value, undefined)
+    return build(kind, true, undefined, 0, end, Math.max(end, this.looked()), entries, value, undefined)
   }
 
   /**
@@ -342,7 +338,7 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
    * so the search starts from the parent of the last one found when
    * `offset` lies inside it: no ancestor of that parent can start there.
    */
-  private find(root: Green, offset: number, key: object): Green | undefined {
+  private find(root: Green, offset: number, key: Key): Green | undefined {
     let node = root
     let start = 0
     const last = this.lastParent
@@ -380,7 +376,7 @@ export class SyntaxState<I extends Input = Input> extends State<I> implements Bu
   }
 
   /** The node of the previous tree built by `key` at `pos` (in the new input), if it can be reused. */
-  private reusable(previous: Previous, key: object, pos: number): Green | undefined {
+  private reusable(previous: Previous, key: Key, pos: number): Green | undefined {
     const segments = previous.segments
     // The segment containing `pos`
     let lo = 0

package/src/internal/text.ts

@@ -1,3 +1,6 @@
+import * as Arr from "effect/Array"
+import * as Predicate from "effect/Predicate"
+import * as Str from "effect/String"
 import type { Encoding } from "../Encoding.js"
 import type { Fail, State } from "../Input.js"
 
@@ -22,6 +25,17 @@ export const End = -3
  */
 const FastLimit: unique symbol = /*#__PURE__*/ Symbol.for("effect-parse/Encoding/FastLimit")
 
+/**
+ * Whether an input, a literal or a chunk has no units (characters, bytes or
+ * tokens).
+ */
+export const isEmptyUnits = (units: string | Uint8Array | ReadonlyArray<unknown>): boolean =>
+  Predicate.isString(units)
+    ? Str.isEmpty(units)
+    : units instanceof Uint8Array
+      ? units.byteLength === 0
+      : Arr.isReadonlyArrayEmpty(units)
+
 /** @internal */
 export const fastLimit = (enc: Encoding<any>): number =>
   (enc as any)[FastLimit] ?? (enc.asciiCompatible && enc.unit === 1 ? 0x80 : 0)
@@ -115,6 +129,7 @@ const decodeRange = (decoder: TextDecoder, b: Uint8Array, start: number, end: nu
 const utf8Encoder = /*#__PURE__*/ new TextEncoder()
 
 /** UTF-8 bytes. */
+// SAFETY: an `Encoding` with the internal `FastLimit` key, which its type doesn't declare
 export const utf8: Encoding<Uint8Array> = {
   name: "UTF-8",
   unit: 1,
@@ -137,13 +152,17 @@ export const utf8: Encoding<Uint8Array> = {
       }
     }
     return out
-  }
-}
+  },
+  // Every built-in encoding has the same keys in the same order, so code
+  // reading them sees one object shape
+  [FastLimit]: 0x80
+} as Encoding<Uint8Array>
 
 const makeUtf16 = (littleEndian: boolean): Encoding<Uint8Array> => {
   const lo = littleEndian ? 0 : 1
   const hi = 1 - lo
   let decoder: TextDecoder | undefined
+  // SAFETY: an `Encoding` with the internal `FastLimit` key, which its type doesn't declare
   return {
     name: littleEndian ? "UTF-16LE" : "UTF-16BE",
     unit: 2,
@@ -176,8 +195,9 @@ const makeUtf16 = (littleEndian: boolean): Encoding<Uint8Array> => {
       }
       decoder ??= new TextDecoder(littleEndian ? "utf-16le" : "utf-16be", { ignoreBOM: true })
       return decodeRange(decoder, b, start, end)
-    }
-  }
+    },
+    [FastLimit]: 0
+  } as Encoding<Uint8Array>
 }
 
 /** UTF-16 bytes, little endian. */

package/src/internal/ties.ts

@@ -0,0 +1,83 @@
+/**
+ * Tied failures, merged only when read.
+ *
+ * A combinator that tries several parsers at the same position (an
+ * alternation, a permutation round, an operator table) reports the merge
+ * of their failures, left to right, as `mergeFailures` would. Most of the
+ * time one of the later attempts succeeds and the merge is thrown away, so
+ * {@link TiedFailures} keeps the last two failures tied with the running
+ * one unmerged, and {@link TiedFailures.fail} hands them to the state with
+ * `State.failTies`, which merges them on first read (#38, #44).
+ *
+ * `Parser.alt` does the same fold in locals, since it is the hottest
+ * combinator and cannot afford an accumulator per call.
+ */
+import type { Failure, Fail, Input, State } from "../Input.js"
+import { mergeExpected, mergeTiedFailures } from "../Input.js"
+
+/** The failures of a round of attempts, folded lazily. */
+export class TiedFailures {
+  /** The running failure, before the pending ties are merged into it. */
+  failure: Failure | undefined = undefined
+  /** The oldest pending tie, at `failure`'s offset. */
+  tie1: Failure | undefined = undefined
+  /** The newest pending tie, at `failure`'s offset. */
+  tie2: Failure | undefined = undefined
+
+  /** Forgets every failure, for a new round. */
+  clear(): void {
+    this.failure = undefined
+    this.tie1 = undefined
+    this.tie2 = undefined
+  }
+
+  /**
+   * Folds in the failure the latest attempt left on `state`, taking over
+   * its own unmerged ties when it is the furthest.
+   */
+  add<I extends Input>(state: State<I>): void {
+    const head = state.failureHead
+    const running = this.failure
+    if (running === undefined || head.offset > running.offset) {
+      this.failure = head
+      this.tie1 = state.failureTie1
+      this.tie2 = state.failureTie2
+      return
+    }
+    if (head.offset < running.offset) return
+    const failure = state.failure
+    const tie1 = this.tie1
+    if (tie1 === undefined) this.tie1 = failure
+    else if (this.tie2 === undefined) this.tie2 = failure
+    else {
+      // A third tie: fold the oldest pending one in, keeping the order
+      this.failure = mergeTiedFailures(running, tie1, mergeExpected(running.expected, tie1.expected))
+      this.tie1 = this.tie2
+      this.tie2 = failure
+    }
+  }
+
+  /**
+   * Fails `state` with the folded failure (recoverably), or with its
+   * current failure when nothing was folded.
+   */
+  fail<I extends Input>(state: State<I>): Fail {
+    const failure = this.failure
+    if (failure === undefined) return state.failWith("Backtrack", state.failure)
+    const tie1 = this.tie1
+    return tie1 === undefined ? state.failWith("Backtrack", failure) : state.failTies(failure, tie1, this.tie2)
+  }
+
+  /**
+   * Records the folded expectations as a hint when the folded failure lies
+   * where `state` stopped (an attempt that fails without consuming input).
+   */
+  hint<I extends Input>(state: State<I>): void {
+    const failure = this.failure
+    if (failure === undefined || failure.offset !== state.pos) return
+    let expected = failure.expected
+    if (this.tie1 !== undefined) expected = mergeExpected(expected, this.tie1.expected)
+    if (this.tie2 !== undefined) expected = mergeExpected(expected, this.tie2.expected)
+    state.setHint(failure.offset, expected)
+  }
+}

package/src/internal/tokens.ts

@@ -1,6 +1,7 @@
 // fallow-ignore-file duplicate-export -- `End` and `scan` here are the token-set versions of the text
 // helpers in ./text.ts; both are internal modules, imported by path, never re-exported from one barrel.
 import * as Predicate from "effect/Predicate"
+import * as Result from "effect/Result"
 import type { Input, State } from "../Input.js"
 import { charOf, firstCodePoint } from "./text.js"
 
@@ -160,11 +161,10 @@ const compileCodeSafe = (set: unknown, kind: "string" | "code"): ((code: number)
       return false
     }
   }
-  try {
-    return compileCode(set)
-  } catch {
-    return () => false
-  }
+  return Result.getOrElse(
+    Result.try(() => compileCode(set)),
+    (): ((code: number) => boolean) => () => false
+  )
 }
 
 /**
@@ -276,17 +276,18 @@ export const nextIf = (state: State<Input>, m: AnyMatcher, negate: boolean): unk
   }
 }
 
+/** How far a {@link scan} got. */
+export interface Scanned extends Advanced {
+  /** Whether the end of input was hit before `max` tokens. */
+  readonly hitEnd: boolean
+}
+
 /**
  * Scans forward while tokens satisfy the matcher (xor `negate`), up to `max`
  * tokens. Returns the offset reached, the number of tokens matched, and
  * whether the end of input was hit before `max` tokens.
  */
-export const scan = (
-  state: State<Input>,
-  m: AnyMatcher,
-  negate: boolean,
-  max: number
-): { readonly offset: number; readonly count: number; readonly hitEnd: boolean } => {
+export const scan = (state: State<Input>, m: AnyMatcher, negate: boolean, max: number): Scanned => {
   let i = state.pos
   const end = state.end
   let count = 0
@@ -335,8 +336,14 @@ export const scan = (
   }
 }
 
+/** The offset reached and the number of tokens passed. */
+export interface Advanced {
+  readonly offset: number
+  readonly count: number
+}
+
 /** Advances `n` tokens from `start`, returning the offset reached and the count. */
-export const advance = (state: State<Input>, start: number, n: number): { offset: number; count: number } => {
+export const advance = (state: State<Input>, start: number, n: number): Advanced => {
   if (state.inputKind !== 0) {
     const offset = Math.min(state.end, start + n)
     return { offset, count: offset - start }
@@ -379,7 +386,7 @@ export const indexOf = (state: State<Input>, needle: Input, from: number): numbe
     case 1: {
       const hay = state.input as Uint8Array
       const n = needle as Uint8Array
-      if (n.length === 0) return from
+      if (n.byteLength === 0) return from
       const first = n[0]
       let i = hay.indexOf(first, from)
       while (i !== -1 && i <= last) {