@thomasfosterau/effect-parse 0.3.0 → 0.4.0

package/src/Parser.ts

@@ -45,6 +45,7 @@ import type { ErrorKind } from "./ParseError.js"
 import * as ParseError from "./ParseError.js"
 import * as Registry from "./internal/compiler/registry.js"
 import * as Entries from "./internal/entries.js"
+import { Key } from "./internal/key.js"
 import * as Resume from "./internal/resume.js"
 import * as Run from "./internal/run.js"
 import * as Stack from "./internal/stack.js"
@@ -273,6 +274,8 @@ const isTwoParserArgs = (args: IArguments) => isParser(args[0]) && isParser(args
  * The step function must either return the parsed value, having advanced
  * `state.pos` past the consumed input, or return the result of one of
  * `state.backtrack`, `state.cut`, `state.incomplete`, ... to signal failure.
+ * A step that moves the cursor backwards (to retry, or after a lookahead)
+ * calls `state.rewind(offset)` rather than assigning `state.pos`.
  *
  * @example
  * ```ts
@@ -723,7 +726,7 @@ export const tryMap: {
         if (a === Fail) return Fail
         const r = f(a)
         if (Result.isSuccess(r)) return r.success
-        state.pos = start
+        state.rewind(start)
         return state.failCause("Backtrack", "Verify", r.failure)
       },
       self.name,
@@ -777,7 +780,7 @@ export const filterMap: {
         if (a === Fail) return Fail
         const o = f(a)
         if (Option.isSome(o)) return o.value
-        state.pos = start
+        state.rewind(start)
         return state.backtrack("Verify")
       },
       self.name,
@@ -808,7 +811,7 @@ export const filter: {
       const a = step(state)
       if (a === Fail) return Fail
       if (predicate(a)) return a
-      state.pos = start
+      state.rewind(start)
       return state.backtrack("Verify")
     },
     self.name,
@@ -852,7 +855,7 @@ export const filterOrFail: {
         const a = step(state)
         if (a === Fail) return Fail
         if (predicate(a)) return a
-        state.pos = start
+        state.rewind(start)
         return state.failCause("Backtrack", "Verify", orFailWith(a))
       },
       self.name,
@@ -1032,7 +1035,7 @@ export const complete = <A, E, R, I extends Input>(self: Parser<A, E, R, I>): Pa
       const start = state.pos
       const a = step(state)
       if (a === Fail && state.mode === "Incomplete") {
-        state.pos = start
+        state.rewind(start)
         return state.backtrack("Complete")
       }
       return a
@@ -1223,7 +1226,7 @@ export const orElseSucceed: {
 
 /** Rewinds after a recoverable failure, remembering it as a hint. */
 const recover = (state: State<any>, start: number): void => {
-  state.pos = start
+  state.rewind(start)
   const f = state.failure
   if (f.offset === start) {
     state.hintOffset = start
@@ -1297,7 +1300,7 @@ export const peek = <A, E, R, I extends Input>(self: Parser<A, E, R, I>): Parser
       if (a !== Fail) {
         // Failures inside a lookahead that succeeded are about input that was
         // never consumed, so they are not reported as the furthest failure
-        state.pos = start
+        state.rewind(start)
         state.hintOffset = hintOffset
         state.hintExpected = hintExpected
         state.deepest = deepest
@@ -1327,12 +1330,12 @@ export const not = <A, E, R, I extends Input>(self: Parser<A, E, R, I>): Parser<
       const a = step(state)
       if (a !== Fail) {
         // The failures inside `self` are not this parser's failure
-        state.pos = start
+        state.rewind(start)
         state.deepest = deepest
         return state.backtrack("Not")
       }
       if (state.mode !== "Backtrack") return Fail
-      state.pos = start
+      state.rewind(start)
       state.hintOffset = hintOffset
       state.hintExpected = hintExpected
       state.deepest = deepest
@@ -1807,7 +1810,7 @@ export const alt = <const Ps extends ReadonlyArray<Parser.Any>>(
           return a
         }
         if (state.mode !== "Backtrack") return Fail
-        state.pos = start
+        state.rewind(start)
         // The branch's failure, whose own ties (from a nested `alt`) can be taken over unmerged
         const head = state.failureHead
         if (failure === undefined || head.offset > failure.offset) {
@@ -1906,7 +1909,7 @@ export const dispatch: {
       if (key === Fail) return Fail
       const parser = choose(key)
       if (parser === undefined) {
-        state.pos = start
+        state.rewind(start)
         return state.backtrack("Dispatch")
       }
       return parser.step(state)
@@ -1961,7 +1964,7 @@ export const permutation = <const Ps extends ReadonlyArray<Parser.Any>>(
           break
         }
         if (state.mode !== "Backtrack") return Fail
-        state.pos = start
+        state.rewind(start)
         ties.add(state)
       }
       // Some parser is not done yet, and each one that fails is folded in
@@ -1995,7 +1998,7 @@ interface FoldCheckpoint<B> {
   readonly pos: number
 }
 
-const saveFold = <B>(state: State<any>, key: object, entry: number, acc: B, pos: number): void =>
+const saveFold = <B>(state: State<any>, key: Key, entry: number, acc: B, pos: number): void =>
   Resume.save(state, key, entry, { acc, count: 0, pos } satisfies FoldCheckpoint<B>)
 
 const foldImpl = <A, B, E, R, I extends Input>(
@@ -2010,7 +2013,7 @@ const foldImpl = <A, B, E, R, I extends Input>(
   const [min, max] = normalizeRange(range)
   const step = self.step
   const checkLoop = max === Infinity
-  const key = {}
+  const key = new Key()
   return described(
     (state) => {
       const entry = state.pos
@@ -2189,7 +2192,7 @@ export const repeatTill: {
             if (b !== Fail) return [acc, b]
             if (state.mode !== "Backtrack") return Fail
             endFailure = state.failure
-            state.pos = start
+            state.rewind(start)
             if (acc.length >= max) return Fail
           }
           const a = step(state)
@@ -2252,7 +2255,7 @@ export const separated: {
     const step = self.step
     const sepStep = separator.step
     const checkLoop = max === Infinity
-    const key = {}
+    const key = new Key()
     return described(
       (state) => {
         const entry = state.pos
@@ -2295,7 +2298,7 @@ export const separated: {
               Resume.save(state, key, entry, { acc, pos: start } satisfies SeparatedCheckpoint<A>)
             }
             if (state.mode !== "Backtrack" || acc.length < min) return Fail
-            state.pos = start
+            state.rewind(start)
             return Resume.done(state, key, entry, resumed !== undefined, acc)
           }
           acc.push(a)
@@ -2336,7 +2339,7 @@ export const separatedFoldl1 = <A, E, R, I extends Input, S, E2, R2, I2 extends
 ): Parser<A, E | E2, R | R2, I & I2> => {
   const step = self.step
   const sepStep = separator.step
-  const key = {}
+  const key = new Key()
   return described(
     (state) => {
       const entry = state.pos
@@ -2367,7 +2370,7 @@ export const separatedFoldl1 = <A, E, R, I extends Input, S, E2, R2, I2 extends
         if (a === Fail) {
           if (state.mode === "Incomplete" && state.resume !== undefined) saveFold(state, key, entry, acc, start)
           if (state.mode !== "Backtrack") return Fail
-          state.pos = start
+          state.rewind(start)
           return Resume.done(state, key, entry, resumed !== undefined, acc as A)
         }
         acc = op(acc, s, a)
@@ -2412,7 +2415,7 @@ export const separatedFoldr1 = <A, E, R, I extends Input, S, E2, R2, I2 extends
         const a = step(state)
         if (a === Fail) {
           if (state.mode !== "Backtrack") return Fail
-          state.pos = start
+          state.rewind(start)
           break
         }
         seps.push(s)
@@ -2584,7 +2587,7 @@ export const resumeAfter: {
         if (state.recovered === undefined || state.mode === "Incomplete") return Fail
         const failure = state.failure
         const mode = state.mode
-        state.pos = start
+        state.rewind(start)
         if (recoverStep(state) === Fail) return state.failWith(mode, failure)
         state.recovered.push(failure)
         state.syntax?.skipped(start, failure, mode)
@@ -2631,7 +2634,7 @@ export const retryAfter: {
           if (state.recovered === undefined || state.mode === "Incomplete") return Fail
           const failure = state.failure
           const mode = state.mode
-          state.pos = start
+          state.rewind(start)
           if (recoverStep(state) === Fail || state.pos === start) return state.failWith(mode, failure)
           state.recovered.push(failure)
           state.syntax?.skipped(start, failure, mode)
@@ -2708,7 +2711,7 @@ const preview = (state: State<any>, offset: number): string => {
  */
 export const consoleTracer = (event: TraceEvent): void => {
   const indent = "  ".repeat(event.depth)
-  if (event._tag === "Enter") {
+  if (Predicate.isTagged(event, "Enter")) {
     console.log(`${indent}> ${event.name} @${event.offset} | ${preview(event.state, event.offset)}`)
   } else {
     const consumed = (event.endOffset ?? event.offset) - event.offset

package/src/ParserStream.ts

@@ -36,6 +36,7 @@ import * as Registry from "./internal/compiler/registry.js"
 import type * as Resume from "./internal/resume.js"
 import * as Run from "./internal/run.js"
 import * as Stack from "./internal/stack.js"
+import { isEmptyUnits } from "./internal/text.js"
 import type { Parser } from "./Parser.js"
 
 /**
@@ -49,7 +50,7 @@ export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
 const concat = <I extends Chunkable>(a: I, b: I): I => {
   if (Predicate.isString(a)) return (a + (b as string)) as I
   if (a instanceof Uint8Array) {
-    if (a.length === 0) return b
+    if (a.byteLength === 0) return b
     const out = new Uint8Array(a.length + (b as Uint8Array).length)
     out.set(a)
     out.set(b as Uint8Array, a.length)
@@ -174,6 +175,7 @@ class Incremental<A, E, I extends Chunkable> {
   /** The input length at the last attempt, and where that attempt ran out of input. */
   private attempted = 0
   private restart = 0
+  private pushed = 0
   /** Set once an attempt settles: a success (with its end offset) or a definite failure. */
   private settled: { readonly value: A; readonly pos: number } | "failed" | undefined
 
@@ -221,9 +223,14 @@ class Incremental<A, E, I extends Chunkable> {
   }
 
   push(chunk: I): void {
-    if (chunk.length === 0) return
+    if (isEmptyUnits(chunk)) return
     this.append(chunk)
     if (this.settled !== undefined) return
+    // The first attempt waits for a second chunk: an input that arrives in
+    // one chunk (most small responses) is parsed once, by `finish`, instead
+    // of once in partial mode and again in complete mode, since a repetition
+    // that completes without resuming keeps no checkpoint
+    if (++this.pushed === 1) return
     // An attempt re-reads what the last one could not finish, from `restart`;
     // waiting until at least half as much new input has arrived keeps the
     // total work linear in the input. Strings are also copied when they are
@@ -234,9 +241,12 @@ class Incremental<A, E, I extends Chunkable> {
     this.attempt()
   }
 
-  /** The step attempts run: the parser's resumable compiled code if it was compiled, else the interpreter. */
-  private step(state: State<I>): unknown {
-    return Stack.run(Registry.stepFor(this.parser, state, true) ?? this.parser.step, state)
+  /**
+   * The step attempts run: the parser's compiled code if it was compiled
+   * (the resumable build, unless `resumable` is false), else the interpreter.
+   */
+  private step(state: State<I>, resumable = true): unknown {
+    return Stack.run(Registry.stepFor(this.parser, state, resumable) ?? this.parser.step, state)
   }
 
   private attempt(): void {
@@ -258,10 +268,12 @@ class Incremental<A, E, I extends Chunkable> {
     if (settled !== undefined && settled !== "failed") {
       if (settled.pos === this.length) return Effect.succeed(settled.value)
     } else if (settled === undefined) {
-      // The last attempt, in complete mode, resumes from the checkpoints too
+      // The last attempt, in complete mode, resumes from the checkpoints too;
+      // with no attempt before it, it is an ordinary parse
       const state = new State(this.input(), { context: this.context })
-      state.resume = this.resume
-      const a = this.step(state)
+      const resumable = this.attempted > 0
+      if (resumable) state.resume = this.resume
+      const a = this.step(state, resumable)
       if (a !== Fail && state.pos === state.end) return Effect.succeed(a as A)
     }
     // The error comes from an ordinary parse of the whole input, so it is the

package/src/Syntax.ts

@@ -46,17 +46,19 @@
  *
  * @since 0.2.0
  */
+import * as Str from "effect/String"
 import * as Predicate from "effect/Predicate"
 import type * as Context from "effect/Context"
 import { dual } from "effect/Function"
 import * as Option from "effect/Option"
 import type { Input, Slice, Source } from "./Input.js"
-import { Fail, fromSource } from "./Input.js"
+import { Fail, fromSource, State } from "./Input.js"
 import * as ParseError from "./ParseError.js"
 import { nodeOf } from "./internal/node.js"
+import { Key } from "./internal/key.js"
 import * as Stack from "./internal/stack.js"
 import type { ErrorInfo, Green, Previous, Segment } from "./internal/syntax.js"
-import { absolute, errorKind, SyntaxState } from "./internal/syntax.js"
+import { absolute, errorKind, SyntaxBuilder } from "./internal/syntax.js"
 import type { Parser } from "./Parser.js"
 import { described, isParser } from "./Parser.js"
 
@@ -67,7 +69,7 @@ import { described, isParser } from "./Parser.js"
 const syntaxNode = (self: Parser.Any, kind: string, named: boolean): Parser.Any => {
   const step = self.step
   // Identifies the nodes this parser builds, so a reparse can reuse them
-  const key = {}
+  const key = new Key()
   return described<unknown, unknown, unknown, Input>(
     (state) => {
       const builder = state.syntax
@@ -133,7 +135,7 @@ export const token: {
 
 const defaultTokenKind = (self: Parser.Any): string => {
   const n = nodeOf(self)
-  if (n._tag === "Literal" && Predicate.isString(n.literal) && !n.caseless) return n.literal
+  if (Predicate.isTagged(n, "Literal") && Predicate.isString(n.literal) && !n.caseless) return n.literal
   return self.name ?? "token"
 }
 
@@ -577,7 +579,7 @@ export class SyntaxNode<I extends Input = string> {
     let out = ""
     const visit = (green: Green, field: string | undefined) => {
       if (green.named || green === this.green) {
-        if (out.length > 0) out += " "
+        if (Str.isNonEmpty(out)) out += " "
         if (field !== undefined) out += `${field}: `
         out += `(${green.kind}`
         for (let i = 0; i < green.children.length; i++) visit(green.children[i], green.fields?.[i])
@@ -708,7 +710,8 @@ export class TreeCursor<I extends Input = string> {
 const expectedEof = ["end of input"]
 
 const run = <A, E, I extends Input>(config: Config, input: I, previous: Previous | undefined): Tree<A, E, I> => {
-  const state = new SyntaxState<I>(input, { context: config.context }, previous)
+  const state = new State<I>(input, { context: config.context })
+  const builder = new SyntaxBuilder(state, previous)
   const a = Stack.run(config.inner.step, state)
   // Text parsers record the encoding when they fail, which they may have done inside reused nodes
   if (state.encoding === undefined && previous !== undefined) state.encoding = previous.encoding
@@ -722,8 +725,8 @@ const run = <A, E, I extends Input>(config: Config, input: I, previous: Previous
   } else {
     value = Option.some(a as A)
   }
-  const green = state.finish(config.kind, Option.getOrUndefined(value), fatal, state.pos)
-  return new Tree<A, E, I>(input, value, state.reused, green, config, state.encoding)
+  const green = builder.finish(config.kind, Option.getOrUndefined(value), fatal, state.pos)
+  return new Tree<A, E, I>(input, value, builder.reused, green, config, state.encoding)
 }
 
 /**
@@ -756,7 +759,7 @@ export const parse: {
   options?: ParseOptions & { readonly context?: Context.Context<R> }
 ): Tree<A, E, I> => {
   const n = nodeOf(grammar)
-  const isRoot = n._tag === "Syntax" && n.kind !== undefined && n.named
+  const isRoot = Predicate.isTagged(n, "Syntax") && n.kind !== undefined && n.named
   const config: Config = {
     kind: isRoot ? n.kind : (options?.kind ?? "document"),
     inner: isRoot ? (n.self as Parser.Any) : grammar,