@thomasfosterau/effect-parse 0.3.0 → 0.4.0

package/src/internal/compiler/jit.ts

@@ -16,26 +16,29 @@
  * the interpreter for what they can't compile) on the shared state, with
  * `state.pos` kept in sync.
  */
+import * as Arr from "effect/Array"
 import * as Predicate from "effect/Predicate"
 import * as Option from "effect/Option"
+import * as Result from "effect/Result"
 import type { Encoding } from "../../Encoding.js"
 import type { State } from "../../Input.js"
 import { Fail } from "../../Input.js"
 import type { Node, NodeOf, ParserLike } from "../node.js"
 import { addEntry, flushEntries, fromEntries, isPlainKey } from "../entries.js"
-import { children, nodeOf } from "../node.js"
+import { children, nodeOf, scansFloat } from "../node.js"
 import { floatScan, floatTexts, scanFloatBytes, scanFloatString, toFloat } from "../numbers.js"
 import { compileAny } from "../tokens.js"
+import { Key } from "../key.js"
 import * as Resume from "../resume.js"
 import { charOf, fastLimit, latin1, string, utf16be, utf16le, utf8, validate } from "../text.js"
-import type { ClosureCompiler, Fallback, Step } from "./closure.js"
+import type { ClosureCompiler, CompiledRoot, Fallback, Step } from "./closure.js"
 import { backend as closureBackend, createClosureCompiler } from "./closure.js"
 import { buildsEntries, pairOf } from "./entries.js"
 import type { Backend } from "./registry.js"
 import type { First } from "./first.js"
 import { planAlt, Strict } from "./first.js"
 import type { CharSet } from "./sets.js"
-import { kindOfEncoding, reachesAbove, TableSize, textSet, tokenSet } from "./sets.js"
+import { acceptTable, kindOfEncoding, reachesAbove, TableSize, textSet, tokenSet } from "./sets.js"
 
 /** Helpers available to generated code as `R`. */
 const runtime = {
@@ -43,7 +46,7 @@ const runtime = {
   save: Resume.save,
   Done: Resume.Done,
   /** Records the result of a resumed repetition (see `Resume.done`). */
-  done: (state: State<any>, key: object, entry: number, value: unknown, pos: number): void =>
+  done: (state: State<any>, key: Key, entry: number, value: unknown, pos: number): void =>
     Resume.save(state, key, entry, new Resume.Done(value, pos)),
   some: Option.some,
   none: Option.none,
@@ -77,11 +80,11 @@ let supported = false
 const jitSupported = (): boolean => {
   if (checked !== globalThis.Function) {
     checked = globalThis.Function
-    try {
-      supported = checked("return true")() === true
-    } catch {
-      supported = false
-    }
+    const fn = checked
+    supported = Result.getOrElse(
+      Result.try(() => fn("return true")() === true),
+      () => false
+    )
   }
   return supported
 }
@@ -141,7 +144,7 @@ const runsOf = (table: Uint8Array): Array<[number, number]> => {
 
 /** A JavaScript expression testing `v` against ranges, or `undefined` if too long. */
 const rangeTest = (runs: ReadonlyArray<readonly [number, number]>, v: string): string | undefined => {
-  if (runs.length === 0) return "false"
+  if (Arr.isReadonlyArrayEmpty(runs)) return "false"
   if (runs.length > 4) return undefined
   return runs.map(([a, b]) => (a === b ? `${v}===${a}` : `(${v}>=${a}&&${v}<=${b})`)).join("||")
 }
@@ -172,8 +175,8 @@ const runCost = ([a, b]: readonly [number, number], size: number): number =>
 const tableTest = (table: Uint8Array, v: string, size: number): string | undefined => {
   const accept = runsWhere(table, size, 1)
   const reject = runsWhere(table, size, 0)
-  if (accept.length === 0) return "false"
-  if (reject.length === 0) return "true"
+  if (Arr.isReadonlyArrayEmpty(accept)) return "false"
+  if (Arr.isReadonlyArrayEmpty(reject)) return "true"
   const cost = (runs: ReadonlyArray<readonly [number, number]>) => runs.reduce((n, r) => n + runCost(r, size), 0)
   const acceptCost = cost(accept)
   const rejectCost = cost(reject)
@@ -309,7 +312,7 @@ class Emitter {
     const done = this.label()
     const body = this.inline(p, used, out, `break ${done};`, fn, 0)
     this.functions.push(
-      `function ${name}(p){const I=$I,E=$E;${fn.vars.length > 0 ? `let ${fn.vars.join(",")};` : ""}` +
+      `function ${name}(p){const I=$I,E=$E;${Arr.isArrayNonEmpty(fn.vars) ? `let ${fn.vars.join(",")};` : ""}` +
         `${done}:{${body}P=p;return ${out ?? "void 0"}}return F}`
     )
     return name
@@ -321,7 +324,7 @@ class Emitter {
    */
   gen(p: ParserLike, used: boolean, out: string | null, fail: string, fn: Fn, depth: number): string {
     const node = nodeOf(p)
-    if (node._tag === "Suspend") return this.call(this.rule(node.resolve(), used), out, fail)
+    if (Predicate.isTagged(node, "Suspend")) return this.call(this.rule(node.resolve(), used), out, fail)
     if (!isLeaf(node) && (((this.refs.get(p) ?? 0) > 1 && !this.small(p)) || depth > 32)) {
       return this.call(this.rule(p, used), out, fail)
     }
@@ -333,7 +336,13 @@ class Emitter {
     let budget = 6
     const visit = (q: ParserLike): boolean => {
       const node = nodeOf(q)
-      if (--budget < 0 || node._tag === "Suspend" || node._tag === "Escaped" || node._tag === "Dispatch") return false
+      if (
+        --budget < 0 ||
+        Predicate.isTagged(node, "Suspend") ||
+        Predicate.isTagged(node, "Escaped") ||
+        Predicate.isTagged(node, "Dispatch")
+      )
+        return false
       return children(node).every(visit)
     }
     return visit(p)
@@ -477,8 +486,7 @@ class Emitter {
       case "Literal":
         return this.leaf(p, node, used, out, fail, fn, node.literal.length * 4)
       case "Escaped": {
-        // Streaming builds use the closure compiler's escapes, which keep checkpoints
-        if (!this.native(node.encoding) || this.resumable) return this.delegate(p, used, out, fail, fn)
+        if (!this.native(node.encoding)) return this.delegate(p, used, out, fail, fn)
         return this.escaped(p, node, out, fail, fn, d)
       }
       case "Map": {
@@ -504,7 +512,7 @@ class Emitter {
       case "FilterMap": {
         const t = this.v(fn)
         const r = this.v(fn)
-        const [tag, field] = node._tag === "TryMap" ? ["Success", "success"] : ["Some", "value"]
+        const [tag, field] = Predicate.isTagged(node, "TryMap") ? ["Success", "success"] : ["Some", "value"]
         return (
           this.gen(node.self, true, t, fail, fn, d) +
           `${r}=${this.const(node.f)}(${t});if(${r}._tag!=="${tag}"){${BT}${fail}}${set(`${r}.${field}`)}`
@@ -542,7 +550,7 @@ class Emitter {
       case "WithSpan": {
         const st = this.v(fn)
         const t = this.v(fn)
-        const extra = node._tag === "WithSpan" ? `{start:${st},end:p}` : `$S.slice(${st},p)`
+        const extra = Predicate.isTagged(node, "WithSpan") ? `{start:${st},end:p}` : `$S.slice(${st},p)`
         return `${st}=p;` + this.gen(node.self, true, t, fail, fn, d) + set(`[${t},${extra}]`)
       }
       case "Span": {
@@ -576,8 +584,8 @@ class Emitter {
         const ok = this.label()
         const bad = this.label()
         const t = out === null ? null : this.v(fn)
-        const some = node._tag === "Opt" ? `R.some(${t})` : `${t}`
-        const orElse = node._tag === "Opt" ? "R.none()" : `${this.const(node.value)}()`
+        const some = Predicate.isTagged(node, "Opt") ? `R.some(${t})` : `${t}`
+        const orElse = Predicate.isTagged(node, "Opt") ? "R.none()" : `${this.const(node.value)}()`
         return (
           `${st}=p;${ok}:{${bad}:{${this.gen(node.self, used, t, `break ${bad};`, fn, d)}${set(some)}break ${ok};}` +
           `if($S.mode!=="Backtrack"){${fail}}p=${st};${set(orElse)}}`
@@ -641,12 +649,7 @@ class Emitter {
    */
   private float(node: NodeOf<"Map">, out: string, fail: string, fn: Fn, d: number): string | undefined {
     if (!floatTexts.has(node.self) || this.kind === 2) return undefined
-    let taken = nodeOf(node.self)
-    while (taken._tag === "Expected") taken = nodeOf(taken.self)
-    if (taken._tag !== "Taken") return undefined
-    const enc = taken.encoding
-    // Decimal notation is ASCII, which these encodings read unit by unit
-    if (enc !== undefined && !(this.native(enc) && enc.unit === 1 && enc.asciiCompatible)) return undefined
+    if (!scansFloat(node.self, (enc) => this.native(enc))) return undefined
     const t = this.v(fn)
     const scan = this.const(floatScan)
     // Near the end of a partial input, the parser may have to report `Incomplete`
@@ -681,7 +684,7 @@ class Emitter {
 
   alt(node: NodeOf<"Alt">, used: boolean, out: string | null, fail: string, fn: Fn, d: number): string {
     const parsers = node.parsers
-    if (parsers.length === 0) return `${BT}${fail}`
+    if (Arr.isReadonlyArrayEmpty(parsers)) return `${BT}${fail}`
     if (parsers.length === 1) return this.gen(parsers[0], used, out, fail, fn, d)
     const firsts = this.kind === 2 ? undefined : parsers.map((q) => this.closure.firsts.of(q))
     const prune = firsts !== undefined && planAlt(firsts) !== undefined
@@ -755,9 +758,9 @@ class Emitter {
       return code + `default:{$S.pos=p;${r}=${q}.step($S);if(${r}===F){${fail}}p=$S.pos;}}`
     }
     let sel = nodeOf(node.selector)
-    const peek = sel._tag === "Peek"
+    const peek = Predicate.isTagged(sel, "Peek")
     if (peek) sel = nodeOf((sel as NodeOf<"Peek">).self)
-    if (sel._tag !== "Any" || !this.native(sel.encoding)) return general(true)
+    if (!Predicate.isTagged(sel, "Any") || !this.native(sel.encoding)) return general(true)
     const enc = sel.encoding
     const r = enc === undefined ? undefined : this.reader(enc)
     const limit = r === undefined ? (this.kind === 0 ? 0xd800 : TableSize) : Math.min(r.fast, 0xd800)
@@ -802,7 +805,7 @@ class Emitter {
     const bad = this.label()
     const n = this.v(fn)
     const st = this.v(fn)
-    const isFold = node._tag === "Fold"
+    const isFold = Predicate.isTagged(node, "Fold")
     const cond = isFold ? (node as NodeOf<"Fold">).while : undefined
     const keep = out !== null && used && collect === undefined
     // `while` needs the accumulator even when the output is unused
@@ -848,7 +851,7 @@ class Emitter {
    * of a `Done` one; a resumed repetition records its result when it ends.
    */
   private resumePoint(fn: Fn) {
-    const key = this.const({})
+    const key = this.const(new Key())
     const entry = this.v(fn)
     const resumed = this.v(fn)
     const done = this.label()
@@ -970,7 +973,7 @@ class Emitter {
             `else ${deferred}=R.addEntry(${out},${deferred},${key},${value});`,
           finish: `if(${deferred}!==void 0)R.flushEntries(${out},${deferred});`
         }
-        return node._tag === "Repeat"
+        return Predicate.isTagged(node, "Repeat")
           ? this.repeat(node, true, out, fail, fn, d, collect)
           : this.separated(node, true, out, fail, fn, d, collect)
       }
@@ -1190,21 +1193,32 @@ class Emitter {
     return `${head}if(${c}<${small}){if(!(${member}))break;p++}else ${rest}${count}}${finish}`
   }
 
+  /**
+   * `escaped`. In a resumable build it keeps the closure compiler's
+   * checkpoints: where the next segment starts and, unless raw, the text
+   * so far, saved when a segment or an escape runs out of input.
+   */
   escaped(p: ParserLike, node: NodeOf<"Escaped">, out: string | null, fail: string, fn: Fn, d: number): string {
     const enc = node.encoding
     const ctrl = node.control
-    const r = enc === undefined ? undefined : this.reader(enc)
-    if (r !== undefined && !(ctrl < r.fast)) return this.delegate(p, !node.raw, out, fail, fn)
+    const rd = enc === undefined ? undefined : this.reader(enc)
+    if (rd !== undefined && !(ctrl < rd.fast)) return this.delegate(p, !node.raw, out, fail, fn)
     const raw = node.raw
+    const r = this.resumable ? this.resumePoint(fn) : undefined
     const acc = raw ? null : (out ?? this.v(fn))
     const x = raw ? null : this.v(fn)
     const b = this.v(fn)
-    const st = this.v(fn)
+    const st = r?.entry ?? this.v(fn)
     const okv = this.v(fn)
     const loop = this.label()
     const bad = this.label()
+    /** Saves the checkpoint of a segment starting at `pos`, when the parse ran out of input. */
+    const save = (pos: string) =>
+      r === undefined
+        ? ""
+        : `if($S.mode==="Incomplete"&&$S.resume!==void 0)R.save($S,${r.key},${r.entry},${raw ? pos : `{out:${acc},pos:${pos}}`});`
     const normal = this.gen(node.normal, !raw, x, `break ${bad};`, fn, d)
-    const escape = this.gen(node.escape, !raw, x, fail, fn, d)
+    const escape = this.gen(node.escape, !raw, x, `${save(b)}${fail}`, fn, d)
     const onNormal = raw
       ? `if(p===${b}){$S.mode="Cut";${fail}}continue ${loop};`
       : `if(p!==${b}){${acc}+=${x};continue ${loop}}`
@@ -1214,37 +1228,40 @@ class Emitter {
         : this.kind === 0
           ? `I.slice(${st},p)`
           : `I.subarray(${st},p)`
-    const ctrlTest = r === undefined ? `p<E&&${this.unit("p")}===${ctrl}` : `${r.guard("p")}&&${r.unit("p")}===${ctrl}`
-    return (
-      `${raw ? `${st}=p;` : `${acc}="";`}${loop}:while(p<E){${b}=p;${okv}=false;` +
+    const ctrlTest =
+      rd === undefined ? `p<E&&${this.unit("p")}===${ctrl}` : `${rd.guard("p")}&&${rd.unit("p")}===${ctrl}`
+    // In a resumable build, running out of a partial input (not stopping
+    // before a non-control character) leaves the text incomplete
+    const stop = r === undefined ? loop : this.label()
+    const loopCode =
+      `${loop}:while(p<E){${b}=p;${okv}=false;` +
       `${bad}:{${normal}${okv}=true;}` +
-      `if(${okv}){${onNormal}}else{if($S.mode!=="Backtrack"){${fail}}p=${b}}` +
-      `if(${ctrlTest}){p+=${r?.width ?? 1};${escape}${raw ? "" : `${acc}+=${x};`}}else break ${loop};}` +
-      (raw && out !== null ? `${out}=${slice};` : "")
-    )
+      `if(${okv}){${onNormal}}else{if($S.mode!=="Backtrack"){${save(b)}${fail}}p=${b}}` +
+      `if(${ctrlTest}){p+=${rd?.width ?? 1};${escape}${raw ? "" : `${acc}+=${x};`}}else break ${stop};}`
+    const body =
+      r === undefined ? loopCode : `${stop}:{${loopCode}if($S.partial){$S.mode="Incomplete";${save("p")}${fail}}}`
+    if (r === undefined) {
+      return `${raw ? `${st}=p;` : `${acc}="";`}${body}${raw && out !== null ? `${out}=${slice};` : ""}`
+    }
+    // `acc` is set unless raw
+    if (acc !== null) return r.wrap(acc, `${acc}=cp.out;p=cp.pos;`, `${acc}="";`, body)
+    const value = out ?? this.v(fn)
+    return r.wrap(value, "p=cp;", "", `${body}${value}=${out === null ? "void 0" : slice};`)
   }
 }
 
 /** The UTF-16 code units of a string. */
 const unitsOf = (s: string): Array<number> => Array.from({ length: s.length }, (_, i) => s.charCodeAt(i))
 
-/** 1 where a small code is accepted (taking `negate` into account). */
-const acceptTable = (set: CharSet, negate: boolean): Uint8Array => {
-  const t = new Uint8Array(TableSize)
-  for (let c = 0; c < TableSize; c++) t[c] = (set.table[c] === 1) !== negate ? 1 : 0
-  return t
-}
-
-/** Generates the source of a compiled parser, without building it. */
-const generate = (
-  root: ParserLike,
-  kind: 0 | 1 | 2,
-  resumable = false
-): {
+/** The source of a compiled parser, with the values it reads as `C`. */
+interface Generated {
   readonly source: string
   readonly constants: ReadonlyArray<unknown>
   readonly fallbacks: ReadonlyArray<Fallback>
-} => {
+}
+
+/** Generates the source of a compiled parser, without building it. */
+const generate = (root: ParserLike, kind: 0 | 1 | 2, resumable = false): Generated => {
   const e = new Emitter(root, kind, resumable)
   const main = e.rule(root, true)
   const source =
@@ -1254,16 +1271,13 @@ const generate = (
   return { source, constants: e.constants, fallbacks: e.fallbacks }
 }
 
-/** Compiles a root parser for one input kind with `new Function`. */
-const compileJit = (
-  root: ParserLike,
-  kind: 0 | 1 | 2,
-  resumable = false
-): {
-  readonly step: Step
-  readonly fallbacks: ReadonlyArray<Fallback>
+/** A root parser compiled with `new Function`, with its generated source. */
+interface JitCompiled extends CompiledRoot {
   readonly source: string
-} => {
+}
+
+/** Compiles a root parser for one input kind with `new Function`. */
+const compileJit = (root: ParserLike, kind: 0 | 1 | 2, resumable = false): JitCompiled => {
   const { constants, fallbacks, source } = generate(root, kind, resumable)
   const run = globalThis.Function("C", "F", "R", source)(constants, Fail, runtime) as Step
   // Generated code reads past `end` expecting nothing to match there, which
@@ -1283,12 +1297,19 @@ const compileJit = (
 export const backend: Backend = {
   build: (p, kind, resumable) => {
     if (!jitSupported()) return closureBackend.build(p, kind, resumable)
-    try {
-      const { fallbacks, source, step } = compileJit(p, kind, resumable)
-      return { backend: "jit", step, fallbacks, error: undefined, source }
-    } catch (error) {
-      // Fall back to the closure compiler, remembering why
-      return { ...closureBackend.build(p, kind, resumable), jitError: error }
-    }
+    return Result.match(
+      Result.try(() => compileJit(p, kind, resumable)),
+      {
+        onSuccess: ({ fallbacks, source, step }) => ({
+          backend: "jit" as const,
+          step,
+          fallbacks,
+          error: undefined,
+          source
+        }),
+        // Fall back to the closure compiler, remembering why
+        onFailure: (error) => ({ ...closureBackend.build(p, kind, resumable), jitError: error })
+      }
+    )
   }
 }

package/src/internal/compiler/sets.ts

@@ -21,6 +21,13 @@ export interface CharSet {
   readonly ranges: ReadonlyArray<readonly [number, number]> | undefined
 }
 
+/** 1 where a small code is accepted (taking `negate` into account). */
+export const acceptTable = (set: CharSet, negate: boolean): Uint8Array => {
+  const t = new Uint8Array(TableSize)
+  for (let c = 0; c < TableSize; c++) t[c] = (set.table[c] === 1) !== negate ? 1 : 0
+  return t
+}
+
 const isRange = (u: unknown): u is { readonly from: string | number; readonly to: string | number } =>
   Predicate.isObjectOrArray(u) && (u as any)._tag === "TokenRange"
 

package/src/internal/key.ts

@@ -0,0 +1,11 @@
+/**
+ * Identity keys: a parser that keeps state across parses (resume
+ * checkpoints, reusable syntax nodes) creates one `Key` when it is built and
+ * looks its state up by that key's reference.
+ */
+
+/** A per-parser identity, compared by reference. */
+export class Key {
+  // Nominal: an empty class would accept any object
+  declare private readonly brand: void
+}

package/src/internal/node.ts

@@ -7,6 +7,8 @@
  * `Suspend` is just a reference.
  */
 import * as Predicate from "effect/Predicate"
+import type * as Option from "effect/Option"
+import type * as Result from "effect/Result"
 import type { Encoding } from "../Encoding.js"
 import type { ErrMode, Input, State } from "../Input.js"
 
@@ -67,8 +69,8 @@ export type Node =
   | { readonly _tag: "Map"; readonly self: P; readonly f: (a: any) => unknown }
   | { readonly _tag: "As"; readonly self: P; readonly value: unknown }
   | { readonly _tag: "Tap"; readonly self: P; readonly f: (a: any) => void }
-  | { readonly _tag: "TryMap"; readonly self: P; readonly f: (a: any) => unknown }
-  | { readonly _tag: "FilterMap"; readonly self: P; readonly f: (a: any) => unknown }
+  | { readonly _tag: "TryMap"; readonly self: P; readonly f: (a: any) => Result.Result<unknown, unknown> }
+  | { readonly _tag: "FilterMap"; readonly self: P; readonly f: (a: any) => Option.Option<unknown> }
   | { readonly _tag: "Filter"; readonly self: P; readonly predicate: (a: any) => boolean }
   | { readonly _tag: "FlatMap"; readonly self: P; readonly f: (a: any) => P }
   // Consumed input
@@ -160,6 +162,21 @@ export const bounds = (
   return { min: range.min ?? 0, max: range.max ?? Infinity }
 }
 
+/**
+ * Whether the compilers' one-pass float scan applies to `p`, the text
+ * parser of `Ascii.float` / `Text.float`: a `taken` (under any `expected`
+ * labels) in an encoding that `native` accepts, reading one ASCII-compatible
+ * unit at a time, or in no encoding at all.
+ */
+export const scansFloat = (p: ParserLike, native: (enc: Encoding<any>) => boolean): boolean => {
+  let taken = nodeOf(p)
+  while (Predicate.isTagged(taken, "Expected")) taken = nodeOf(taken.self)
+  if (!Predicate.isTagged(taken, "Taken")) return false
+  const enc = taken.encoding
+  // Decimal notation is ASCII, which these encodings read unit by unit
+  return enc === undefined || (native(enc) && enc.unit === 1 && enc.asciiCompatible)
+}
+
 /** The child parsers of a node, in evaluation order. */
 export const children = (node: Node): ReadonlyArray<P> => {
   switch (node._tag) {

package/src/internal/resume.ts

@@ -19,12 +19,13 @@
  * it all again. `Done` checkpoints stay in place when taken.
  */
 import type { State } from "../Input.js"
+import type { Key } from "./key.js"
 
 /** The checkpoints of a resumable parse. */
-export type Checkpoints = Map<object, Map<number, unknown>>
+export type Checkpoints = Map<Key, Map<number, unknown>>
 
 /** Takes (and removes) the checkpoint of `key` entered at `offset`, if any. */
-export const take = (state: State<any>, key: object, offset: number): unknown => {
+export const take = (state: State<any>, key: Key, offset: number): unknown => {
   const byKey = state.resume?.get(key)
   if (byKey === undefined) return undefined
   const checkpoint = byKey.get(offset)
@@ -44,13 +45,13 @@ export class Done {
  * Returns `value`, the result of a repetition entered at `entry`; when the
  * repetition was resumed from a checkpoint, records it as {@link Done}.
  */
-export const done = <A>(state: State<any>, key: object, entry: number, resumed: boolean, value: A): A => {
+export const done = <A>(state: State<any>, key: Key, entry: number, resumed: boolean, value: A): A => {
   if (resumed && state.resume !== undefined) save(state, key, entry, new Done(value, state.pos))
   return value
 }
 
 /** Records the checkpoint of `key` entered at `offset`. */
-export const save = (state: State<any>, key: object, offset: number, checkpoint: unknown): void => {
+export const save = (state: State<any>, key: Key, offset: number, checkpoint: unknown): void => {
   const resume = state.resume
   if (resume === undefined) return
   let byKey = resume.get(key)

package/src/internal/run.ts

@@ -33,7 +33,7 @@ export const runOn = <A>(
         // SAFETY: the compiled step returns what the parser's step returns
         return a as A
       }
-      state.pos = start
+      state.rewind(start)
       if (!diagnostics) return Fail
     }
   }