@thomasfosterau/effect-parse 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +44 -1
  2. package/dist/Ascii.d.ts +0 -2
  3. package/dist/Ascii.d.ts.map +1 -1
  4. package/dist/Ascii.js +6 -6
  5. package/dist/Ascii.js.map +1 -1
  6. package/dist/Binary.d.ts +0 -2
  7. package/dist/Binary.d.ts.map +1 -1
  8. package/dist/Binary.js +1 -2
  9. package/dist/Bits.d.ts +0 -2
  10. package/dist/Bits.d.ts.map +1 -1
  11. package/dist/Bits.js +2 -3
  12. package/dist/Bits.js.map +1 -1
  13. package/dist/Compiler.d.ts +0 -2
  14. package/dist/Compiler.d.ts.map +1 -1
  15. package/dist/Compiler.js +1 -2
  16. package/dist/Encoding.d.ts +0 -2
  17. package/dist/Encoding.d.ts.map +1 -1
  18. package/dist/Encoding.js +1 -2
  19. package/dist/Expression.d.ts +0 -2
  20. package/dist/Expression.d.ts.map +1 -1
  21. package/dist/Expression.js +61 -18
  22. package/dist/Expression.js.map +1 -1
  23. package/dist/Input.d.ts +80 -7
  24. package/dist/Input.d.ts.map +1 -1
  25. package/dist/Input.js +149 -18
  26. package/dist/Input.js.map +1 -1
  27. package/dist/Nom.d.ts +1 -3
  28. package/dist/Nom.d.ts.map +1 -1
  29. package/dist/Nom.js +1 -2
  30. package/dist/ParseError.d.ts +0 -2
  31. package/dist/ParseError.d.ts.map +1 -1
  32. package/dist/ParseError.js +6 -10
  33. package/dist/ParseError.js.map +1 -1
  34. package/dist/Parser.d.ts +2 -2
  35. package/dist/Parser.d.ts.map +1 -1
  36. package/dist/Parser.js +58 -32
  37. package/dist/Parser.js.map +1 -1
  38. package/dist/ParserSchema.d.ts +198 -2
  39. package/dist/ParserSchema.d.ts.map +1 -1
  40. package/dist/ParserSchema.js +646 -18
  41. package/dist/ParserSchema.js.map +1 -1
  42. package/dist/ParserStream.d.ts +0 -2
  43. package/dist/ParserStream.d.ts.map +1 -1
  44. package/dist/ParserStream.js +15 -9
  45. package/dist/ParserStream.js.map +1 -1
  46. package/dist/Syntax.d.ts +0 -2
  47. package/dist/Syntax.d.ts.map +1 -1
  48. package/dist/Syntax.js +13 -11
  49. package/dist/Syntax.js.map +1 -1
  50. package/dist/Text.d.ts +0 -2
  51. package/dist/Text.d.ts.map +1 -1
  52. package/dist/Text.js +1 -2
  53. package/dist/Token.d.ts +0 -2
  54. package/dist/Token.d.ts.map +1 -1
  55. package/dist/Token.js +1 -2
  56. package/dist/internal/compiler/closure.d.ts +5 -2
  57. package/dist/internal/compiler/closure.d.ts.map +1 -1
  58. package/dist/internal/compiler/closure.js +160 -92
  59. package/dist/internal/compiler/closure.js.map +1 -1
  60. package/dist/internal/compiler/entries.d.ts +0 -2
  61. package/dist/internal/compiler/entries.d.ts.map +1 -1
  62. package/dist/internal/compiler/entries.js +6 -7
  63. package/dist/internal/compiler/entries.js.map +1 -1
  64. package/dist/internal/compiler/first.d.ts +0 -2
  65. package/dist/internal/compiler/first.d.ts.map +1 -1
  66. package/dist/internal/compiler/first.js +10 -10
  67. package/dist/internal/compiler/first.js.map +1 -1
  68. package/dist/internal/compiler/jit.d.ts +0 -2
  69. package/dist/internal/compiler/jit.d.ts.map +1 -1
  70. package/dist/internal/compiler/jit.js +52 -51
  71. package/dist/internal/compiler/jit.js.map +1 -1
  72. package/dist/internal/compiler/registry.d.ts +0 -2
  73. package/dist/internal/compiler/registry.d.ts.map +1 -1
  74. package/dist/internal/compiler/registry.js +1 -2
  75. package/dist/internal/compiler/sets.d.ts +2 -2
  76. package/dist/internal/compiler/sets.d.ts.map +1 -1
  77. package/dist/internal/compiler/sets.js +8 -3
  78. package/dist/internal/compiler/sets.js.map +1 -1
  79. package/dist/internal/entries.d.ts +0 -2
  80. package/dist/internal/entries.d.ts.map +1 -1
  81. package/dist/internal/entries.js +1 -2
  82. package/dist/internal/key.d.ts +10 -0
  83. package/dist/internal/key.d.ts.map +1 -0
  84. package/dist/internal/key.js +11 -0
  85. package/dist/internal/key.js.map +1 -0
  86. package/dist/internal/node.d.ts +11 -4
  87. package/dist/internal/node.d.ts.map +1 -1
  88. package/dist/internal/node.js +15 -3
  89. package/dist/internal/node.js.map +1 -1
  90. package/dist/internal/numbers.d.ts +0 -2
  91. package/dist/internal/numbers.d.ts.map +1 -1
  92. package/dist/internal/numbers.js +1 -2
  93. package/dist/internal/resume.d.ts +5 -6
  94. package/dist/internal/resume.d.ts.map +1 -1
  95. package/dist/internal/resume.js +1 -2
  96. package/dist/internal/resume.js.map +1 -1
  97. package/dist/internal/run.d.ts +0 -2
  98. package/dist/internal/run.d.ts.map +1 -1
  99. package/dist/internal/run.js +2 -3
  100. package/dist/internal/run.js.map +1 -1
  101. package/dist/internal/stack.d.ts +0 -2
  102. package/dist/internal/stack.d.ts.map +1 -1
  103. package/dist/internal/stack.js +12 -3
  104. package/dist/internal/stack.js.map +1 -1
  105. package/dist/internal/syntax.d.ts +30 -19
  106. package/dist/internal/syntax.d.ts.map +1 -1
  107. package/dist/internal/syntax.js +74 -63
  108. package/dist/internal/syntax.js.map +1 -1
  109. package/dist/internal/text.d.ts +5 -2
  110. package/dist/internal/text.d.ts.map +1 -1
  111. package/dist/internal/text.js +14 -5
  112. package/dist/internal/text.js.map +1 -1
  113. package/dist/internal/ties.d.ts +28 -0
  114. package/dist/internal/ties.d.ts.map +1 -0
  115. package/dist/internal/ties.js +66 -0
  116. package/dist/internal/ties.js.map +1 -0
  117. package/dist/internal/tokens.d.ts +10 -9
  118. package/dist/internal/tokens.d.ts.map +1 -1
  119. package/dist/internal/tokens.js +4 -8
  120. package/dist/internal/tokens.js.map +1 -1
  121. package/package.json +6 -6
  122. package/src/Ascii.ts +6 -5
  123. package/src/Bits.ts +1 -1
  124. package/src/Expression.ts +27 -23
  125. package/src/Input.ts +157 -21
  126. package/src/ParseError.ts +10 -8
  127. package/src/Parser.ts +67 -32
  128. package/src/ParserSchema.ts +982 -23
  129. package/src/ParserStream.ts +20 -8
  130. package/src/Syntax.ts +12 -9
  131. package/src/internal/compiler/closure.ts +202 -90
  132. package/src/internal/compiler/entries.ts +14 -11
  133. package/src/internal/compiler/first.ts +9 -8
  134. package/src/internal/compiler/jit.ts +95 -74
  135. package/src/internal/compiler/sets.ts +7 -0
  136. package/src/internal/key.ts +11 -0
  137. package/src/internal/node.ts +19 -2
  138. package/src/internal/resume.ts +5 -4
  139. package/src/internal/run.ts +1 -1
  140. package/src/internal/stack.ts +6 -1
  141. package/src/internal/syntax.ts +80 -84
  142. package/src/internal/text.ts +24 -4
  143. package/src/internal/ties.ts +83 -0
  144. package/src/internal/tokens.ts +20 -13
@@ -36,6 +36,7 @@ import * as Registry from "./internal/compiler/registry.js"
36
36
  import type * as Resume from "./internal/resume.js"
37
37
  import * as Run from "./internal/run.js"
38
38
  import * as Stack from "./internal/stack.js"
39
+ import { isEmptyUnits } from "./internal/text.js"
39
40
  import type { Parser } from "./Parser.js"
40
41
 
41
42
  /**
@@ -49,7 +50,7 @@ export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
49
50
  const concat = <I extends Chunkable>(a: I, b: I): I => {
50
51
  if (Predicate.isString(a)) return (a + (b as string)) as I
51
52
  if (a instanceof Uint8Array) {
52
- if (a.length === 0) return b
53
+ if (a.byteLength === 0) return b
53
54
  const out = new Uint8Array(a.length + (b as Uint8Array).length)
54
55
  out.set(a)
55
56
  out.set(b as Uint8Array, a.length)
@@ -174,6 +175,7 @@ class Incremental<A, E, I extends Chunkable> {
174
175
  /** The input length at the last attempt, and where that attempt ran out of input. */
175
176
  private attempted = 0
176
177
  private restart = 0
178
+ private pushed = 0
177
179
  /** Set once an attempt settles: a success (with its end offset) or a definite failure. */
178
180
  private settled: { readonly value: A; readonly pos: number } | "failed" | undefined
179
181
 
@@ -221,9 +223,14 @@ class Incremental<A, E, I extends Chunkable> {
221
223
  }
222
224
 
223
225
  push(chunk: I): void {
224
- if (chunk.length === 0) return
226
+ if (isEmptyUnits(chunk)) return
225
227
  this.append(chunk)
226
228
  if (this.settled !== undefined) return
229
+ // The first attempt waits for a second chunk: an input that arrives in
230
+ // one chunk (most small responses) is parsed once, by `finish`, instead
231
+ // of once in partial mode and again in complete mode, since a repetition
232
+ // that completes without resuming keeps no checkpoint
233
+ if (++this.pushed === 1) return
227
234
  // An attempt re-reads what the last one could not finish, from `restart`;
228
235
  // waiting until at least half as much new input has arrived keeps the
229
236
  // total work linear in the input. Strings are also copied when they are
@@ -234,9 +241,12 @@ class Incremental<A, E, I extends Chunkable> {
234
241
  this.attempt()
235
242
  }
236
243
 
237
- /** The step attempts run: the parser's resumable compiled code if it was compiled, else the interpreter. */
238
- private step(state: State<I>): unknown {
239
- return Stack.run(Registry.stepFor(this.parser, state, true) ?? this.parser.step, state)
244
+ /**
245
+ * The step attempts run: the parser's compiled code if it was compiled
246
+ * (the resumable build, unless `resumable` is false), else the interpreter.
247
+ */
248
+ private step(state: State<I>, resumable = true): unknown {
249
+ return Stack.run(Registry.stepFor(this.parser, state, resumable) ?? this.parser.step, state)
240
250
  }
241
251
 
242
252
  private attempt(): void {
@@ -258,10 +268,12 @@ class Incremental<A, E, I extends Chunkable> {
258
268
  if (settled !== undefined && settled !== "failed") {
259
269
  if (settled.pos === this.length) return Effect.succeed(settled.value)
260
270
  } else if (settled === undefined) {
261
- // The last attempt, in complete mode, resumes from the checkpoints too
271
+ // The last attempt, in complete mode, resumes from the checkpoints too;
272
+ // with no attempt before it, it is an ordinary parse
262
273
  const state = new State(this.input(), { context: this.context })
263
- state.resume = this.resume
264
- const a = this.step(state)
274
+ const resumable = this.attempted > 0
275
+ if (resumable) state.resume = this.resume
276
+ const a = this.step(state, resumable)
265
277
  if (a !== Fail && state.pos === state.end) return Effect.succeed(a as A)
266
278
  }
267
279
  // The error comes from an ordinary parse of the whole input, so it is the
package/src/Syntax.ts CHANGED
@@ -46,17 +46,19 @@
46
46
  *
47
47
  * @since 0.2.0
48
48
  */
49
+ import * as Str from "effect/String"
49
50
  import * as Predicate from "effect/Predicate"
50
51
  import type * as Context from "effect/Context"
51
52
  import { dual } from "effect/Function"
52
53
  import * as Option from "effect/Option"
53
54
  import type { Input, Slice, Source } from "./Input.js"
54
- import { Fail, fromSource } from "./Input.js"
55
+ import { Fail, fromSource, State } from "./Input.js"
55
56
  import * as ParseError from "./ParseError.js"
56
57
  import { nodeOf } from "./internal/node.js"
58
+ import { Key } from "./internal/key.js"
57
59
  import * as Stack from "./internal/stack.js"
58
60
  import type { ErrorInfo, Green, Previous, Segment } from "./internal/syntax.js"
59
- import { absolute, errorKind, SyntaxState } from "./internal/syntax.js"
61
+ import { absolute, errorKind, SyntaxBuilder } from "./internal/syntax.js"
60
62
  import type { Parser } from "./Parser.js"
61
63
  import { described, isParser } from "./Parser.js"
62
64
 
@@ -67,7 +69,7 @@ import { described, isParser } from "./Parser.js"
67
69
  const syntaxNode = (self: Parser.Any, kind: string, named: boolean): Parser.Any => {
68
70
  const step = self.step
69
71
  // Identifies the nodes this parser builds, so a reparse can reuse them
70
- const key = {}
72
+ const key = new Key()
71
73
  return described<unknown, unknown, unknown, Input>(
72
74
  (state) => {
73
75
  const builder = state.syntax
@@ -133,7 +135,7 @@ export const token: {
133
135
 
134
136
  const defaultTokenKind = (self: Parser.Any): string => {
135
137
  const n = nodeOf(self)
136
- if (n._tag === "Literal" && Predicate.isString(n.literal) && !n.caseless) return n.literal
138
+ if (Predicate.isTagged(n, "Literal") && Predicate.isString(n.literal) && !n.caseless) return n.literal
137
139
  return self.name ?? "token"
138
140
  }
139
141
 
@@ -577,7 +579,7 @@ export class SyntaxNode<I extends Input = string> {
577
579
  let out = ""
578
580
  const visit = (green: Green, field: string | undefined) => {
579
581
  if (green.named || green === this.green) {
580
- if (out.length > 0) out += " "
582
+ if (Str.isNonEmpty(out)) out += " "
581
583
  if (field !== undefined) out += `${field}: `
582
584
  out += `(${green.kind}`
583
585
  for (let i = 0; i < green.children.length; i++) visit(green.children[i], green.fields?.[i])
@@ -708,7 +710,8 @@ export class TreeCursor<I extends Input = string> {
708
710
  const expectedEof = ["end of input"]
709
711
 
710
712
  const run = <A, E, I extends Input>(config: Config, input: I, previous: Previous | undefined): Tree<A, E, I> => {
711
- const state = new SyntaxState<I>(input, { context: config.context }, previous)
713
+ const state = new State<I>(input, { context: config.context })
714
+ const builder = new SyntaxBuilder(state, previous)
712
715
  const a = Stack.run(config.inner.step, state)
713
716
  // Text parsers record the encoding when they fail, which they may have done inside reused nodes
714
717
  if (state.encoding === undefined && previous !== undefined) state.encoding = previous.encoding
@@ -722,8 +725,8 @@ const run = <A, E, I extends Input>(config: Config, input: I, previous: Previous
722
725
  } else {
723
726
  value = Option.some(a as A)
724
727
  }
725
- const green = state.finish(config.kind, Option.getOrUndefined(value), fatal, state.pos)
726
- return new Tree<A, E, I>(input, value, state.reused, green, config, state.encoding)
728
+ const green = builder.finish(config.kind, Option.getOrUndefined(value), fatal, state.pos)
729
+ return new Tree<A, E, I>(input, value, builder.reused, green, config, state.encoding)
727
730
  }
728
731
 
729
732
  /**
@@ -756,7 +759,7 @@ export const parse: {
756
759
  options?: ParseOptions & { readonly context?: Context.Context<R> }
757
760
  ): Tree<A, E, I> => {
758
761
  const n = nodeOf(grammar)
759
- const isRoot = n._tag === "Syntax" && n.kind !== undefined && n.named
762
+ const isRoot = Predicate.isTagged(n, "Syntax") && n.kind !== undefined && n.named
760
763
  const config: Config = {
761
764
  kind: isRoot ? n.kind : (options?.kind ?? "document"),
762
765
  inner: isRoot ? (n.self as Parser.Any) : grammar,
@@ -12,20 +12,21 @@ import type { Encoding } from "../../Encoding.js"
12
12
  import type { ErrMode, State } from "../../Input.js"
13
13
  import { Fail } from "../../Input.js"
14
14
  import * as Option from "effect/Option"
15
+ import * as Result from "effect/Result"
15
16
  import type { Node, NodeOf, ParserLike } from "../node.js"
16
17
  import { addEntry, flushEntries, fromEntries, isPlainKey } from "../entries.js"
17
- import { nodeOf } from "../node.js"
18
+ import { nodeOf, scansFloat } from "../node.js"
18
19
  import { floatScan, floatTexts, scanFloatBytes, scanFloatString, toFloat } from "../numbers.js"
19
- import { charOf, fastLimit, firstCodePoint, fold, validate } from "../text.js"
20
+ import { charOf, fastLimit, firstCodePoint, fold, utf16be, utf16le, validate } from "../text.js"
20
21
  import { countTokens, indexOf } from "../tokens.js"
21
22
  import type { AltPlan } from "./first.js"
23
+ import { Key } from "../key.js"
22
24
  import * as Resume from "../resume.js"
23
25
  import type { Backend } from "./registry.js"
24
26
  import type { EntryPair } from "./entries.js"
25
27
  import { buildsEntries, pairOf } from "./entries.js"
26
28
  import { FirstSets, planAlt } from "./first.js"
27
- import type { CharSet } from "./sets.js"
28
- import { kindOfEncoding, TableSize, textSet, tokenSet } from "./sets.js"
29
+ import { acceptTable, kindOfEncoding, TableSize, textSet, tokenSet } from "./sets.js"
29
30
 
30
31
  export type Step = (state: State<any>) => any
31
32
 
@@ -53,9 +54,6 @@ const Margin = 4
53
54
 
54
55
  const lowerAscii = (c: number): number => (c >= 65 && c <= 90 ? c + 32 : c)
55
56
 
56
- const isSuccess = (r: any): boolean => r._tag === "Success"
57
- const isSome = (o: any): boolean => o._tag === "Some"
58
-
59
57
  interface Slot {
60
58
  fn: Step | undefined
61
59
  }
@@ -183,7 +181,9 @@ class Compiler {
183
181
  return this.partialSafe(
184
182
  p,
185
183
  node.encoding === undefined ? this.token(p, node, used) : this.text(p, node, node.encoding, used),
186
- node._tag === "Literal" || node._tag === "TakeUntil" ? node.literal.length * 4 : Margin
184
+ Predicate.isTagged(node, "Literal") || Predicate.isTagged(node, "TakeUntil")
185
+ ? node.literal.length * 4
186
+ : Margin
187
187
  )
188
188
  case "TillLineEnding":
189
189
  case "Identifier":
@@ -228,8 +228,8 @@ class Compiler {
228
228
  return (s) => {
229
229
  const a = c(s)
230
230
  if (a === Fail) return Fail
231
- const r = f(a) as any
232
- if (isSuccess(r)) return r.success
231
+ const r = f(a)
232
+ if (Result.isSuccess(r)) return r.success
233
233
  s.mode = BT
234
234
  return Fail
235
235
  }
@@ -240,8 +240,8 @@ class Compiler {
240
240
  return (s) => {
241
241
  const a = c(s)
242
242
  if (a === Fail) return Fail
243
- const o = f(a) as any
244
- if (isSome(o)) return o.value
243
+ const o = f(a)
244
+ if (Option.isSome(o)) return o.value
245
245
  s.mode = BT
246
246
  return Fail
247
247
  }
@@ -326,8 +326,8 @@ class Compiler {
326
326
  case "Opt":
327
327
  case "OrElseSucceed": {
328
328
  const c = this.compile(node.self, used)
329
- const orElse = !used ? undefined : node._tag === "Opt" ? Option.none : node.value
330
- const some = used && node._tag === "Opt"
329
+ const orElse = !used ? undefined : Predicate.isTagged(node, "Opt") ? Option.none : node.value
330
+ const some = used && Predicate.isTagged(node, "Opt")
331
331
  return (s) => {
332
332
  const start = s.pos
333
333
  const a = c(s)
@@ -558,9 +558,9 @@ class Compiler {
558
558
  }
559
559
  // Switch on the next unit when the selector is `any` or `peek(any)`
560
560
  let sel = nodeOf(node.selector)
561
- const peek = sel._tag === "Peek"
561
+ const peek = Predicate.isTagged(sel, "Peek")
562
562
  if (peek) sel = nodeOf((sel as NodeOf<"Peek">).self)
563
- if (sel._tag !== "Any" || !this.native(sel.encoding)) return general
563
+ if (!Predicate.isTagged(sel, "Any") || !this.native(sel.encoding)) return general
564
564
  const enc = sel.encoding
565
565
  // Units below `limit` are whole tokens, so the key is known in advance
566
566
  const limit = enc === undefined ? (this.kind === 0 ? 0xd800 : TableSize) : Math.min(fastLimit(enc), 0xd800)
@@ -596,11 +596,11 @@ class Compiler {
596
596
  repeat(node: NodeOf<"Repeat"> | NodeOf<"Fold">, used: boolean): Step {
597
597
  const { max, min } = node
598
598
  const checkLoop = max === Infinity
599
- const isFold = node._tag === "Fold"
599
+ const isFold = Predicate.isTagged(node, "Fold")
600
600
  const cond = isFold ? (node as NodeOf<"Fold">).while : undefined
601
601
  const c = this.compile(node.self, used)
602
602
  const resumable = this.resumable
603
- const key = {}
603
+ const key = new Key()
604
604
  // The accumulator: an array, the fold's value, or nothing when unused (and unneeded by `while`)
605
605
  const tracked = used || cond !== undefined
606
606
  const init: () => unknown = !tracked ? () => undefined : isFold ? (node as NodeOf<"Fold">).init : () => []
@@ -715,7 +715,7 @@ class Compiler {
715
715
  }
716
716
  const { max, min } = node
717
717
  const checkLoop = max === Infinity
718
- if (node._tag === "Repeat") {
718
+ if (Predicate.isTagged(node, "Repeat")) {
719
719
  return (s) => {
720
720
  const acc = new EntriesAcc()
721
721
  for (let count = 0; count < max; count++) {
@@ -812,7 +812,7 @@ class Compiler {
812
812
  const c = this.compile(node.self, used)
813
813
  const sep = this.compile(node.separator, false)
814
814
  const resumable = this.resumable
815
- const key = {}
815
+ const key = new Key()
816
816
  return (s) => {
817
817
  const entry = s.pos
818
818
  let acc: Array<unknown> | undefined
@@ -878,7 +878,7 @@ class Compiler {
878
878
  const sep = this.compile(node.separator, true)
879
879
  const op = node.op
880
880
  const resumable = this.resumable
881
- const key = {}
881
+ const key = new Key()
882
882
  if (!node.right) {
883
883
  return (s) => {
884
884
  const entry = s.pos
@@ -1169,22 +1169,19 @@ class Compiler {
1169
1169
  // ---------------------------------------------------------------------------
1170
1170
 
1171
1171
  text(p: ParserLike, node: TokenNode, enc: Encoding<any>, used: boolean): Step {
1172
- const kind = this.kind as 0 | 1
1173
- const fast = fastLimit(enc)
1172
+ const { fast, unitAt, width: unitWidth } = unitReader(this.kind, enc)
1174
1173
  const decode = enc.decode
1175
1174
  const width = enc.width
1176
1175
  const toText = enc.text
1177
- const unitAt: (input: any, i: number) => number =
1178
- kind === 0 ? (input: string, i) => input.charCodeAt(i) : (input: Uint8Array, i) => input[i]
1179
1176
  switch (node._tag) {
1180
1177
  case "Any":
1181
1178
  return (s) => {
1182
1179
  const pos = s.pos
1183
1180
  const end = s.end
1184
1181
  if (pos >= end) return fail(s)
1185
- const c = unitAt(s.input, pos)
1182
+ const c = unitAt(s.input, pos, end)
1186
1183
  if (c < fast) {
1187
- s.pos = pos + 1
1184
+ s.pos = pos + unitWidth
1188
1185
  return used ? charOf(c) : undefined
1189
1186
  }
1190
1187
  const cp = decode(s.input, pos, end)
@@ -1204,11 +1201,11 @@ class Compiler {
1204
1201
  let pos = start
1205
1202
  for (let i = 0; i < n; i++) {
1206
1203
  if (pos >= end) return failAt(s, pos)
1207
- const c = unitAt(input, pos)
1204
+ const c = unitAt(input, pos, end)
1208
1205
  let cp: number
1209
1206
  if (c < fast) {
1210
1207
  cp = c
1211
- pos++
1208
+ pos += unitWidth
1212
1209
  } else {
1213
1210
  cp = decode(input, pos, end)
1214
1211
  if (cp < 0) return failAt(s, pos)
@@ -1253,15 +1250,15 @@ class Compiler {
1253
1250
  const pos = s.pos
1254
1251
  const end = s.end
1255
1252
  if (pos >= end) return fail(s)
1256
- const c = unitAt(s.input, pos)
1253
+ const c = unitAt(s.input, pos, end)
1257
1254
  if (c < fast && c < TableSize) {
1258
1255
  if (accept[c] === 0) return fail(s)
1259
- s.pos = pos + 1
1256
+ s.pos = pos + unitWidth
1260
1257
  return used ? charOf(c) : undefined
1261
1258
  }
1262
1259
  const cp = c < fast ? c : decode(s.input, pos, end)
1263
1260
  if (cp < 0 || test(cp) === negate) return fail(s)
1264
- s.pos = pos + (c < fast ? 1 : width(cp))
1261
+ s.pos = pos + (c < fast ? unitWidth : width(cp))
1265
1262
  return used ? charOf(cp) : undefined
1266
1263
  }
1267
1264
  }
@@ -1274,8 +1271,8 @@ class Compiler {
1274
1271
  let pos = start
1275
1272
  for (let got = 0; got < count; got++) {
1276
1273
  if (pos >= end) return failAt(s, pos)
1277
- const c = unitAt(input, pos)
1278
- if (c < fast) pos++
1274
+ const c = unitAt(input, pos, end)
1275
+ if (c < fast) pos += unitWidth
1279
1276
  else {
1280
1277
  const cp = decode(input, pos, end)
1281
1278
  if (cp < 0) return failAt(s, pos)
@@ -1291,32 +1288,12 @@ class Compiler {
1291
1288
  const set = textSet(node.set)
1292
1289
  const accept = acceptTable(set, negate)
1293
1290
  const test = set.test
1294
- const small = Math.min(fast, TableSize)
1295
- return (s) => {
1296
- const input = s.input
1297
- const start = s.pos
1298
- const end = s.end
1299
- let i = start
1300
- let count = 0
1301
- while (count < max && i < end) {
1302
- const c = unitAt(input, i)
1303
- if (c < small) {
1304
- if (accept[c] === 0) break
1305
- i++
1306
- } else if (c < fast) {
1307
- if (test(c) === negate) break
1308
- i++
1309
- } else {
1310
- const cp = decode(input, i, end)
1311
- if (cp < 0 || test(cp) === negate) break
1312
- i += width(cp)
1313
- }
1314
- count++
1315
- }
1316
- if (count < min) return failAt(s, i)
1317
- s.pos = i
1318
- return used ? toText(input, start, i) : undefined
1319
- }
1291
+ const loop: TextLoop = { max, min, negate, accept, test, small: Math.min(fast, TableSize), fast, used }
1292
+ // One function per kind of unit read, from its own source: a loop
1293
+ // shared by every encoding sees several readers once a program
1294
+ // parses more than one, and stops inlining them
1295
+ if (unitWidth === 2) return takeWhileUtf16(loop, enc === utf16le ? 0 : 1, enc)
1296
+ return this.kind === 0 ? takeWhileString(loop, enc) : takeWhileBytes(loop, enc)
1320
1297
  }
1321
1298
  case "TakeUntil":
1322
1299
  case "Rest":
@@ -1341,17 +1318,15 @@ class Compiler {
1341
1318
  }
1342
1319
  }
1343
1320
  // `Text.taken` checks that the text is well formed, then decodes it
1344
- const fast = fastLimit(enc)
1321
+ const { fast, unitAt, width } = unitReader(this.kind, enc)
1345
1322
  const toText = enc.text
1346
- const unitAt: (input: any, i: number) => number =
1347
- this.kind === 0 ? (input: string, i) => input.charCodeAt(i) : (input: Uint8Array, i) => input[i]
1348
1323
  return (s) => {
1349
1324
  const start = s.pos
1350
1325
  if (c(s) === Fail) return Fail
1351
1326
  const end = s.pos
1352
1327
  const input = s.input
1353
1328
  let i = start
1354
- while (i < end && unitAt(input, i) < fast) i++
1329
+ while (i < end && unitAt(input, i, end) < fast) i += width
1355
1330
  if (i < end && validate(enc, s, i, end) < 0) return fail(s)
1356
1331
  return used ? toText(input, start, end) : undefined
1357
1332
  }
@@ -1364,11 +1339,7 @@ class Compiler {
1364
1339
  */
1365
1340
  private float(node: NodeOf<"Map">): Step | undefined {
1366
1341
  if (!floatTexts.has(node.self) || this.kind === 2) return undefined
1367
- let taken = nodeOf(node.self)
1368
- while (taken._tag === "Expected") taken = nodeOf(taken.self)
1369
- if (taken._tag !== "Taken") return undefined
1370
- const enc = taken.encoding
1371
- if (enc !== undefined && !(this.native(enc) && enc.unit === 1 && enc.asciiCompatible)) return undefined
1342
+ if (!scansFloat(node.self, (enc) => this.native(enc))) return undefined
1372
1343
  const c = this.compile(node.self, true)
1373
1344
  const scan = (this.kind === 0 ? scanFloatString : scanFloatBytes) as (
1374
1345
  input: unknown,
@@ -1415,6 +1386,15 @@ class Compiler {
1415
1386
  }
1416
1387
  return false
1417
1388
  }
1389
+ } else if (ctrl < unitReader(kind, enc).fast) {
1390
+ const { unitAt, width } = unitReader(kind, enc)
1391
+ control = (s) => {
1392
+ if (s.pos < s.end && unitAt(s.input, s.pos, s.end) === ctrl) {
1393
+ s.pos += width
1394
+ return true
1395
+ }
1396
+ return false
1397
+ }
1418
1398
  } else {
1419
1399
  control = (s) => {
1420
1400
  if (s.pos >= s.end) return false
@@ -1425,7 +1405,7 @@ class Compiler {
1425
1405
  }
1426
1406
  }
1427
1407
  const resumable = this.resumable
1428
- const key = {}
1408
+ const key = new Key()
1429
1409
  /** Fails as incomplete, keeping a checkpoint in a streaming parse. */
1430
1410
  const incomplete = (s: State<any>, start: number, checkpoint: unknown): typeof Fail => {
1431
1411
  if (s.resume !== undefined) Resume.save(s, key, start, checkpoint)
@@ -1522,19 +1502,149 @@ const fail = (s: State<any>): typeof Fail => {
1522
1502
  }
1523
1503
 
1524
1504
  /** Fails, leaving the cursor at `pos`: how far a token parser read (see `partialSafe`). */
1505
+ /** The settings of a compiled `Text` `takeWhile` loop. */
1506
+ interface TextLoop {
1507
+ readonly max: number
1508
+ readonly min: number
1509
+ readonly negate: boolean
1510
+ /** 1 where a code below `small` is accepted (see `acceptTable`). */
1511
+ readonly accept: Uint8Array
1512
+ readonly test: (code: number) => boolean
1513
+ readonly small: number
1514
+ /** Units below it are whole code points. */
1515
+ readonly fast: number
1516
+ readonly used: boolean
1517
+ }
1518
+
1519
+ // `takeWhile` for each kind of unit read. The three loops differ only in
1520
+ // how they read a unit; they are written out because a single loop calling
1521
+ // a reader, or branching on one, measured slower (see `text`).
1522
+
1523
+ const takeWhileString = (loop: TextLoop, enc: Encoding<string>): Step => {
1524
+ const { accept, fast, max, min, negate, small, test, used } = loop
1525
+ const { decode, text, width } = enc
1526
+ return (s) => {
1527
+ const input: string = s.input
1528
+ const start = s.pos
1529
+ const end = s.end
1530
+ let i = start
1531
+ let count = 0
1532
+ while (count < max && i < end) {
1533
+ const c = input.charCodeAt(i)
1534
+ if (c < small) {
1535
+ if (accept[c] === 0) break
1536
+ i++
1537
+ } else if (c < fast) {
1538
+ if (test(c) === negate) break
1539
+ i++
1540
+ } else {
1541
+ const cp = decode(input, i, end)
1542
+ if (cp < 0 || test(cp) === negate) break
1543
+ i += width(cp)
1544
+ }
1545
+ count++
1546
+ }
1547
+ if (count < min) return failAt(s, i)
1548
+ s.pos = i
1549
+ return used ? text(input, start, i) : undefined
1550
+ }
1551
+ }
1552
+
1553
+ const takeWhileBytes = (loop: TextLoop, enc: Encoding<Uint8Array>): Step => {
1554
+ const { accept, max, min, negate, small, test, used } = loop
1555
+ const { decode, text, width } = enc
1556
+ return (s) => {
1557
+ const input: Uint8Array = s.input
1558
+ const start = s.pos
1559
+ const end = s.end
1560
+ let i = start
1561
+ let count = 0
1562
+ while (count < max && i < end) {
1563
+ const c = input[i]
1564
+ // A byte below `fast` is below `small` too, as `small` is `fast` or the table size
1565
+ if (c < small) {
1566
+ if (accept[c] === 0) break
1567
+ i++
1568
+ } else {
1569
+ const cp = decode(input, i, end)
1570
+ if (cp < 0 || test(cp) === negate) break
1571
+ i += width(cp)
1572
+ }
1573
+ count++
1574
+ }
1575
+ if (count < min) return failAt(s, i)
1576
+ s.pos = i
1577
+ return used ? text(input, start, i) : undefined
1578
+ }
1579
+ }
1580
+
1581
+ /** For UTF-16 bytes; `lo` is the offset of a unit's low byte (0 for little endian). */
1582
+ const takeWhileUtf16 = (loop: TextLoop, lo: 0 | 1, enc: Encoding<Uint8Array>): Step => {
1583
+ const { accept, fast, max, min, negate, small, test, used } = loop
1584
+ const { decode, text, width } = enc
1585
+ const hi = 1 - lo
1586
+ return (s) => {
1587
+ const input: Uint8Array = s.input
1588
+ const start = s.pos
1589
+ const end = s.end
1590
+ let i = start
1591
+ let count = 0
1592
+ while (count < max && i < end) {
1593
+ // A unit cut off by `end` reads as 0xFFFF, so `decode` reports it
1594
+ const c = i + 1 < end ? input[i + lo] | (input[i + hi] << 8) : 0xffff
1595
+ if (c < fast ? (c < small ? accept[c] === 0 : test(c) === negate) : false) break
1596
+ if (c < fast) i += 2
1597
+ else {
1598
+ const cp = decode(input, i, end)
1599
+ if (cp < 0 || test(cp) === negate) break
1600
+ i += width(cp)
1601
+ }
1602
+ count++
1603
+ }
1604
+ if (count < min) return failAt(s, i)
1605
+ s.pos = i
1606
+ return used ? text(input, start, i) : undefined
1607
+ }
1608
+ }
1609
+
1610
+ /**
1611
+ * How a `Text` parser reads its encoding's code units: a unit below `fast`
1612
+ * is a whole code point, `width` input units long. UTF-16 bytes get a unit
1613
+ * reader, as in the JIT; a unit cut off by `end` reads as 0xFFFF, so it
1614
+ * takes the `decode` path, which reports it truncated.
1615
+ */
1616
+ interface UnitReader {
1617
+ readonly width: 1 | 2
1618
+ readonly fast: number
1619
+ readonly unitAt: (input: any, i: number, end: number) => number
1620
+ }
1621
+
1622
+ const unitReader = (kind: 0 | 1 | 2, enc: Encoding<any>): UnitReader => {
1623
+ if (kind === 1 && (enc === utf16le || enc === utf16be)) {
1624
+ return {
1625
+ width: 2,
1626
+ fast: 0xd800,
1627
+ unitAt:
1628
+ enc === utf16le
1629
+ ? (input: Uint8Array, i, end) => (i + 1 < end ? input[i] | (input[i + 1] << 8) : 0xffff)
1630
+ : (input: Uint8Array, i, end) => (i + 1 < end ? (input[i] << 8) | input[i + 1] : 0xffff)
1631
+ }
1632
+ }
1633
+ // `end` is declared though unused: calls with more arguments than
1634
+ // parameters measured 1-2% slower on the JSON benchmark
1635
+ return {
1636
+ width: 1,
1637
+ fast: fastLimit(enc),
1638
+ unitAt: kind === 0 ? (input: string, i, _end) => input.charCodeAt(i) : (input: Uint8Array, i, _end) => input[i]
1639
+ }
1640
+ }
1641
+
1525
1642
  const failAt = (s: State<any>, pos: number): typeof Fail => {
1526
1643
  s.pos = pos
1527
1644
  s.mode = BT
1528
1645
  return Fail
1529
1646
  }
1530
1647
 
1531
- /** 1 where a small code is accepted (taking `negate` into account). */
1532
- const acceptTable = (set: CharSet, negate: boolean): Uint8Array => {
1533
- const t = new Uint8Array(TableSize)
1534
- for (let c = 0; c < TableSize; c++) t[c] = (set.table[c] === 1) !== negate ? 1 : 0
1535
- return t
1536
- }
1537
-
1538
1648
  /** A literal matched against a string input, returning `value`. */
1539
1649
  const stringLiteral = (str: string, value: unknown): Step => {
1540
1650
  const len = str.length
@@ -1592,12 +1702,14 @@ export const createClosureCompiler = (kind: 0 | 1 | 2, resumable = false): Closu
1592
1702
  }
1593
1703
  }
1594
1704
 
1705
+ /** A compiled root parser, with the parts of it that run interpreted. */
1706
+ export interface CompiledRoot {
1707
+ readonly step: Step
1708
+ readonly fallbacks: ReadonlyArray<Fallback>
1709
+ }
1710
+
1595
1711
  /** Compiles a root parser for one input kind. */
1596
- const compileClosure = (
1597
- root: ParserLike,
1598
- kind: 0 | 1 | 2,
1599
- resumable = false
1600
- ): { readonly step: Step; readonly fallbacks: ReadonlyArray<Fallback> } => {
1712
+ const compileClosure = (root: ParserLike, kind: 0 | 1 | 2, resumable = false): CompiledRoot => {
1601
1713
  const compiler = new Compiler(kind, resumable)
1602
1714
  const step = compiler.compile(root, true)
1603
1715
  return { step, fallbacks: compiler.fallbacks }
@@ -1605,12 +1717,12 @@ const compileClosure = (
1605
1717
 
1606
1718
  /** The closure compiler as a backend. */
1607
1719
  export const backend: Backend = {
1608
- build: (p, kind, resumable) => {
1609
- try {
1610
- const { fallbacks, step } = compileClosure(p, kind, resumable)
1611
- return { backend: "closure", step, fallbacks, error: undefined }
1612
- } catch (error) {
1613
- return { backend: undefined, step: undefined, fallbacks: [], error }
1614
- }
1615
- }
1720
+ build: (p, kind, resumable) =>
1721
+ Result.match(
1722
+ Result.try(() => compileClosure(p, kind, resumable)),
1723
+ {
1724
+ onSuccess: ({ fallbacks, step }) => ({ backend: "closure" as const, step, fallbacks, error: undefined }),
1725
+ onFailure: (error) => ({ backend: undefined, step: undefined, fallbacks: [], error })
1726
+ }
1727
+ )
1616
1728
  }