@thomasfosterau/effect-parse 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +550 -2
  3. package/dist/Ascii.d.ts +392 -0
  4. package/dist/Ascii.d.ts.map +1 -0
  5. package/dist/Ascii.js +610 -0
  6. package/dist/Ascii.js.map +1 -0
  7. package/dist/Binary.d.ts +153 -0
  8. package/dist/Binary.d.ts.map +1 -0
  9. package/dist/Binary.js +171 -0
  10. package/dist/Binary.js.map +1 -0
  11. package/dist/Bits.d.ts +60 -0
  12. package/dist/Bits.d.ts.map +1 -0
  13. package/dist/Bits.js +144 -0
  14. package/dist/Bits.js.map +1 -0
  15. package/dist/Encoding.d.ts +116 -0
  16. package/dist/Encoding.d.ts.map +1 -0
  17. package/dist/Encoding.js +85 -0
  18. package/dist/Encoding.js.map +1 -0
  19. package/dist/Expression.d.ts +124 -0
  20. package/dist/Expression.d.ts.map +1 -0
  21. package/dist/Expression.js +139 -0
  22. package/dist/Expression.js.map +1 -0
  23. package/dist/Input.d.ts +301 -0
  24. package/dist/Input.d.ts.map +1 -0
  25. package/dist/Input.js +312 -0
  26. package/dist/Input.js.map +1 -0
  27. package/dist/Nom.d.ts +518 -0
  28. package/dist/Nom.d.ts.map +1 -0
  29. package/dist/Nom.js +519 -0
  30. package/dist/Nom.js.map +1 -0
  31. package/dist/ParseError.d.ts +120 -0
  32. package/dist/ParseError.d.ts.map +1 -0
  33. package/dist/ParseError.js +301 -0
  34. package/dist/ParseError.js.map +1 -0
  35. package/dist/Parser.d.ts +1312 -0
  36. package/dist/Parser.d.ts.map +1 -0
  37. package/dist/Parser.js +1936 -0
  38. package/dist/Parser.js.map +1 -0
  39. package/dist/ParserSchema.d.ts +31 -0
  40. package/dist/ParserSchema.d.ts.map +1 -0
  41. package/dist/ParserSchema.js +56 -0
  42. package/dist/ParserSchema.js.map +1 -0
  43. package/dist/ParserStream.d.ts +127 -0
  44. package/dist/ParserStream.d.ts.map +1 -0
  45. package/dist/ParserStream.js +270 -0
  46. package/dist/ParserStream.js.map +1 -0
  47. package/dist/Text.d.ts +330 -0
  48. package/dist/Text.d.ts.map +1 -0
  49. package/dist/Text.js +529 -0
  50. package/dist/Text.js.map +1 -0
  51. package/dist/Token.d.ts +225 -0
  52. package/dist/Token.d.ts.map +1 -0
  53. package/dist/Token.js +294 -0
  54. package/dist/Token.js.map +1 -0
  55. package/dist/index.d.ts +17 -0
  56. package/dist/index.d.ts.map +1 -0
  57. package/dist/index.js +17 -0
  58. package/dist/index.js.map +1 -0
  59. package/dist/internal/compiler/closure.d.ts +25 -0
  60. package/dist/internal/compiler/closure.d.ts.map +1 -0
  61. package/dist/internal/compiler/closure.js +1668 -0
  62. package/dist/internal/compiler/closure.js.map +1 -0
  63. package/dist/internal/compiler/entries.d.ts +19 -0
  64. package/dist/internal/compiler/entries.d.ts.map +1 -0
  65. package/dist/internal/compiler/entries.js +32 -0
  66. package/dist/internal/compiler/entries.js.map +1 -0
  67. package/dist/internal/compiler/first.d.ts +33 -0
  68. package/dist/internal/compiler/first.d.ts.map +1 -0
  69. package/dist/internal/compiler/first.js +235 -0
  70. package/dist/internal/compiler/first.js.map +1 -0
  71. package/dist/internal/compiler/jit.d.ts +20 -0
  72. package/dist/internal/compiler/jit.d.ts.map +1 -0
  73. package/dist/internal/compiler/jit.js +1030 -0
  74. package/dist/internal/compiler/jit.js.map +1 -0
  75. package/dist/internal/compiler/registry.d.ts +34 -0
  76. package/dist/internal/compiler/registry.d.ts.map +1 -0
  77. package/dist/internal/compiler/registry.js +46 -0
  78. package/dist/internal/compiler/registry.js.map +1 -0
  79. package/dist/internal/compiler/sets.d.ts +29 -0
  80. package/dist/internal/compiler/sets.d.ts.map +1 -0
  81. package/dist/internal/compiler/sets.js +85 -0
  82. package/dist/internal/compiler/sets.js.map +1 -0
  83. package/dist/internal/entries.d.ts +27 -0
  84. package/dist/internal/entries.d.ts.map +1 -0
  85. package/dist/internal/entries.js +42 -0
  86. package/dist/internal/entries.js.map +1 -0
  87. package/dist/internal/node.d.ts +251 -0
  88. package/dist/internal/node.d.ts.map +1 -0
  89. package/dist/internal/node.js +73 -0
  90. package/dist/internal/node.js.map +1 -0
  91. package/dist/internal/numbers.d.ts +41 -0
  92. package/dist/internal/numbers.d.ts.map +1 -0
  93. package/dist/internal/numbers.js +302 -0
  94. package/dist/internal/numbers.js.map +1 -0
  95. package/dist/internal/resume.d.ts +39 -0
  96. package/dist/internal/resume.d.ts.map +1 -0
  97. package/dist/internal/resume.js +37 -0
  98. package/dist/internal/resume.js.map +1 -0
  99. package/dist/internal/text.d.ts +72 -0
  100. package/dist/internal/text.d.ts.map +1 -0
  101. package/dist/internal/text.js +391 -0
  102. package/dist/internal/text.js.map +1 -0
  103. package/dist/internal/tokens.d.ts +93 -0
  104. package/dist/internal/tokens.d.ts.map +1 -0
  105. package/dist/internal/tokens.js +400 -0
  106. package/dist/internal/tokens.js.map +1 -0
  107. package/package.json +115 -3
  108. package/src/Ascii.ts +741 -0
  109. package/src/Binary.ts +228 -0
  110. package/src/Bits.ts +192 -0
  111. package/src/Encoding.ts +128 -0
  112. package/src/Expression.ts +245 -0
  113. package/src/Input.ts +429 -0
  114. package/src/Nom.ts +642 -0
  115. package/src/ParseError.ts +378 -0
  116. package/src/Parser.ts +2763 -0
  117. package/src/ParserSchema.ts +87 -0
  118. package/src/ParserStream.ts +390 -0
  119. package/src/Text.ts +757 -0
  120. package/src/Token.ts +416 -0
  121. package/src/index.ts +16 -0
  122. package/src/internal/compiler/closure.ts +1583 -0
  123. package/src/internal/compiler/entries.ts +36 -0
  124. package/src/internal/compiler/first.ts +252 -0
  125. package/src/internal/compiler/jit.ts +1091 -0
  126. package/src/internal/compiler/registry.ts +79 -0
  127. package/src/internal/compiler/sets.ts +100 -0
  128. package/src/internal/entries.ts +51 -0
  129. package/src/internal/node.ts +207 -0
  130. package/src/internal/numbers.ts +291 -0
  131. package/src/internal/resume.ts +55 -0
  132. package/src/internal/text.ts +411 -0
  133. package/src/internal/tokens.ts +409 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Thomas Foster
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,551 @@
1
- # Temporary Holding Version
1
+ # effect-parse
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Parser combinators for [Effect](https://effect.website), ported from Rust's
4
+ [nom](https://github.com/rust-bakery/nom) and
5
+ [winnow](https://github.com/winnow-rs/winnow).
6
+
7
+ effect-parse keeps the core model of nom and winnow: small parsers composed
8
+ into larger ones; `Backtrack` / `Cut` / `Incomplete` error modes; one set of
9
+ parsers for both complete and streaming input; text, bytes, bits and token
10
+ streams. On top of that it follows Effect's conventions:
11
+
12
+ - pipeable, dual (data-first or data-last) combinators
13
+ - generator syntax with `Parser.gen`
14
+ - typed user errors in the `E` channel, and `ParseError` as a `Data.TaggedError`
15
+ - dependency injection with `Context` services in the `R` channel
16
+ - `Option`, `Result`, `Effect`, `Stream` and `Schema` integration
17
+
18
+ ```ts
19
+ import { Ascii, Parser, Token } from "@thomasfosterau/effect-parse"
20
+
21
+ const point = Parser.gen(function*() {
22
+ yield* Token.literal("(")
23
+ const x = yield* Ascii.decInt
24
+ yield* Token.literal(",")
25
+ const y = yield* Ascii.decInt
26
+ yield* Token.literal(")")
27
+ return { x, y }
28
+ })
29
+
30
+ Parser.parse(point, "(1,-2)")
31
+ // Result.succeed({ x: 1, y: -2 })
32
+
33
+ Parser.parse(point, "(1;-2)")
34
+ // Result.fail(ParseError):
35
+ // Parse error at line 1, column 3:
36
+ // 1 | (1;-2)
37
+ // | ^
38
+ // expected digit or ",", found ";"
39
+ ```
40
+
41
+ ## Installation
42
+
43
+ ```sh
44
+ pnpm add @thomasfosterau/effect-parse effect
45
+ ```
46
+
47
+ effect-parse requires `effect` 4.x (`^4.0.0`) as a peer dependency; the
48
+ consuming application owns the single copy of `effect`.
49
+
50
+ Every module is available from the root entry as a namespace, and also on its
51
+ own subpath (`@thomasfosterau/effect-parse/Parser`, `…/Ascii`, and so on).
52
+
53
+ The package was developed as `effect-nom`. Releases and the changelog are
54
+ managed with Changesets; see [PUBLISHING.md](PUBLISHING.md).
55
+
56
+ ## Contents
57
+
58
+ - [The `Parser` type](#the-parser-type)
59
+ - [Writing parsers](#writing-parsers)
60
+ - [Running parsers](#running-parsers)
61
+ - [Compiled parsers](#compiled-parsers)
62
+ - [Errors](#errors)
63
+ - [Streaming and partial input](#streaming-and-partial-input)
64
+ - [Services](#services)
65
+ - [Unicode text and encodings](#unicode-text-and-encodings)
66
+ - [Modules](#modules)
67
+ - [Coming from nom or winnow](#coming-from-nom-or-winnow)
68
+ - [Examples](#examples)
69
+ - [Design](#design)
70
+
71
+ ## The `Parser` type
72
+
73
+ ```ts
74
+ interface Parser<A, E = never, R = never, I extends Input = string>
75
+ ```
76
+
77
+ | Parameter | Meaning |
78
+ |---|---|
79
+ | `A` | the value produced |
80
+ | `E` | the type of user errors (see [Errors](#errors)); `never` for pure syntax |
81
+ | `R` | the services required (see [Services](#services)) |
82
+ | `I` | the input: `string` (the default), `Uint8Array`, or `ReadonlyArray<T>` |
83
+
84
+ Like winnow, a parser is a function over a mutable cursor. Failing is cheap,
85
+ and backtracking means resetting an offset. String tokens are Unicode code
86
+ points, `Uint8Array` tokens are bytes, and array tokens are elements, which
87
+ is useful for parsing the output of a lexer.
88
+
89
+ ## Writing parsers
90
+
91
+ There are three styles, and they mix freely.
92
+
93
+ **Combinators**, data-first or piped:
94
+
95
+ ```ts
96
+ import { Ascii, Parser, Token } from "@thomasfosterau/effect-parse"
97
+
98
+ const keyValue = Parser.separatedPair(
99
+ Ascii.alpha1,
100
+ Parser.delimited(Ascii.space0, Token.literal("="), Ascii.space0),
101
+ Ascii.decInt
102
+ )
103
+
104
+ const list = Parser.separated(Ascii.decInt, Token.literal(",")).pipe(
105
+ Parser.map((ns) => ns.reduce((a, b) => a + b, 0)),
106
+ Parser.context("list")
107
+ )
108
+ ```
109
+
110
+ **Generators**, like `Effect.gen`:
111
+
112
+ ```ts
113
+ const header = Parser.gen(function*() {
114
+ const name = yield* Token.takeWhile(Ascii.isAlphanumeric, { min: 1 })
115
+ yield* Token.literal(": ")
116
+ const value = yield* Ascii.tillLineEnding
117
+ yield* Ascii.lineEnding
118
+ return { name, value }
119
+ })
120
+ ```
121
+
122
+ **Records, tuples and Do notation**:
123
+
124
+ ```ts
125
+ const version = Parser.struct({
126
+ major: Ascii.decUint,
127
+ minor: Parser.preceded(Token.literal("."), Ascii.decUint)
128
+ })
129
+
130
+ const range = Parser.Do.pipe(
131
+ Parser.bind("start", () => Ascii.decUint),
132
+ Parser.zipLeft(Token.literal("..")),
133
+ Parser.bind("end", () => Ascii.decUint)
134
+ )
135
+ ```
136
+
137
+ Recursive grammars use `Parser.suspend`:
138
+
139
+ ```ts
140
+ type Tree = ReadonlyArray<Tree>
141
+ const tree: Parser.Parser<Tree> = Parser.delimited(
142
+ Token.literal("["),
143
+ Parser.separated(Parser.suspend(() => tree), Token.literal(",")),
144
+ Token.literal("]")
145
+ )
146
+ ```
147
+
148
+ Low-level parsers can be written directly against the cursor with
149
+ `Parser.make`, in the same way that any function can be a parser in winnow:
150
+
151
+ ```ts
152
+ const upper = Parser.make<string>((state) => {
153
+ const c = state.input.charCodeAt(state.pos)
154
+ if (c >= 65 && c <= 90) {
155
+ state.pos++
156
+ return String.fromCharCode(c)
157
+ }
158
+ return state.atEnd() ? state.eoi("Token", ["uppercase letter"], 1) : state.backtrack("Token", ["uppercase letter"])
159
+ })
160
+ ```
161
+
162
+ ### Combinator overview
163
+
164
+ | Category | Combinators |
165
+ |---|---|
166
+ | Constructors | `succeed`, `sync`, `empty`, `fail`, `failMessage`, `reject`, `todo`, `suspend`, `make`, `location`, `eof` |
167
+ | Mapping | `map`, `as`, `asVoid`, `asSome`, `tap`, `tryMap`, `tryMapUnknown`, `filterMap`, `filter`, `filterOrFail`, `mapParser`, `fromEntries` |
168
+ | Sequencing | `seq`, `struct`, `zip`, `zipWith`, `zipLeft`, `zipRight`, `preceded`, `terminated`, `delimited`, `separatedPair`, `flatMap`, `gen`, `Do`/`bind`/`bindTo`/`let` |
169
+ | Choice | `alt`, `orElse`, `orElseSucceed`, `dispatch`, `permutation` |
170
+ | Repetition | `repeat`, `repeatTill`, `separated`, `separatedFoldl1`, `separatedFoldr1`, `fold` |
171
+ | Control | `opt`, `optional`, `cond`, `peek`, `not`, `allConsuming` |
172
+ | Consumed input | `taken`, `withTaken`, `span`, `withSpan` |
173
+ | Errors | `cut`, `backtrack`, `complete`, `context`, `expected`, `mapError`, `orFail` |
174
+ | Services | `service`, `provideService`, `provide` |
175
+ | Recovery | `resumeAfter`, `retryAfter` |
176
+ | Debugging | `trace`, `named` |
177
+
178
+ Repetition ranges follow winnow: `Parser.repeat(p)` matches zero or more
179
+ times, `Parser.repeat(p, 3)` exactly three times, and
180
+ `Parser.repeat(p, { min: 1, max: 4 })` between one and four times. When
181
+ `min` is a positive literal, the result is typed as a non-empty array.
182
+
183
+ ## Running parsers
184
+
185
+ | Runner | Result |
186
+ |---|---|
187
+ | `Parser.parse(p, input)` | `Result<A, ParseError<E>>`; the whole input must be consumed |
188
+ | `Parser.parseSync(p, input)` | `A`; throws `ParseError<E>` |
189
+ | `Parser.parsePeek(p, input)` | `Result<[remaining, A], ParseError<E>>`, nom-style |
190
+ | `Parser.parsePartial(p, input)` | `Result<Done \| Incomplete, ParseError<E>>` for streaming input |
191
+ | `Parser.parseEffect(p, input)` | `Effect<A, ParseError<E>, R>`; services come from the Effect context |
192
+ | `Parser.parseRecoverable(p, input)` | `{ value: Option<A>, errors }` with error recovery enabled |
193
+ | `ParserStream.parseMany(p)` | `Stream<I> => Stream<A>` (see [Streaming](#streaming-and-partial-input)) |
194
+ | `ParserSchema.make(p)` | a `Schema` decoding strings with the parser |
195
+
196
+ Parsers over bytes (`Uint8Array`) also accept an `ArrayBuffer`, a
197
+ `SharedArrayBuffer`, a `DataView` or any other typed array, in the runners
198
+ and as `ParserStream` chunks. The data is viewed as a `Uint8Array` of the
199
+ same memory, not copied, so offsets are byte offsets and slices are
200
+ `Uint8Array`s:
201
+
202
+ ```ts
203
+ const response = await fetch(url)
204
+ Parser.parse(Text.utf8.rest, await response.arrayBuffer())
205
+ Parser.parse(Binary.beU32, new DataView(buffer, 4))
206
+ ```
207
+
208
+ Wider typed arrays are viewed in the platform's byte order, which is little
209
+ endian on all common platforms. So a `Uint16Array` of UTF-16 code units can
210
+ be parsed with `Text.utf16le`. `Input.fromSource` performs the same
211
+ conversion.
212
+
213
+ ## Compiled parsers
214
+
215
+ Combinators are flexible but run as a tree of closures. `Parser.compile`
216
+ turns a grammar into fused, specialized code, without changing how it is
217
+ written or what it returns:
218
+
219
+ ```ts
220
+ export const json = Parser.compile(Parser.preceded(ws, value))
221
+
222
+ Parser.parse(json, text) // same Result as before, about 5 times faster
223
+ ```
224
+
225
+ On the JSON example, compiled parsing runs at 5–6 times the speed of the
226
+ interpreter, for strings, UTF-8 and UTF-16 alike, and at 1–4 times the
227
+ time of `JSON.parse` depending on the document (see
228
+ [the design notes](docs/DESIGN.md#results)).
229
+
230
+ `compile` returns the same parser. The code is built on the first run, for
231
+ each kind of input, and cached. `parse`, `parseSync`, `parsePeek` and
232
+ `parseEffect` use it; partial input, `parseRecoverable` and tracing stay
233
+ interpreted. With `{ backend: "auto" }` (the default) it generates
234
+ JavaScript where `new Function` is allowed; `{ backend: "closure" }` never
235
+ generates code, for pages with a Content Security Policy.
236
+
237
+ Compiled code records no diagnostics. When it fails, the parser runs again
238
+ interpreted, so errors are exactly the same, byte for byte. Callbacks should
239
+ therefore be pure: on failing input they may run twice, and those whose
240
+ results are unused may be skipped. Parsers built with `Parser.make` or
241
+ `Parser.gen` run interpreted inside compiled code. See the
242
+ [design notes](docs/DESIGN.md#6-compiled-parsers) for details.
243
+
244
+ Some ways of writing a grammar compile to faster code than others:
245
+
246
+ - Build objects with `Parser.fromEntries` (or `Parser.map(Object.fromEntries)`)
247
+ directly on a `separated` or `repeat` of pairs: compiled code assigns the
248
+ properties as it parses, without the pairs and their array. A lambda that
249
+ calls `Object.fromEntries` can't be recognized.
250
+ - Prefer token sets (`"abc"`, `Token.range`, the `Ascii` predicates) to
251
+ arbitrary predicates: they compile to comparisons.
252
+ - Avoid `flatMap` on hot paths: the parser it returns runs interpreted.
253
+ - `Ascii.float` and `Text.float` convert numbers as they match them.
254
+
255
+ [`json-fast.ts`](examples/json-fast.ts) is the JSON example written this
256
+ way; it runs records-heavy documents about 1.5 times as fast as
257
+ [`json-text.ts`](examples/json-text.ts).
258
+
259
+ ## Errors
260
+
261
+ Failures have a mode, as in winnow:
262
+
263
+ - **Backtrack**: recoverable. `alt`, `opt`, `repeat` and friends try
264
+ something else.
265
+ - **Cut**: unrecoverable. Use `Parser.cut(p)` once a branch is certain
266
+ (e.g. after an opening `[`), so errors point at the real problem instead of
267
+ backtracking to an unhelpful alternative.
268
+ - **Incomplete**: partial input ran out (see below).
269
+
270
+ `ParseError` is a `Data.TaggedError("ParseError")`. It records the
271
+ `offset`, `kind`, `expected` set, `context` labels and an optional typed
272
+ `cause`, and its `message` points at the problem:
273
+
274
+ ```text
275
+ Parse error at line 1, column 4:
276
+ 1 | [1,,2]
277
+ | ^
278
+ while parsing array
279
+ expected value, found ","
280
+ ```
281
+
282
+ effect-parse tries hard to produce good messages without annotations:
283
+
284
+ - `alt` merges the expectations of failed branches:
285
+ `expected "true", "false" or "null"`.
286
+ - Optional and repeated parsers leave *hints*, so after
287
+ `repeat(digit)` a failure reads `expected digit or ")"`.
288
+ - The furthest failure wins (the PEG heuristic), so a value that fails
289
+ after a separator is reported where it failed.
290
+ - `Parser.expected("value")` names a rule; `Parser.context("array")` adds
291
+ "while parsing array".
292
+
293
+ User errors are typed. `Parser.fail(e)`, `Parser.tryMap(f)` and
294
+ `Parser.filterOrFail(pred, f)` add their error type to `E`, and the error
295
+ arrives as `ParseError.cause`:
296
+
297
+ ```ts
298
+ class OutOfRange extends Data.TaggedError("OutOfRange")<{ value: number }> {}
299
+
300
+ const port = Ascii.decUint.pipe(
301
+ Parser.filterOrFail((n) => n < 65536, (value) => new OutOfRange({ value }))
302
+ )
303
+ // Parser<number, string | OutOfRange>
304
+ ```
305
+
306
+ ## Streaming and partial input
307
+
308
+ nom has separate `streaming` and `complete` modules. winnow marks the input
309
+ as `Partial`. In effect-parse, partiality is decided by the runner, so every
310
+ parser works in both modes:
311
+
312
+ ```ts
313
+ Parser.parsePartial(Ascii.digit1, "123")
314
+ // Result.succeed({ _tag: "Incomplete", needed: Option.some(1) }) -- more digits may follow
315
+ Parser.parsePartial(Ascii.digit1, "123;")
316
+ // Result.succeed({ _tag: "Done", value: "123", offset: 3 })
317
+ ```
318
+
319
+ `ParserStream.parseMany` connects this to Effect streams. It buffers chunks,
320
+ retries on `Incomplete`, and parses the tail in complete mode when the stream
321
+ ends:
322
+
323
+ ```ts
324
+ import { Effect, Stream } from "effect"
325
+ import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
326
+
327
+ const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
328
+
329
+ Stream.make("1\n2", "3\n", "4\n").pipe(ParserStream.parseMany(line), Stream.runCollect)
330
+ // Effect<[1, 23, 4], ParseError<string>>
331
+ ```
332
+
333
+ ### Parsing a download as it arrives
334
+
335
+ `ParserStream.parseAll` parses a stream as one document, and does not wait
336
+ for it to end: each chunk is parsed as it arrives. When the input runs out
337
+ mid-way, repetitions (`repeat`, `separated`, `fold`, `escaped`, ...)
338
+ remember how far they got, so the next attempt carries on from there rather
339
+ than starting over. `parseResponse` and `parseReadable` do the same for web
340
+ streams, so the body of a `fetch` response is parsed straight from its
341
+ bytes, without decoding it first or waiting for the download:
342
+
343
+ ```ts
344
+ import { Effect } from "effect"
345
+ import { Parser, ParserStream, Text } from "@thomasfosterau/effect-parse"
346
+
347
+ const json = Parser.compile(makeJson(Text.utf8)) // see examples/json-text.ts
348
+
349
+ const value = await Effect.runPromise(ParserStream.parseResponse(await fetch(url), json))
350
+ ```
351
+
352
+ The result, and any error, is the same as parsing the whole body at once.
353
+ Compiled parsers stream compiled. With the JSON example on a 2 MB document,
354
+ `parseResponse` returns about 1 ms after the last byte at 10 MB/s, and keeps
355
+ pace with `Response.json()` at 25 MB/s; on faster connections the native
356
+ parser is quicker (`pnpm bench:stream`).
357
+
358
+ ## Services
359
+
360
+ Parsers can require Effect services. These replace winnow's `Stateful`
361
+ input: configuration, symbol tables, indentation stacks and so on.
362
+
363
+ ```ts
364
+ import { Context, Effect } from "effect"
365
+
366
+ class Radix extends Context.Service<Radix, number>()("Radix") {}
367
+
368
+ const number = Parser.service(Radix).pipe(
369
+ Parser.flatMap((radix) => Ascii.alphanumeric1.pipe(Parser.map((s) => parseInt(s, radix))))
370
+ )
371
+ // Parser<number, never, Radix>
372
+
373
+ Parser.parse(Parser.provideService(number, Radix, 16), "ff") // 255
374
+ Parser.parseEffect(number, "777").pipe(Effect.provideService(Radix, 8)) // Effect<511>
375
+ ```
376
+
377
+ ## Unicode text and encodings
378
+
379
+ `Token` reads strings by code point, but the `Ascii` character classes and
380
+ numbers only cover ASCII, and they treat bytes as ASCII. The `Text` module
381
+ adds Unicode parsers that work with any encoding. `Text.make(encoding)`
382
+ builds a set of parsers (`Text.TextParsers<I>`), and sets are provided for
383
+ the common encodings:
384
+
385
+ | Parsers | Input | Offsets |
386
+ |---|---|---|
387
+ | `Text.string` | `string` | UTF-16 code units |
388
+ | `Text.utf8` | `Uint8Array` | bytes |
389
+ | `Text.utf16le`, `Text.utf16be` | `Uint8Array` | bytes |
390
+ | `Text.latin1` | `Uint8Array` | bytes |
391
+
392
+ Every set has the same members (`any`, `literal`, `caseless`, `oneOf`,
393
+ `takeWhile`, `takeUntil`, `alpha1`, `digit1`, `space1`, `identifier`,
394
+ `lineEnding`, `decInt`, `float`, `escaped`, `bom`, ...), and every parser
395
+ returns `string`s. A grammar written once against `TextParsers<I>`
396
+ therefore parses any of these inputs:
397
+
398
+ ```ts
399
+ import { Encoding, Parser, Text } from "@thomasfosterau/effect-parse"
400
+
401
+ const assignment = <I extends string | Uint8Array>(T: Text.TextParsers<I>) =>
402
+ Parser.separatedPair(T.identifier, Parser.delimited(T.space0, T.literal("≔"), T.space0), T.decInt)
403
+
404
+ Parser.parse(assignment(Text.string), "größe ≔ 42") // ["größe", 42]
405
+ Parser.parse(assignment(Text.utf8), new TextEncoder().encode("größe ≔ 42")) // ["größe", 42]
406
+ Parser.parse(assignment(Text.utf16le), Encoding.utf16le.encode("größe ≔ 42"))
407
+ ```
408
+
409
+ - **Unicode classes.** `alpha1` matches `"Ωμέγα"`, `digit1` matches
410
+ `"١٢٣"`, `space1` matches a no-break space, and `identifier` follows UAX
411
+ #31. The predicates (`Text.isLetter`, `Text.isWhitespace`,
412
+ `Text.isIdStart`, ...) also work with `Token`, and `Text.property("Script=Greek")`
413
+ tests any Unicode property. Numbers (`decInt`, `float`) take ASCII digits
414
+ only, as most languages do.
415
+ - **Bytes are decoded as they are parsed.** Offsets stay byte offsets, and
416
+ `Parser.taken` still returns a zero-copy `Uint8Array`. On partial input,
417
+ a character split across chunks reports `Incomplete`, so
418
+ `ParserStream.parseMany` parses a byte stream as text without decoding it
419
+ first.
420
+ - **Decoding is strict.** Invalid UTF-8 and unpaired surrogates never match.
421
+ Errors on encoded bytes show the input as text:
422
+
423
+ ```text
424
+ Parse error at line 1, column 3 (byte 2):
425
+ 1 | ab�c
426
+ | ^
427
+ expected letter or end of input, found invalid UTF-8 0xff
428
+ ```
429
+
430
+ - **Byte order marks.** `Encoding.sniff(bytes)` picks an encoding from a BOM,
431
+ and `T.bom` skips one.
432
+ - **Binary data.** The byte parsers accept `ArrayBuffer`s, `DataView`s and
433
+ typed arrays too (see [Running parsers](#running-parsers)).
434
+ - **Custom encodings.** `Text.make` accepts any `Encoding.Encoding`: a
435
+ `decode` function plus a few properties.
436
+
437
+ JavaScript strings are parsed natively, and the `Text` parsers only decode
438
+ byte input. Converting strings to bytes up front would be slower: a
439
+ `TextDecoder` call to produce each slice costs several times more than
440
+ `String.prototype.slice`. `pnpm bench` compares the paths with a JSON grammar
441
+ written once for every encoding ([`json-text.ts`](examples/json-text.ts)).
442
+ In our runs `Text.string` was roughly 5–15% slower than the `Token`/`Ascii`
443
+ parsers, and parsing UTF-8 directly was within about 10% of decoding it
444
+ first.
445
+
446
+ ## Modules
447
+
448
+ | Module | Contents | winnow / nom equivalent |
449
+ |---|---|---|
450
+ | `Parser` | the type, combinators, runners | `Parser` trait, `combinator` / `combinator`, `branch`, `multi`, `sequence` |
451
+ | `Token` | `any`, `literal`, `oneOf`, `noneOf`, `take`, `takeWhile`, `takeTill`, `takeUntil`, `rest`, `range` | `token` / `bytes` |
452
+ | `Text` | Unicode parsers for any encoding: `string`, `utf8`, `utf16le`, `utf16be`, `latin1`, `make`; Unicode predicates | — |
453
+ | `Encoding` | text encodings for `Text`, `sniff` | — |
454
+ | `Ascii` | character classes, `lineEnding`, `decInt`, `decUint`, `hexUint`, `float`, `caseless`, `takeEscaped`, `escaped`, `Ascii.bytes.*` | `ascii` / `character`, `number` |
455
+ | `Binary` | `u8`, `beU16` … `leI128`, floats, `lengthTake`, `lengthRepeat`, `lengthAndThen` | `binary` / `number`, `multi::length_*` |
456
+ | `Bits` | `bits`, `bytes`, `take`, `takeBig`, `pattern`, `bool` | `binary::bits` / `bits` |
457
+ | `Expression` | Pratt parser with prefix, postfix and infix operators | `combinator::expression` / `nom-language::precedence` |
458
+ | `ParseError` | the error type, `lineColumn`, `formatExpected` | `error` |
459
+ | `ParserStream` | `parseMany`, `parseAll` over Effect `Stream`s | `Partial` / `streaming` |
460
+ | `ParserSchema` | `make`: a `Schema` from a parser | — |
461
+ | `Nom` | nom names and argument orders (`tag`, `many0`, `separatedList1`, `mapRes`, …) | nom |
462
+ | `Input` | input types and the `State` cursor for `Parser.make` | `stream` |
463
+
464
+ Token parsers that cannot infer their input type take a type argument that
465
+ defaults to `string`, like a Rust turbofish:
466
+
467
+ ```ts
468
+ Token.take(4) // Parser<string>
469
+ Token.take<Uint8Array>(4) // Parser<Uint8Array, never, never, Uint8Array>
470
+ Token.oneOf<ReadonlyArray<Tok>>((t) => t.kind === "Number")
471
+ ```
472
+
473
+ ## Coming from nom or winnow
474
+
475
+ | nom | winnow | effect-parse |
476
+ |---|---|---|
477
+ | `tag("x")` | `"x"`, `literal("x")` | `Token.literal("x")` |
478
+ | `char('x')` | `'x'` | `Token.literal("x")` |
479
+ | `alt((a, b))` | `alt((a, b))` | `Parser.alt(a, b)` |
480
+ | `(a, b)` / `tuple` | `(a, b)` | `Parser.seq(a, b)` |
481
+ | — | `seq!(S { a: .., b: .. })` | `Parser.struct({ a, b })` |
482
+ | `many0(p)` | `repeat(0.., p)` | `Parser.repeat(p)` |
483
+ | `many1(p)` | `repeat(1.., p)` | `Parser.repeat(p, { min: 1 })` |
484
+ | `count(p, n)` | `repeat(n, p)` | `Parser.repeat(p, n)` |
485
+ | `separated_list0(s, p)` | `separated(0.., p, s)` | `Parser.separated(p, s)` |
486
+ | `fold_many0(p, i, f)` | `repeat(0.., p).fold(i, f)` | `Parser.fold(p, { init, f })` |
487
+ | `many_till(p, e)` | `repeat_till(0.., p, e)` | `Parser.repeatTill(p, e)` |
488
+ | `take_while1(f)` | `take_while(1.., f)` | `Token.takeWhile(f, { min: 1 })` |
489
+ | `map(p, f)` | `p.map(f)` | `Parser.map(p, f)` / `p.pipe(Parser.map(f))` |
490
+ | `map_res(p, f)` | `p.try_map(f)` | `Parser.tryMap(p, f)` (`f` returns a `Result`) |
491
+ | `map_opt(p, f)` | `p.verify_map(f)` | `Parser.filterMap(p, f)` (`f` returns an `Option`) |
492
+ | `verify(p, f)` | `p.verify(f)` | `Parser.filter(p, f)` |
493
+ | `value(v, p)` | `p.value(v)` | `Parser.as(p, v)` |
494
+ | `recognize(p)` | `p.take()` | `Parser.taken(p)` |
495
+ | `consumed(p)` | `p.with_taken()` | `Parser.withTaken(p)` |
496
+ | `map_parser(p, q)` | `p.and_then(q)` | `Parser.mapParser(p, q)` |
497
+ | `flat_map(p, f)` | `p.flat_map(f)` | `Parser.flatMap(p, f)` |
498
+ | `cut(p)` | `cut_err(p)` | `Parser.cut(p)` |
499
+ | `context("x", p)` | `p.context(StrContext::Label("x"))` | `Parser.context(p, "x")` |
500
+ | `complete(p)` | `p.complete_err()` | `Parser.complete(p)` |
501
+ | `IResult` | `parse_peek` | `Parser.parsePeek` |
502
+ | `streaming::*` | `Partial<I>` | `Parser.parsePartial`, `ParserStream` |
503
+ | — | `dispatch!` | `Parser.dispatch(selector, cases)` |
504
+ | — | `Stateful<I, S>` | `Parser.service` + Effect `Context` |
505
+ | — | `LocatingSlice` | `Parser.span`, `Parser.withSpan`, `Parser.location` |
506
+ | — | `resume_after` | `Parser.resumeAfter` + `Parser.parseRecoverable` |
507
+
508
+ The `Nom` module provides the nom names directly, with nom's argument
509
+ orders, for mechanical ports.
510
+
511
+ ## Examples
512
+
513
+ [`examples/`](examples) contains ports of winnow's examples, each with tests
514
+ in [`test/examples.test.ts`](test/examples.test.ts):
515
+
516
+ - [`json.ts`](examples/json.ts): JSON with `dispatch`, `cut` and escapes
517
+ - [`json-text.ts`](examples/json-text.ts): the same grammar written once
518
+ against `Text.TextParsers`, for strings, UTF-8 and UTF-16
519
+ - [`json-fast.ts`](examples/json-fast.ts): the same grammar written for
520
+ compiled parsers, with `Parser.fromEntries`
521
+ - [`arithmetic.ts`](examples/arithmetic.ts): direct evaluation, an AST via
522
+ the Pratt parser, and a lexer plus a parser over the token array
523
+ - [`ini.ts`](examples/ini.ts): INI files with a generator-style parser
524
+ - [`http.ts`](examples/http.ts): HTTP requests over bytes, complete and
525
+ streamed
526
+ - [`sexpr.ts`](examples/sexpr.ts): s-expressions with typed evaluation
527
+ errors
528
+
529
+ `pnpm build && pnpm bench` compares the JSON examples, on each encoding,
530
+ with `JSON.parse`.
531
+
532
+ ## Design
533
+
534
+ [`docs/DESIGN.md`](docs/DESIGN.md) records the research into nom, winnow and
535
+ Effect, and the reasoning behind each decision in this port.
536
+
537
+ ## Development
538
+
539
+ ```sh
540
+ pnpm install
541
+ pnpm check # type-check sources, tests and examples
542
+ pnpm test # run the test suite
543
+ pnpm build # emit dist/
544
+ pnpm bench # JSON example: interpreted vs compiled, per encoding (after a build)
545
+ pnpm bench:native # compiled JSON parser vs JSON.parse on several document shapes
546
+ pnpm bench:stream # parsing a fetch body as it downloads vs Response.json()
547
+ ```
548
+
549
+ ## License
550
+
551
+ MIT