@thomasfosterau/effect-parse 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +550 -2
- package/dist/Ascii.d.ts +392 -0
- package/dist/Ascii.d.ts.map +1 -0
- package/dist/Ascii.js +610 -0
- package/dist/Ascii.js.map +1 -0
- package/dist/Binary.d.ts +153 -0
- package/dist/Binary.d.ts.map +1 -0
- package/dist/Binary.js +171 -0
- package/dist/Binary.js.map +1 -0
- package/dist/Bits.d.ts +60 -0
- package/dist/Bits.d.ts.map +1 -0
- package/dist/Bits.js +144 -0
- package/dist/Bits.js.map +1 -0
- package/dist/Encoding.d.ts +116 -0
- package/dist/Encoding.d.ts.map +1 -0
- package/dist/Encoding.js +85 -0
- package/dist/Encoding.js.map +1 -0
- package/dist/Expression.d.ts +124 -0
- package/dist/Expression.d.ts.map +1 -0
- package/dist/Expression.js +139 -0
- package/dist/Expression.js.map +1 -0
- package/dist/Input.d.ts +301 -0
- package/dist/Input.d.ts.map +1 -0
- package/dist/Input.js +312 -0
- package/dist/Input.js.map +1 -0
- package/dist/Nom.d.ts +518 -0
- package/dist/Nom.d.ts.map +1 -0
- package/dist/Nom.js +519 -0
- package/dist/Nom.js.map +1 -0
- package/dist/ParseError.d.ts +120 -0
- package/dist/ParseError.d.ts.map +1 -0
- package/dist/ParseError.js +301 -0
- package/dist/ParseError.js.map +1 -0
- package/dist/Parser.d.ts +1312 -0
- package/dist/Parser.d.ts.map +1 -0
- package/dist/Parser.js +1936 -0
- package/dist/Parser.js.map +1 -0
- package/dist/ParserSchema.d.ts +31 -0
- package/dist/ParserSchema.d.ts.map +1 -0
- package/dist/ParserSchema.js +56 -0
- package/dist/ParserSchema.js.map +1 -0
- package/dist/ParserStream.d.ts +127 -0
- package/dist/ParserStream.d.ts.map +1 -0
- package/dist/ParserStream.js +270 -0
- package/dist/ParserStream.js.map +1 -0
- package/dist/Text.d.ts +330 -0
- package/dist/Text.d.ts.map +1 -0
- package/dist/Text.js +529 -0
- package/dist/Text.js.map +1 -0
- package/dist/Token.d.ts +225 -0
- package/dist/Token.d.ts.map +1 -0
- package/dist/Token.js +294 -0
- package/dist/Token.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/compiler/closure.d.ts +25 -0
- package/dist/internal/compiler/closure.d.ts.map +1 -0
- package/dist/internal/compiler/closure.js +1668 -0
- package/dist/internal/compiler/closure.js.map +1 -0
- package/dist/internal/compiler/entries.d.ts +19 -0
- package/dist/internal/compiler/entries.d.ts.map +1 -0
- package/dist/internal/compiler/entries.js +32 -0
- package/dist/internal/compiler/entries.js.map +1 -0
- package/dist/internal/compiler/first.d.ts +33 -0
- package/dist/internal/compiler/first.d.ts.map +1 -0
- package/dist/internal/compiler/first.js +235 -0
- package/dist/internal/compiler/first.js.map +1 -0
- package/dist/internal/compiler/jit.d.ts +20 -0
- package/dist/internal/compiler/jit.d.ts.map +1 -0
- package/dist/internal/compiler/jit.js +1030 -0
- package/dist/internal/compiler/jit.js.map +1 -0
- package/dist/internal/compiler/registry.d.ts +34 -0
- package/dist/internal/compiler/registry.d.ts.map +1 -0
- package/dist/internal/compiler/registry.js +46 -0
- package/dist/internal/compiler/registry.js.map +1 -0
- package/dist/internal/compiler/sets.d.ts +29 -0
- package/dist/internal/compiler/sets.d.ts.map +1 -0
- package/dist/internal/compiler/sets.js +85 -0
- package/dist/internal/compiler/sets.js.map +1 -0
- package/dist/internal/entries.d.ts +27 -0
- package/dist/internal/entries.d.ts.map +1 -0
- package/dist/internal/entries.js +42 -0
- package/dist/internal/entries.js.map +1 -0
- package/dist/internal/node.d.ts +251 -0
- package/dist/internal/node.d.ts.map +1 -0
- package/dist/internal/node.js +73 -0
- package/dist/internal/node.js.map +1 -0
- package/dist/internal/numbers.d.ts +41 -0
- package/dist/internal/numbers.d.ts.map +1 -0
- package/dist/internal/numbers.js +302 -0
- package/dist/internal/numbers.js.map +1 -0
- package/dist/internal/resume.d.ts +39 -0
- package/dist/internal/resume.d.ts.map +1 -0
- package/dist/internal/resume.js +37 -0
- package/dist/internal/resume.js.map +1 -0
- package/dist/internal/text.d.ts +72 -0
- package/dist/internal/text.d.ts.map +1 -0
- package/dist/internal/text.js +391 -0
- package/dist/internal/text.js.map +1 -0
- package/dist/internal/tokens.d.ts +93 -0
- package/dist/internal/tokens.d.ts.map +1 -0
- package/dist/internal/tokens.js +400 -0
- package/dist/internal/tokens.js.map +1 -0
- package/package.json +115 -3
- package/src/Ascii.ts +741 -0
- package/src/Binary.ts +228 -0
- package/src/Bits.ts +192 -0
- package/src/Encoding.ts +128 -0
- package/src/Expression.ts +245 -0
- package/src/Input.ts +429 -0
- package/src/Nom.ts +642 -0
- package/src/ParseError.ts +378 -0
- package/src/Parser.ts +2763 -0
- package/src/ParserSchema.ts +87 -0
- package/src/ParserStream.ts +390 -0
- package/src/Text.ts +757 -0
- package/src/Token.ts +416 -0
- package/src/index.ts +16 -0
- package/src/internal/compiler/closure.ts +1583 -0
- package/src/internal/compiler/entries.ts +36 -0
- package/src/internal/compiler/first.ts +252 -0
- package/src/internal/compiler/jit.ts +1091 -0
- package/src/internal/compiler/registry.ts +79 -0
- package/src/internal/compiler/sets.ts +100 -0
- package/src/internal/entries.ts +51 -0
- package/src/internal/node.ts +207 -0
- package/src/internal/numbers.ts +291 -0
- package/src/internal/resume.ts +55 -0
- package/src/internal/text.ts +411 -0
- package/src/internal/tokens.ts +409 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Thomas Foster
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,551 @@
|
|
|
1
|
-
#
|
|
1
|
+
# effect-parse
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Parser combinators for [Effect](https://effect.website), ported from Rust's
|
|
4
|
+
[nom](https://github.com/rust-bakery/nom) and
|
|
5
|
+
[winnow](https://github.com/winnow-rs/winnow).
|
|
6
|
+
|
|
7
|
+
effect-parse keeps the core model of nom and winnow: small parsers composed
|
|
8
|
+
into larger ones; `Backtrack` / `Cut` / `Incomplete` error modes; one set of
|
|
9
|
+
parsers for both complete and streaming input; text, bytes, bits and token
|
|
10
|
+
streams. On top of that it follows Effect's conventions:
|
|
11
|
+
|
|
12
|
+
- pipeable, dual (data-first or data-last) combinators
|
|
13
|
+
- generator syntax with `Parser.gen`
|
|
14
|
+
- typed user errors in the `E` channel, and `ParseError` as a `Data.TaggedError`
|
|
15
|
+
- dependency injection with `Context` services in the `R` channel
|
|
16
|
+
- `Option`, `Result`, `Effect`, `Stream` and `Schema` integration
|
|
17
|
+
|
|
18
|
+
```ts
|
|
19
|
+
import { Ascii, Parser, Token } from "@thomasfosterau/effect-parse"
|
|
20
|
+
|
|
21
|
+
const point = Parser.gen(function*() {
|
|
22
|
+
yield* Token.literal("(")
|
|
23
|
+
const x = yield* Ascii.decInt
|
|
24
|
+
yield* Token.literal(",")
|
|
25
|
+
const y = yield* Ascii.decInt
|
|
26
|
+
yield* Token.literal(")")
|
|
27
|
+
return { x, y }
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
Parser.parse(point, "(1,-2)")
|
|
31
|
+
// Result.succeed({ x: 1, y: -2 })
|
|
32
|
+
|
|
33
|
+
Parser.parse(point, "(1;-2)")
|
|
34
|
+
// Result.fail(ParseError):
|
|
35
|
+
// Parse error at line 1, column 3:
|
|
36
|
+
// 1 | (1;-2)
|
|
37
|
+
// | ^
|
|
38
|
+
// expected digit or ",", found ";"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Installation
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
pnpm add @thomasfosterau/effect-parse effect
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
effect-parse requires `effect` 4.x (`^4.0.0`) as a peer dependency; the
|
|
48
|
+
consuming application owns the single copy of `effect`.
|
|
49
|
+
|
|
50
|
+
Every module is available from the root entry as a namespace, and also on its
|
|
51
|
+
own subpath (`@thomasfosterau/effect-parse/Parser`, `…/Ascii`, and so on).
|
|
52
|
+
|
|
53
|
+
The package was developed as `effect-nom`. Releases and the changelog are
|
|
54
|
+
managed with Changesets; see [PUBLISHING.md](PUBLISHING.md).
|
|
55
|
+
|
|
56
|
+
## Contents
|
|
57
|
+
|
|
58
|
+
- [The `Parser` type](#the-parser-type)
|
|
59
|
+
- [Writing parsers](#writing-parsers)
|
|
60
|
+
- [Running parsers](#running-parsers)
|
|
61
|
+
- [Compiled parsers](#compiled-parsers)
|
|
62
|
+
- [Errors](#errors)
|
|
63
|
+
- [Streaming and partial input](#streaming-and-partial-input)
|
|
64
|
+
- [Services](#services)
|
|
65
|
+
- [Unicode text and encodings](#unicode-text-and-encodings)
|
|
66
|
+
- [Modules](#modules)
|
|
67
|
+
- [Coming from nom or winnow](#coming-from-nom-or-winnow)
|
|
68
|
+
- [Examples](#examples)
|
|
69
|
+
- [Design](#design)
|
|
70
|
+
|
|
71
|
+
## The `Parser` type
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
interface Parser<A, E = never, R = never, I extends Input = string>
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
| Parameter | Meaning |
|
|
78
|
+
|---|---|
|
|
79
|
+
| `A` | the value produced |
|
|
80
|
+
| `E` | the type of user errors (see [Errors](#errors)); `never` for pure syntax |
|
|
81
|
+
| `R` | the services required (see [Services](#services)) |
|
|
82
|
+
| `I` | the input: `string` (the default), `Uint8Array`, or `ReadonlyArray<T>` |
|
|
83
|
+
|
|
84
|
+
Like winnow, a parser is a function over a mutable cursor. Failing is cheap,
|
|
85
|
+
and backtracking means resetting an offset. String tokens are Unicode code
|
|
86
|
+
points, `Uint8Array` tokens are bytes, and array tokens are elements, which
|
|
87
|
+
is useful for parsing the output of a lexer.
|
|
88
|
+
|
|
89
|
+
## Writing parsers
|
|
90
|
+
|
|
91
|
+
There are three styles, and they mix freely.
|
|
92
|
+
|
|
93
|
+
**Combinators**, data-first or piped:
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import { Ascii, Parser, Token } from "@thomasfosterau/effect-parse"
|
|
97
|
+
|
|
98
|
+
const keyValue = Parser.separatedPair(
|
|
99
|
+
Ascii.alpha1,
|
|
100
|
+
Parser.delimited(Ascii.space0, Token.literal("="), Ascii.space0),
|
|
101
|
+
Ascii.decInt
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
const list = Parser.separated(Ascii.decInt, Token.literal(",")).pipe(
|
|
105
|
+
Parser.map((ns) => ns.reduce((a, b) => a + b, 0)),
|
|
106
|
+
Parser.context("list")
|
|
107
|
+
)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**Generators**, like `Effect.gen`:
|
|
111
|
+
|
|
112
|
+
```ts
|
|
113
|
+
const header = Parser.gen(function*() {
|
|
114
|
+
const name = yield* Token.takeWhile(Ascii.isAlphanumeric, { min: 1 })
|
|
115
|
+
yield* Token.literal(": ")
|
|
116
|
+
const value = yield* Ascii.tillLineEnding
|
|
117
|
+
yield* Ascii.lineEnding
|
|
118
|
+
return { name, value }
|
|
119
|
+
})
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Records, tuples and Do notation**:
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
const version = Parser.struct({
|
|
126
|
+
major: Ascii.decUint,
|
|
127
|
+
minor: Parser.preceded(Token.literal("."), Ascii.decUint)
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
const range = Parser.Do.pipe(
|
|
131
|
+
Parser.bind("start", () => Ascii.decUint),
|
|
132
|
+
Parser.zipLeft(Token.literal("..")),
|
|
133
|
+
Parser.bind("end", () => Ascii.decUint)
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Recursive grammars use `Parser.suspend`:
|
|
138
|
+
|
|
139
|
+
```ts
|
|
140
|
+
type Tree = ReadonlyArray<Tree>
|
|
141
|
+
const tree: Parser.Parser<Tree> = Parser.delimited(
|
|
142
|
+
Token.literal("["),
|
|
143
|
+
Parser.separated(Parser.suspend(() => tree), Token.literal(",")),
|
|
144
|
+
Token.literal("]")
|
|
145
|
+
)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Low-level parsers can be written directly against the cursor with
|
|
149
|
+
`Parser.make`, in the same way that any function can be a parser in winnow:
|
|
150
|
+
|
|
151
|
+
```ts
|
|
152
|
+
const upper = Parser.make<string>((state) => {
|
|
153
|
+
const c = state.input.charCodeAt(state.pos)
|
|
154
|
+
if (c >= 65 && c <= 90) {
|
|
155
|
+
state.pos++
|
|
156
|
+
return String.fromCharCode(c)
|
|
157
|
+
}
|
|
158
|
+
return state.atEnd() ? state.eoi("Token", ["uppercase letter"], 1) : state.backtrack("Token", ["uppercase letter"])
|
|
159
|
+
})
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### Combinator overview
|
|
163
|
+
|
|
164
|
+
| Category | Combinators |
|
|
165
|
+
|---|---|
|
|
166
|
+
| Constructors | `succeed`, `sync`, `empty`, `fail`, `failMessage`, `reject`, `todo`, `suspend`, `make`, `location`, `eof` |
|
|
167
|
+
| Mapping | `map`, `as`, `asVoid`, `asSome`, `tap`, `tryMap`, `tryMapUnknown`, `filterMap`, `filter`, `filterOrFail`, `mapParser`, `fromEntries` |
|
|
168
|
+
| Sequencing | `seq`, `struct`, `zip`, `zipWith`, `zipLeft`, `zipRight`, `preceded`, `terminated`, `delimited`, `separatedPair`, `flatMap`, `gen`, `Do`/`bind`/`bindTo`/`let` |
|
|
169
|
+
| Choice | `alt`, `orElse`, `orElseSucceed`, `dispatch`, `permutation` |
|
|
170
|
+
| Repetition | `repeat`, `repeatTill`, `separated`, `separatedFoldl1`, `separatedFoldr1`, `fold` |
|
|
171
|
+
| Control | `opt`, `optional`, `cond`, `peek`, `not`, `allConsuming` |
|
|
172
|
+
| Consumed input | `taken`, `withTaken`, `span`, `withSpan` |
|
|
173
|
+
| Errors | `cut`, `backtrack`, `complete`, `context`, `expected`, `mapError`, `orFail` |
|
|
174
|
+
| Services | `service`, `provideService`, `provide` |
|
|
175
|
+
| Recovery | `resumeAfter`, `retryAfter` |
|
|
176
|
+
| Debugging | `trace`, `named` |
|
|
177
|
+
|
|
178
|
+
Repetition ranges follow winnow: `Parser.repeat(p)` matches zero or more
|
|
179
|
+
times, `Parser.repeat(p, 3)` exactly three times, and
|
|
180
|
+
`Parser.repeat(p, { min: 1, max: 4 })` between one and four times. When
|
|
181
|
+
`min` is a positive literal, the result is typed as a non-empty array.
|
|
182
|
+
|
|
183
|
+
## Running parsers
|
|
184
|
+
|
|
185
|
+
| Runner | Result |
|
|
186
|
+
|---|---|
|
|
187
|
+
| `Parser.parse(p, input)` | `Result<A, ParseError<E>>`; the whole input must be consumed |
|
|
188
|
+
| `Parser.parseSync(p, input)` | `A`; throws `ParseError<E>` |
|
|
189
|
+
| `Parser.parsePeek(p, input)` | `Result<[remaining, A], ParseError<E>>`, nom-style |
|
|
190
|
+
| `Parser.parsePartial(p, input)` | `Result<Done \| Incomplete, ParseError<E>>` for streaming input |
|
|
191
|
+
| `Parser.parseEffect(p, input)` | `Effect<A, ParseError<E>, R>`; services come from the Effect context |
|
|
192
|
+
| `Parser.parseRecoverable(p, input)` | `{ value: Option<A>, errors }` with error recovery enabled |
|
|
193
|
+
| `ParserStream.parseMany(p)` | `Stream<I> => Stream<A>` (see [Streaming](#streaming-and-partial-input)) |
|
|
194
|
+
| `ParserSchema.make(p)` | a `Schema` decoding strings with the parser |
|
|
195
|
+
|
|
196
|
+
Parsers over bytes (`Uint8Array`) also accept an `ArrayBuffer`, a
|
|
197
|
+
`SharedArrayBuffer`, a `DataView` or any other typed array, in the runners
|
|
198
|
+
and as `ParserStream` chunks. The data is viewed as a `Uint8Array` of the
|
|
199
|
+
same memory, not copied, so offsets are byte offsets and slices are
|
|
200
|
+
`Uint8Array`s:
|
|
201
|
+
|
|
202
|
+
```ts
|
|
203
|
+
const response = await fetch(url)
|
|
204
|
+
Parser.parse(Text.utf8.rest, await response.arrayBuffer())
|
|
205
|
+
Parser.parse(Binary.beU32, new DataView(buffer, 4))
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Wider typed arrays are viewed in the platform's byte order, which is little
|
|
209
|
+
endian on all common platforms. So a `Uint16Array` of UTF-16 code units can
|
|
210
|
+
be parsed with `Text.utf16le`. `Input.fromSource` performs the same
|
|
211
|
+
conversion.
|
|
212
|
+
|
|
213
|
+
## Compiled parsers
|
|
214
|
+
|
|
215
|
+
Combinators are flexible but run as a tree of closures. `Parser.compile`
|
|
216
|
+
turns a grammar into fused, specialized code, without changing how it is
|
|
217
|
+
written or what it returns:
|
|
218
|
+
|
|
219
|
+
```ts
|
|
220
|
+
export const json = Parser.compile(Parser.preceded(ws, value))
|
|
221
|
+
|
|
222
|
+
Parser.parse(json, text) // same Result as before, about 5 times faster
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
On the JSON example, compiled parsing runs at 5–6 times the speed of the
|
|
226
|
+
interpreter, for strings, UTF-8 and UTF-16 alike, and at 1–4 times the
|
|
227
|
+
time of `JSON.parse` depending on the document (see
|
|
228
|
+
[the design notes](docs/DESIGN.md#results)).
|
|
229
|
+
|
|
230
|
+
`compile` returns the same parser. The code is built on the first run, for
|
|
231
|
+
each kind of input, and cached. `parse`, `parseSync`, `parsePeek` and
|
|
232
|
+
`parseEffect` use it; partial input, `parseRecoverable` and tracing stay
|
|
233
|
+
interpreted. With `{ backend: "auto" }` (the default) it generates
|
|
234
|
+
JavaScript where `new Function` is allowed; `{ backend: "closure" }` never
|
|
235
|
+
generates code, for pages with a Content Security Policy.
|
|
236
|
+
|
|
237
|
+
Compiled code records no diagnostics. When it fails, the parser runs again
|
|
238
|
+
interpreted, so errors are exactly the same, byte for byte. Callbacks should
|
|
239
|
+
therefore be pure: on failing input they may run twice, and those whose
|
|
240
|
+
results are unused may be skipped. Parsers built with `Parser.make` or
|
|
241
|
+
`Parser.gen` run interpreted inside compiled code. See the
|
|
242
|
+
[design notes](docs/DESIGN.md#6-compiled-parsers) for details.
|
|
243
|
+
|
|
244
|
+
Some ways of writing a grammar compile to faster code than others:
|
|
245
|
+
|
|
246
|
+
- Build objects with `Parser.fromEntries` (or `Parser.map(Object.fromEntries)`)
|
|
247
|
+
directly on a `separated` or `repeat` of pairs: compiled code assigns the
|
|
248
|
+
properties as it parses, without the pairs and their array. A lambda that
|
|
249
|
+
calls `Object.fromEntries` can't be recognized.
|
|
250
|
+
- Prefer token sets (`"abc"`, `Token.range`, the `Ascii` predicates) to
|
|
251
|
+
arbitrary predicates: they compile to comparisons.
|
|
252
|
+
- Avoid `flatMap` on hot paths: the parser it returns runs interpreted.
|
|
253
|
+
- `Ascii.float` and `Text.float` convert numbers as they match them.
|
|
254
|
+
|
|
255
|
+
[`json-fast.ts`](examples/json-fast.ts) is the JSON example written this
|
|
256
|
+
way; it runs records-heavy documents about 1.5 times as fast as
|
|
257
|
+
[`json-text.ts`](examples/json-text.ts).
|
|
258
|
+
|
|
259
|
+
## Errors
|
|
260
|
+
|
|
261
|
+
Failures have a mode, as in winnow:
|
|
262
|
+
|
|
263
|
+
- **Backtrack**: recoverable. `alt`, `opt`, `repeat` and friends try
|
|
264
|
+
something else.
|
|
265
|
+
- **Cut**: unrecoverable. Use `Parser.cut(p)` once a branch is certain
|
|
266
|
+
(e.g. after an opening `[`), so errors point at the real problem instead of
|
|
267
|
+
backtracking to an unhelpful alternative.
|
|
268
|
+
- **Incomplete**: partial input ran out (see below).
|
|
269
|
+
|
|
270
|
+
`ParseError` is a `Data.TaggedError("ParseError")`. It records the
|
|
271
|
+
`offset`, `kind`, `expected` set, `context` labels and an optional typed
|
|
272
|
+
`cause`, and its `message` points at the problem:
|
|
273
|
+
|
|
274
|
+
```text
|
|
275
|
+
Parse error at line 1, column 4:
|
|
276
|
+
1 | [1,,2]
|
|
277
|
+
| ^
|
|
278
|
+
while parsing array
|
|
279
|
+
expected value, found ","
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
effect-parse tries hard to produce good messages without annotations:
|
|
283
|
+
|
|
284
|
+
- `alt` merges the expectations of failed branches:
|
|
285
|
+
`expected "true", "false" or "null"`.
|
|
286
|
+
- Optional and repeated parsers leave *hints*, so after
|
|
287
|
+
`repeat(digit)` a failure reads `expected digit or ")"`.
|
|
288
|
+
- The furthest failure wins (the PEG heuristic), so a value that fails
|
|
289
|
+
after a separator is reported where it failed.
|
|
290
|
+
- `Parser.expected("value")` names a rule; `Parser.context("array")` adds
|
|
291
|
+
"while parsing array".
|
|
292
|
+
|
|
293
|
+
User errors are typed. `Parser.fail(e)`, `Parser.tryMap(f)` and
|
|
294
|
+
`Parser.filterOrFail(pred, f)` add their error type to `E`, and the error
|
|
295
|
+
arrives as `ParseError.cause`:
|
|
296
|
+
|
|
297
|
+
```ts
|
|
298
|
+
class OutOfRange extends Data.TaggedError("OutOfRange")<{ value: number }> {}
|
|
299
|
+
|
|
300
|
+
const port = Ascii.decUint.pipe(
|
|
301
|
+
Parser.filterOrFail((n) => n < 65536, (value) => new OutOfRange({ value }))
|
|
302
|
+
)
|
|
303
|
+
// Parser<number, string | OutOfRange>
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
## Streaming and partial input
|
|
307
|
+
|
|
308
|
+
nom has separate `streaming` and `complete` modules. winnow marks the input
|
|
309
|
+
as `Partial`. In effect-parse, partiality is decided by the runner, so every
|
|
310
|
+
parser works in both modes:
|
|
311
|
+
|
|
312
|
+
```ts
|
|
313
|
+
Parser.parsePartial(Ascii.digit1, "123")
|
|
314
|
+
// Result.succeed({ _tag: "Incomplete", needed: Option.some(1) }) -- more digits may follow
|
|
315
|
+
Parser.parsePartial(Ascii.digit1, "123;")
|
|
316
|
+
// Result.succeed({ _tag: "Done", value: "123", offset: 3 })
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
`ParserStream.parseMany` connects this to Effect streams. It buffers chunks,
|
|
320
|
+
retries on `Incomplete`, and parses the tail in complete mode when the stream
|
|
321
|
+
ends:
|
|
322
|
+
|
|
323
|
+
```ts
|
|
324
|
+
import { Effect, Stream } from "effect"
|
|
325
|
+
import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
|
|
326
|
+
|
|
327
|
+
const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
|
|
328
|
+
|
|
329
|
+
Stream.make("1\n2", "3\n", "4\n").pipe(ParserStream.parseMany(line), Stream.runCollect)
|
|
330
|
+
// Effect<[1, 23, 4], ParseError<string>>
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
### Parsing a download as it arrives
|
|
334
|
+
|
|
335
|
+
`ParserStream.parseAll` parses a stream as one document, and does not wait
|
|
336
|
+
for it to end: each chunk is parsed as it arrives. When the input runs out
|
|
337
|
+
mid-way, repetitions (`repeat`, `separated`, `fold`, `escaped`, ...)
|
|
338
|
+
remember how far they got, so the next attempt carries on from there rather
|
|
339
|
+
than starting over. `parseResponse` and `parseReadable` do the same for web
|
|
340
|
+
streams, so the body of a `fetch` response is parsed straight from its
|
|
341
|
+
bytes, without decoding it first or waiting for the download:
|
|
342
|
+
|
|
343
|
+
```ts
|
|
344
|
+
import { Effect } from "effect"
|
|
345
|
+
import { Parser, ParserStream, Text } from "@thomasfosterau/effect-parse"
|
|
346
|
+
|
|
347
|
+
const json = Parser.compile(makeJson(Text.utf8)) // see examples/json-text.ts
|
|
348
|
+
|
|
349
|
+
const value = await Effect.runPromise(ParserStream.parseResponse(await fetch(url), json))
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
The result, and any error, is the same as parsing the whole body at once.
|
|
353
|
+
Compiled parsers stream compiled. With the JSON example on a 2 MB document,
|
|
354
|
+
`parseResponse` returns about 1 ms after the last byte at 10 MB/s, and keeps
|
|
355
|
+
pace with `Response.json()` at 25 MB/s; on faster connections the native
|
|
356
|
+
parser is quicker (`pnpm bench:stream`).
|
|
357
|
+
|
|
358
|
+
## Services
|
|
359
|
+
|
|
360
|
+
Parsers can require Effect services. These replace winnow's `Stateful`
|
|
361
|
+
input: configuration, symbol tables, indentation stacks and so on.
|
|
362
|
+
|
|
363
|
+
```ts
|
|
364
|
+
import { Context, Effect } from "effect"
|
|
365
|
+
|
|
366
|
+
class Radix extends Context.Service<Radix, number>()("Radix") {}
|
|
367
|
+
|
|
368
|
+
const number = Parser.service(Radix).pipe(
|
|
369
|
+
Parser.flatMap((radix) => Ascii.alphanumeric1.pipe(Parser.map((s) => parseInt(s, radix))))
|
|
370
|
+
)
|
|
371
|
+
// Parser<number, never, Radix>
|
|
372
|
+
|
|
373
|
+
Parser.parse(Parser.provideService(number, Radix, 16), "ff") // 255
|
|
374
|
+
Parser.parseEffect(number, "777").pipe(Effect.provideService(Radix, 8)) // Effect<511>
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
## Unicode text and encodings
|
|
378
|
+
|
|
379
|
+
`Token` reads strings by code point, but the `Ascii` character classes and
|
|
380
|
+
numbers only cover ASCII, and they treat bytes as ASCII. The `Text` module
|
|
381
|
+
adds Unicode parsers that work with any encoding. `Text.make(encoding)`
|
|
382
|
+
builds a set of parsers (`Text.TextParsers<I>`), and sets are provided for
|
|
383
|
+
the common encodings:
|
|
384
|
+
|
|
385
|
+
| Parsers | Input | Offsets |
|
|
386
|
+
|---|---|---|
|
|
387
|
+
| `Text.string` | `string` | UTF-16 code units |
|
|
388
|
+
| `Text.utf8` | `Uint8Array` | bytes |
|
|
389
|
+
| `Text.utf16le`, `Text.utf16be` | `Uint8Array` | bytes |
|
|
390
|
+
| `Text.latin1` | `Uint8Array` | bytes |
|
|
391
|
+
|
|
392
|
+
Every set has the same members (`any`, `literal`, `caseless`, `oneOf`,
|
|
393
|
+
`takeWhile`, `takeUntil`, `alpha1`, `digit1`, `space1`, `identifier`,
|
|
394
|
+
`lineEnding`, `decInt`, `float`, `escaped`, `bom`, ...), and every parser
|
|
395
|
+
returns `string`s. A grammar written once against `TextParsers<I>`
|
|
396
|
+
therefore parses any of these inputs:
|
|
397
|
+
|
|
398
|
+
```ts
|
|
399
|
+
import { Encoding, Parser, Text } from "@thomasfosterau/effect-parse"
|
|
400
|
+
|
|
401
|
+
const assignment = <I extends string | Uint8Array>(T: Text.TextParsers<I>) =>
|
|
402
|
+
Parser.separatedPair(T.identifier, Parser.delimited(T.space0, T.literal("≔"), T.space0), T.decInt)
|
|
403
|
+
|
|
404
|
+
Parser.parse(assignment(Text.string), "größe ≔ 42") // ["größe", 42]
|
|
405
|
+
Parser.parse(assignment(Text.utf8), new TextEncoder().encode("größe ≔ 42")) // ["größe", 42]
|
|
406
|
+
Parser.parse(assignment(Text.utf16le), Encoding.utf16le.encode("größe ≔ 42"))
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
- **Unicode classes.** `alpha1` matches `"Ωμέγα"`, `digit1` matches
|
|
410
|
+
`"١٢٣"`, `space1` matches a no-break space, and `identifier` follows UAX
|
|
411
|
+
#31. The predicates (`Text.isLetter`, `Text.isWhitespace`,
|
|
412
|
+
`Text.isIdStart`, ...) also work with `Token`, and `Text.property("Script=Greek")`
|
|
413
|
+
tests any Unicode property. Numbers (`decInt`, `float`) take ASCII digits
|
|
414
|
+
only, as most languages do.
|
|
415
|
+
- **Bytes are decoded as they are parsed.** Offsets stay byte offsets, and
|
|
416
|
+
`Parser.taken` still returns a zero-copy `Uint8Array`. On partial input,
|
|
417
|
+
a character split across chunks reports `Incomplete`, so
|
|
418
|
+
`ParserStream.parseMany` parses a byte stream as text without decoding it
|
|
419
|
+
first.
|
|
420
|
+
- **Decoding is strict.** Invalid UTF-8 and unpaired surrogates never match.
|
|
421
|
+
Errors on encoded bytes show the input as text:
|
|
422
|
+
|
|
423
|
+
```text
|
|
424
|
+
Parse error at line 1, column 3 (byte 2):
|
|
425
|
+
1 | ab�c
|
|
426
|
+
| ^
|
|
427
|
+
expected letter or end of input, found invalid UTF-8 0xff
|
|
428
|
+
```
|
|
429
|
+
|
|
430
|
+
- **Byte order marks.** `Encoding.sniff(bytes)` picks an encoding from a BOM,
|
|
431
|
+
and `T.bom` skips one.
|
|
432
|
+
- **Binary data.** The byte parsers accept `ArrayBuffer`s, `DataView`s and
|
|
433
|
+
typed arrays too (see [Running parsers](#running-parsers)).
|
|
434
|
+
- **Custom encodings.** `Text.make` accepts any `Encoding.Encoding`: a
|
|
435
|
+
`decode` function plus a few properties.
|
|
436
|
+
|
|
437
|
+
JavaScript strings are parsed natively, and the `Text` parsers only decode
|
|
438
|
+
byte input. Converting strings to bytes up front would be slower: a
|
|
439
|
+
`TextDecoder` call to produce each slice costs several times more than
|
|
440
|
+
`String.prototype.slice`. `pnpm bench` compares the paths with a JSON grammar
|
|
441
|
+
written once for every encoding ([`json-text.ts`](examples/json-text.ts)).
|
|
442
|
+
In our runs `Text.string` was roughly 5–15% slower than the `Token`/`Ascii`
|
|
443
|
+
parsers, and parsing UTF-8 directly was within about 10% of decoding it
|
|
444
|
+
first.
|
|
445
|
+
|
|
446
|
+
## Modules
|
|
447
|
+
|
|
448
|
+
| Module | Contents | winnow / nom equivalent |
|
|
449
|
+
|---|---|---|
|
|
450
|
+
| `Parser` | the type, combinators, runners | `Parser` trait, `combinator` / `combinator`, `branch`, `multi`, `sequence` |
|
|
451
|
+
| `Token` | `any`, `literal`, `oneOf`, `noneOf`, `take`, `takeWhile`, `takeTill`, `takeUntil`, `rest`, `range` | `token` / `bytes` |
|
|
452
|
+
| `Text` | Unicode parsers for any encoding: `string`, `utf8`, `utf16le`, `utf16be`, `latin1`, `make`; Unicode predicates | — |
|
|
453
|
+
| `Encoding` | text encodings for `Text`, `sniff` | — |
|
|
454
|
+
| `Ascii` | character classes, `lineEnding`, `decInt`, `decUint`, `hexUint`, `float`, `caseless`, `takeEscaped`, `escaped`, `Ascii.bytes.*` | `ascii` / `character`, `number` |
|
|
455
|
+
| `Binary` | `u8`, `beU16` … `leI128`, floats, `lengthTake`, `lengthRepeat`, `lengthAndThen` | `binary` / `number`, `multi::length_*` |
|
|
456
|
+
| `Bits` | `bits`, `bytes`, `take`, `takeBig`, `pattern`, `bool` | `binary::bits` / `bits` |
|
|
457
|
+
| `Expression` | Pratt parser with prefix, postfix and infix operators | `combinator::expression` / `nom-language::precedence` |
|
|
458
|
+
| `ParseError` | the error type, `lineColumn`, `formatExpected` | `error` |
|
|
459
|
+
| `ParserStream` | `parseMany`, `parseAll` over Effect `Stream`s | `Partial` / `streaming` |
|
|
460
|
+
| `ParserSchema` | `make`: a `Schema` from a parser | — |
|
|
461
|
+
| `Nom` | nom names and argument orders (`tag`, `many0`, `separatedList1`, `mapRes`, …) | nom |
|
|
462
|
+
| `Input` | input types and the `State` cursor for `Parser.make` | `stream` |
|
|
463
|
+
|
|
464
|
+
Token parsers that cannot infer their input type take a type argument that
|
|
465
|
+
defaults to `string`, like a Rust turbofish:
|
|
466
|
+
|
|
467
|
+
```ts
|
|
468
|
+
Token.take(4) // Parser<string>
|
|
469
|
+
Token.take<Uint8Array>(4) // Parser<Uint8Array, never, never, Uint8Array>
|
|
470
|
+
Token.oneOf<ReadonlyArray<Tok>>((t) => t.kind === "Number")
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
## Coming from nom or winnow
|
|
474
|
+
|
|
475
|
+
| nom | winnow | effect-parse |
|
|
476
|
+
|---|---|---|
|
|
477
|
+
| `tag("x")` | `"x"`, `literal("x")` | `Token.literal("x")` |
|
|
478
|
+
| `char('x')` | `'x'` | `Token.literal("x")` |
|
|
479
|
+
| `alt((a, b))` | `alt((a, b))` | `Parser.alt(a, b)` |
|
|
480
|
+
| `(a, b)` / `tuple` | `(a, b)` | `Parser.seq(a, b)` |
|
|
481
|
+
| — | `seq!(S { a: .., b: .. })` | `Parser.struct({ a, b })` |
|
|
482
|
+
| `many0(p)` | `repeat(0.., p)` | `Parser.repeat(p)` |
|
|
483
|
+
| `many1(p)` | `repeat(1.., p)` | `Parser.repeat(p, { min: 1 })` |
|
|
484
|
+
| `count(p, n)` | `repeat(n, p)` | `Parser.repeat(p, n)` |
|
|
485
|
+
| `separated_list0(s, p)` | `separated(0.., p, s)` | `Parser.separated(p, s)` |
|
|
486
|
+
| `fold_many0(p, i, f)` | `repeat(0.., p).fold(i, f)` | `Parser.fold(p, { init, f })` |
|
|
487
|
+
| `many_till(p, e)` | `repeat_till(0.., p, e)` | `Parser.repeatTill(p, e)` |
|
|
488
|
+
| `take_while1(f)` | `take_while(1.., f)` | `Token.takeWhile(f, { min: 1 })` |
|
|
489
|
+
| `map(p, f)` | `p.map(f)` | `Parser.map(p, f)` / `p.pipe(Parser.map(f))` |
|
|
490
|
+
| `map_res(p, f)` | `p.try_map(f)` | `Parser.tryMap(p, f)` (`f` returns a `Result`) |
|
|
491
|
+
| `map_opt(p, f)` | `p.verify_map(f)` | `Parser.filterMap(p, f)` (`f` returns an `Option`) |
|
|
492
|
+
| `verify(p, f)` | `p.verify(f)` | `Parser.filter(p, f)` |
|
|
493
|
+
| `value(v, p)` | `p.value(v)` | `Parser.as(p, v)` |
|
|
494
|
+
| `recognize(p)` | `p.take()` | `Parser.taken(p)` |
|
|
495
|
+
| `consumed(p)` | `p.with_taken()` | `Parser.withTaken(p)` |
|
|
496
|
+
| `map_parser(p, q)` | `p.and_then(q)` | `Parser.mapParser(p, q)` |
|
|
497
|
+
| `flat_map(p, f)` | `p.flat_map(f)` | `Parser.flatMap(p, f)` |
|
|
498
|
+
| `cut(p)` | `cut_err(p)` | `Parser.cut(p)` |
|
|
499
|
+
| `context("x", p)` | `p.context(StrContext::Label("x"))` | `Parser.context(p, "x")` |
|
|
500
|
+
| `complete(p)` | `p.complete_err()` | `Parser.complete(p)` |
|
|
501
|
+
| `IResult` | `parse_peek` | `Parser.parsePeek` |
|
|
502
|
+
| `streaming::*` | `Partial<I>` | `Parser.parsePartial`, `ParserStream` |
|
|
503
|
+
| — | `dispatch!` | `Parser.dispatch(selector, cases)` |
|
|
504
|
+
| — | `Stateful<I, S>` | `Parser.service` + Effect `Context` |
|
|
505
|
+
| — | `LocatingSlice` | `Parser.span`, `Parser.withSpan`, `Parser.location` |
|
|
506
|
+
| — | `resume_after` | `Parser.resumeAfter` + `Parser.parseRecoverable` |
|
|
507
|
+
|
|
508
|
+
The `Nom` module provides the nom names directly, with nom's argument
|
|
509
|
+
orders, for mechanical ports.
|
|
510
|
+
|
|
511
|
+
## Examples
|
|
512
|
+
|
|
513
|
+
[`examples/`](examples) contains ports of winnow's examples, each with tests
|
|
514
|
+
in [`test/examples.test.ts`](test/examples.test.ts):
|
|
515
|
+
|
|
516
|
+
- [`json.ts`](examples/json.ts): JSON with `dispatch`, `cut` and escapes
|
|
517
|
+
- [`json-text.ts`](examples/json-text.ts): the same grammar written once
|
|
518
|
+
against `Text.TextParsers`, for strings, UTF-8 and UTF-16
|
|
519
|
+
- [`json-fast.ts`](examples/json-fast.ts): the same grammar written for
|
|
520
|
+
compiled parsers, with `Parser.fromEntries`
|
|
521
|
+
- [`arithmetic.ts`](examples/arithmetic.ts): direct evaluation, an AST via
|
|
522
|
+
the Pratt parser, and a lexer plus a parser over the token array
|
|
523
|
+
- [`ini.ts`](examples/ini.ts): INI files with a generator-style parser
|
|
524
|
+
- [`http.ts`](examples/http.ts): HTTP requests over bytes, complete and
|
|
525
|
+
streamed
|
|
526
|
+
- [`sexpr.ts`](examples/sexpr.ts): s-expressions with typed evaluation
|
|
527
|
+
errors
|
|
528
|
+
|
|
529
|
+
`pnpm build && pnpm bench` compares the JSON examples, on each encoding,
|
|
530
|
+
with `JSON.parse`.
|
|
531
|
+
|
|
532
|
+
## Design
|
|
533
|
+
|
|
534
|
+
[`docs/DESIGN.md`](docs/DESIGN.md) records the research into nom, winnow and
|
|
535
|
+
Effect, and the reasoning behind each decision in this port.
|
|
536
|
+
|
|
537
|
+
## Development
|
|
538
|
+
|
|
539
|
+
```sh
|
|
540
|
+
pnpm install
|
|
541
|
+
pnpm check # type-check sources, tests and examples
|
|
542
|
+
pnpm test # run the test suite
|
|
543
|
+
pnpm build # emit dist/
|
|
544
|
+
pnpm bench # JSON example: interpreted vs compiled, per encoding (after a build)
|
|
545
|
+
pnpm bench:native # compiled JSON parser vs JSON.parse on several document shapes
|
|
546
|
+
pnpm bench:stream # parsing a fetch body as it downloads vs Response.json()
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
## License
|
|
550
|
+
|
|
551
|
+
MIT
|