@thomasfosterau/effect-parse 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/README.md +56 -14
- package/dist/Ascii.d.ts +11 -4
- package/dist/Ascii.d.ts.map +1 -1
- package/dist/Ascii.js +12 -4
- package/dist/Ascii.js.map +1 -1
- package/dist/Binary.d.ts +5 -2
- package/dist/Binary.d.ts.map +1 -1
- package/dist/Binary.js +83 -37
- package/dist/Binary.js.map +1 -1
- package/dist/Bits.d.ts.map +1 -1
- package/dist/Bits.js +11 -4
- package/dist/Bits.js.map +1 -1
- package/dist/Compiler.d.ts +74 -2
- package/dist/Compiler.d.ts.map +1 -1
- package/dist/Compiler.js +78 -1
- package/dist/Compiler.js.map +1 -1
- package/dist/Expression.d.ts.map +1 -1
- package/dist/Expression.js +86 -74
- package/dist/Expression.js.map +1 -1
- package/dist/Input.d.ts +56 -5
- package/dist/Input.d.ts.map +1 -1
- package/dist/Input.js +70 -1
- package/dist/Input.js.map +1 -1
- package/dist/Nom.d.ts +5 -15
- package/dist/Nom.d.ts.map +1 -1
- package/dist/Nom.js.map +1 -1
- package/dist/ParseError.d.ts +69 -11
- package/dist/ParseError.d.ts.map +1 -1
- package/dist/ParseError.js +129 -28
- package/dist/ParseError.js.map +1 -1
- package/dist/Parser.d.ts +375 -29
- package/dist/Parser.d.ts.map +1 -1
- package/dist/Parser.js +394 -37
- package/dist/Parser.js.map +1 -1
- package/dist/ParserSchema.d.ts.map +1 -1
- package/dist/ParserSchema.js +2 -3
- package/dist/ParserSchema.js.map +1 -1
- package/dist/ParserStream.d.ts +122 -14
- package/dist/ParserStream.d.ts.map +1 -1
- package/dist/ParserStream.js +293 -62
- package/dist/ParserStream.js.map +1 -1
- package/dist/Syntax.d.ts +23 -5
- package/dist/Syntax.d.ts.map +1 -1
- package/dist/Syntax.js +1 -1
- package/dist/Syntax.js.map +1 -1
- package/dist/Text.d.ts.map +1 -1
- package/dist/Text.js +21 -15
- package/dist/Text.js.map +1 -1
- package/dist/Token.d.ts +65 -11
- package/dist/Token.d.ts.map +1 -1
- package/dist/Token.js +73 -26
- package/dist/Token.js.map +1 -1
- package/dist/internal/compiler/closure.d.ts +1 -1
- package/dist/internal/compiler/closure.d.ts.map +1 -1
- package/dist/internal/compiler/closure.js +263 -113
- package/dist/internal/compiler/closure.js.map +1 -1
- package/dist/internal/compiler/entries.d.ts +3 -1
- package/dist/internal/compiler/entries.d.ts.map +1 -1
- package/dist/internal/compiler/entries.js +3 -2
- package/dist/internal/compiler/entries.js.map +1 -1
- package/dist/internal/compiler/first.d.ts.map +1 -1
- package/dist/internal/compiler/first.js +9 -2
- package/dist/internal/compiler/first.js.map +1 -1
- package/dist/internal/compiler/graph.d.ts +39 -0
- package/dist/internal/compiler/graph.d.ts.map +1 -0
- package/dist/internal/compiler/graph.js +78 -0
- package/dist/internal/compiler/graph.js.map +1 -0
- package/dist/internal/compiler/jit.d.ts +5 -0
- package/dist/internal/compiler/jit.d.ts.map +1 -1
- package/dist/internal/compiler/jit.js +208 -106
- package/dist/internal/compiler/jit.js.map +1 -1
- package/dist/internal/compiler/registry.d.ts +11 -3
- package/dist/internal/compiler/registry.d.ts.map +1 -1
- package/dist/internal/compiler/registry.js.map +1 -1
- package/dist/internal/compiler/search.d.ts +32 -0
- package/dist/internal/compiler/search.d.ts.map +1 -0
- package/dist/internal/compiler/search.js +68 -0
- package/dist/internal/compiler/search.js.map +1 -0
- package/dist/internal/compiler/sets.d.ts.map +1 -1
- package/dist/internal/compiler/sets.js +1 -1
- package/dist/internal/compiler/sets.js.map +1 -1
- package/dist/internal/compiler/wasm.d.ts +8 -0
- package/dist/internal/compiler/wasm.d.ts.map +1 -0
- package/dist/internal/compiler/wasm.js +2840 -0
- package/dist/internal/compiler/wasm.js.map +1 -0
- package/dist/internal/compiler/wasmAsm.d.ts +199 -0
- package/dist/internal/compiler/wasmAsm.d.ts.map +1 -0
- package/dist/internal/compiler/wasmAsm.js +427 -0
- package/dist/internal/compiler/wasmAsm.js.map +1 -0
- package/dist/internal/entries.d.ts +3 -1
- package/dist/internal/entries.d.ts.map +1 -1
- package/dist/internal/entries.js.map +1 -1
- package/dist/internal/node.d.ts +77 -3
- package/dist/internal/node.d.ts.map +1 -1
- package/dist/internal/node.js +54 -10
- package/dist/internal/node.js.map +1 -1
- package/dist/internal/numbers.d.ts.map +1 -1
- package/dist/internal/numbers.js.map +1 -1
- package/dist/internal/run.d.ts +14 -0
- package/dist/internal/run.d.ts.map +1 -1
- package/dist/internal/run.js +53 -11
- package/dist/internal/run.js.map +1 -1
- package/dist/internal/stack.d.ts +14 -0
- package/dist/internal/stack.d.ts.map +1 -1
- package/dist/internal/stack.js +24 -2
- package/dist/internal/stack.js.map +1 -1
- package/dist/internal/syntax.d.ts +4 -4
- package/dist/internal/syntax.d.ts.map +1 -1
- package/dist/internal/syntax.js +18 -10
- package/dist/internal/syntax.js.map +1 -1
- package/dist/internal/text.d.ts +32 -0
- package/dist/internal/text.d.ts.map +1 -1
- package/dist/internal/text.js +68 -14
- package/dist/internal/text.js.map +1 -1
- package/dist/internal/tokens.d.ts +20 -0
- package/dist/internal/tokens.d.ts.map +1 -1
- package/dist/internal/tokens.js +72 -39
- package/dist/internal/tokens.js.map +1 -1
- package/dist/internal/tuple.d.ts +6 -0
- package/dist/internal/tuple.d.ts.map +1 -0
- package/dist/internal/tuple.js +170 -0
- package/dist/internal/tuple.js.map +1 -0
- package/package.json +18 -10
- package/src/Ascii.ts +47 -12
- package/src/Binary.ts +174 -73
- package/src/Bits.ts +16 -6
- package/src/Compiler.ts +101 -10
- package/src/Expression.ts +94 -81
- package/src/Input.ts +99 -7
- package/src/Nom.ts +15 -5
- package/src/ParseError.ts +209 -52
- package/src/Parser.ts +684 -143
- package/src/ParserSchema.ts +1 -2
- package/src/ParserStream.ts +361 -68
- package/src/Syntax.ts +60 -36
- package/src/Text.ts +32 -27
- package/src/Token.ts +152 -34
- package/src/internal/compiler/closure.ts +371 -153
- package/src/internal/compiler/entries.ts +4 -2
- package/src/internal/compiler/first.ts +16 -0
- package/src/internal/compiler/graph.ts +96 -0
- package/src/internal/compiler/jit.ts +319 -169
- package/src/internal/compiler/registry.ts +10 -2
- package/src/internal/compiler/search.ts +68 -0
- package/src/internal/compiler/sets.ts +7 -3
- package/src/internal/compiler/wasm.ts +3276 -0
- package/src/internal/compiler/wasmAsm.ts +484 -0
- package/src/internal/entries.ts +8 -2
- package/src/internal/node.ts +103 -7
- package/src/internal/numbers.ts +3 -1
- package/src/internal/run.ts +73 -12
- package/src/internal/stack.ts +25 -1
- package/src/internal/syntax.ts +39 -18
- package/src/internal/text.ts +97 -9
- package/src/internal/tokens.ts +106 -14
- package/src/internal/tuple.ts +141 -0
package/src/ParserSchema.ts
CHANGED
|
@@ -120,8 +120,7 @@ const decodeWith = <T extends Schema.Top>(
|
|
|
120
120
|
if (encode === undefined) {
|
|
121
121
|
return Effect.fail(new SchemaIssue.Forbidden({ message: "Encoding is not supported" }, value, parseOptions))
|
|
122
122
|
}
|
|
123
|
-
|
|
124
|
-
return Result.isSuccess(result) ? Effect.succeed(result.success) : Effect.fail(result.failure)
|
|
123
|
+
return Effect.fromResult(encode(value))
|
|
125
124
|
}
|
|
126
125
|
})
|
|
127
126
|
)
|
package/src/ParserStream.ts
CHANGED
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
* retried once more data arrives. When the stream ends, the leftover input
|
|
8
8
|
* is parsed in complete mode.
|
|
9
9
|
*
|
|
10
|
+
* Every runner bounds what it buffers: `maxBufferSize` (see
|
|
11
|
+
* {@link BufferOptions}) defaults to 64 MiB ({@link defaultMaxBufferSize}),
|
|
12
|
+
* and a stream that exceeds it fails with a {@link BufferLimitError}, so a
|
|
13
|
+
* socket or an HTTP body can be parsed without wrapping the stream first.
|
|
14
|
+
* Pass `Infinity` for input that is trusted to be large.
|
|
15
|
+
*
|
|
10
16
|
* @example
|
|
11
17
|
* ```ts
|
|
12
18
|
* import { Effect, Stream } from "effect"
|
|
@@ -38,6 +44,7 @@ import * as Run from "./internal/run.js"
|
|
|
38
44
|
import * as Stack from "./internal/stack.js"
|
|
39
45
|
import { isEmptyUnits } from "./internal/text.js"
|
|
40
46
|
import type { Parser } from "./Parser.js"
|
|
47
|
+
import { isParser } from "./Parser.js"
|
|
41
48
|
|
|
42
49
|
/**
|
|
43
50
|
* Input kinds that can be streamed in chunks.
|
|
@@ -47,22 +54,202 @@ import type { Parser } from "./Parser.js"
|
|
|
47
54
|
*/
|
|
48
55
|
export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
|
|
49
56
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
57
|
+
/**
|
|
58
|
+
* The default `maxBufferSize` of {@link parseMany}, {@link parseAll},
|
|
59
|
+
* {@link parseReadable} and {@link parseResponse}: 64 MiB, counted in tokens
|
|
60
|
+
* of the input (UTF-16 code units of a string, bytes of a `Uint8Array`,
|
|
61
|
+
* elements of an array).
|
|
62
|
+
*
|
|
63
|
+
* @since 0.6.0
|
|
64
|
+
* @category constants
|
|
65
|
+
*/
|
|
66
|
+
export const defaultMaxBufferSize: number = 64 * 1024 * 1024
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* A stream runner would have buffered more than its `maxBufferSize`: `size`
|
|
70
|
+
* tokens against a limit of `limit`. It is a different failure from a
|
|
71
|
+
* `ParseError`: the input is too large, not malformed (an HTTP server
|
|
72
|
+
* answers 413 for the one and 400 for the other), and nothing of the input
|
|
73
|
+
* is attached to it.
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```ts
|
|
77
|
+
* import { Effect, Stream } from "effect"
|
|
78
|
+
* import { Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
|
|
79
|
+
*
|
|
80
|
+
* const line = Parser.terminated(Token.takeTill("\n"), Token.literal("\n"))
|
|
81
|
+
*
|
|
82
|
+
* Stream.make("no newline, ", "ever").pipe(
|
|
83
|
+
* ParserStream.parseMany(line, { maxBufferSize: 8 }),
|
|
84
|
+
* Stream.runCollect,
|
|
85
|
+
* Effect.catchTag("BufferLimitError", (e) => Effect.succeed(`too large: ${e.size} > ${e.limit}`))
|
|
86
|
+
* )
|
|
87
|
+
* ```
|
|
88
|
+
*
|
|
89
|
+
* @since 0.6.0
|
|
90
|
+
* @category errors
|
|
91
|
+
*/
|
|
92
|
+
export class BufferLimitError
|
|
93
|
+
extends /*#__PURE__*/ Data.TaggedError("BufferLimitError")<{
|
|
94
|
+
/** The `maxBufferSize` in force, in tokens. */
|
|
95
|
+
readonly limit: number
|
|
96
|
+
/** The tokens the runner held, or was about to hold (for `parseResponse`, the declared `Content-Length`). */
|
|
97
|
+
readonly size: number
|
|
98
|
+
}>
|
|
99
|
+
{
|
|
100
|
+
override get message(): string {
|
|
101
|
+
return `the input is larger than maxBufferSize: ${this.size} tokens, limit ${this.limit}`
|
|
60
102
|
}
|
|
61
|
-
return [...a, ...(b as ReadonlyArray<unknown>)] as unknown as I
|
|
62
103
|
}
|
|
63
104
|
|
|
64
|
-
|
|
65
|
-
|
|
105
|
+
/**
|
|
106
|
+
* The limit on buffered input that the stream runners share.
|
|
107
|
+
*
|
|
108
|
+
* @since 0.6.0
|
|
109
|
+
* @category models
|
|
110
|
+
*/
|
|
111
|
+
export interface BufferOptions {
|
|
112
|
+
/**
|
|
113
|
+
* The most tokens a runner buffers (UTF-16 code units of a string, bytes
|
|
114
|
+
* of a `Uint8Array`, elements of an array) before it fails with a
|
|
115
|
+
* {@link BufferLimitError}. Defaults to {@link defaultMaxBufferSize}
|
|
116
|
+
* (64 MiB); `Infinity` turns the limit off.
|
|
117
|
+
*
|
|
118
|
+
* {@link parseMany} counts the input it has not consumed yet, which is one
|
|
119
|
+
* unfinished value: complete values in a chunk are parsed and emitted
|
|
120
|
+
* first, whatever the chunk's size. {@link parseAll},
|
|
121
|
+
* {@link parseReadable} and {@link parseResponse} hold the whole input, so
|
|
122
|
+
* they count all of it.
|
|
123
|
+
*/
|
|
124
|
+
readonly maxBufferSize?: number | undefined
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The limit `options` asks for: a positive number of tokens, or `Infinity`. */
|
|
128
|
+
const limitOf = (options: BufferOptions | undefined): number => {
|
|
129
|
+
const limit = options?.maxBufferSize ?? defaultMaxBufferSize
|
|
130
|
+
if (!(limit > 0)) throw new RangeError("maxBufferSize must be a positive number of tokens, or Infinity")
|
|
131
|
+
return limit
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Above this many unconsumed tokens, {@link parseMany} stops retrying its
|
|
136
|
+
* unfinished value on every chunk (see {@link Pending.due}).
|
|
137
|
+
*/
|
|
138
|
+
const BackoffThreshold = 64 * 1024
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The input `parseMany` has not consumed yet: what is left of the chunks
|
|
142
|
+
* after the values parsed from them, which is one unfinished value.
|
|
143
|
+
*
|
|
144
|
+
* Appending is amortized constant time per token: text is concatenated (a
|
|
145
|
+
* rope until it is parsed), bytes go into a buffer that doubles, and
|
|
146
|
+
* elements are pushed. Bytes that a parse has seen are never overwritten,
|
|
147
|
+
* since the values parsed from them may be views of the buffer
|
|
148
|
+
* (`Token.take` returns a `subarray`): the buffer is only ever appended to
|
|
149
|
+
* past its end, or replaced by a larger one.
|
|
150
|
+
*/
|
|
151
|
+
class Pending<I extends Chunkable> {
|
|
152
|
+
private kind: 0 | 1 | 2 | undefined
|
|
153
|
+
private text = ""
|
|
154
|
+
private bytes: Uint8Array = new Uint8Array(0)
|
|
155
|
+
/** Where the unconsumed bytes start and end in `bytes`, and how far `bytes` may be written. */
|
|
156
|
+
private start = 0
|
|
157
|
+
private end = 0
|
|
158
|
+
private capacity = 0
|
|
159
|
+
private items: Array<unknown> = []
|
|
160
|
+
/** The unconsumed size at the last attempt that ran out of input, or 0 when none is outstanding. */
|
|
161
|
+
private attempted = 0
|
|
162
|
+
|
|
163
|
+
/** The number of unconsumed tokens. */
|
|
164
|
+
get size(): number {
|
|
165
|
+
return this.kind === 1 ? this.end - this.start : this.kind === 2 ? this.items.length : this.text.length
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
get isEmpty(): boolean {
|
|
169
|
+
return this.kind === undefined
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
append(chunk: I): void {
|
|
173
|
+
if (Predicate.isString(chunk)) {
|
|
174
|
+
this.kind ??= 0
|
|
175
|
+
this.text += chunk
|
|
176
|
+
} else if (chunk instanceof Uint8Array) {
|
|
177
|
+
this.kind ??= 1
|
|
178
|
+
this.appendBytes(chunk)
|
|
179
|
+
} else {
|
|
180
|
+
this.kind ??= 2
|
|
181
|
+
for (let i = 0; i < chunk.length; i++) this.items.push(chunk[i])
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
private appendBytes(chunk: Uint8Array): void {
|
|
186
|
+
const held = this.end - this.start
|
|
187
|
+
if (held === 0) {
|
|
188
|
+
// Nothing is pending: the chunk is the buffer, uncopied. It is the
|
|
189
|
+
// caller's, so it is never written to (`capacity` is its end)
|
|
190
|
+
this.bytes = chunk
|
|
191
|
+
this.start = 0
|
|
192
|
+
this.end = this.capacity = chunk.length
|
|
193
|
+
return
|
|
194
|
+
}
|
|
195
|
+
const needed = held + chunk.length
|
|
196
|
+
if (this.end + chunk.length > this.capacity) {
|
|
197
|
+
// Twice what is needed, so appending costs amortized constant time per byte
|
|
198
|
+
const grown = new Uint8Array(Math.max(needed * 2, 4096))
|
|
199
|
+
grown.set(this.bytes.subarray(this.start, this.end))
|
|
200
|
+
this.bytes = grown
|
|
201
|
+
this.start = 0
|
|
202
|
+
this.end = held
|
|
203
|
+
this.capacity = grown.length
|
|
204
|
+
}
|
|
205
|
+
this.bytes.set(chunk, this.end)
|
|
206
|
+
this.end += chunk.length
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** The unconsumed input. */
|
|
210
|
+
input(): I {
|
|
211
|
+
if (this.kind === 1) {
|
|
212
|
+
// SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
|
|
213
|
+
return this.bytes.subarray(this.start, this.end) as I
|
|
214
|
+
}
|
|
215
|
+
if (this.kind === 2) {
|
|
216
|
+
const items: Chunkable = this.items
|
|
217
|
+
// SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
|
|
218
|
+
return items as I
|
|
219
|
+
}
|
|
220
|
+
// SAFETY: `kind` 0 is set by the first chunk being a string, so `I` is `string`
|
|
221
|
+
return this.text as I
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Drops the first `count` tokens, which values were parsed from. */
|
|
225
|
+
consume(count: number): void {
|
|
226
|
+
if (count === 0) return
|
|
227
|
+
if (this.kind === 1) this.start += count
|
|
228
|
+
// A new array: the one parsed is the `input` of the state that read it
|
|
229
|
+
else if (this.kind === 2) this.items = this.items.slice(count)
|
|
230
|
+
else this.text = this.text.slice(count)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Records whether the attempt just made left an unfinished value. */
|
|
234
|
+
attemptedWith(unfinished: boolean): void {
|
|
235
|
+
this.attempted = unfinished ? this.size : 0
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Whether to parse now. A value that is still unfinished is parsed again
|
|
240
|
+
* from its start, so retrying on every chunk costs the square of its
|
|
241
|
+
* size: a megabyte with no delimiter, in 1 KiB chunks, took seconds. Past
|
|
242
|
+
* {@link BackoffThreshold} tokens, the next attempt therefore waits until
|
|
243
|
+
* the buffer has grown by an eighth, which keeps the total work linear
|
|
244
|
+
* (each token is re-read at most about nine times). Smaller values are
|
|
245
|
+
* retried on every chunk, as cheaply as before, so a record is emitted
|
|
246
|
+
* as soon as its last chunk arrives.
|
|
247
|
+
*/
|
|
248
|
+
due(): boolean {
|
|
249
|
+
const attempted = this.attempted
|
|
250
|
+
return attempted <= BackoffThreshold || this.size - attempted >= attempted >>> 3
|
|
251
|
+
}
|
|
252
|
+
}
|
|
66
253
|
|
|
67
254
|
const noProgress = (input: Input, offset: number) =>
|
|
68
255
|
ParseError.fromFailure<never>(
|
|
@@ -72,12 +259,12 @@ const noProgress = (input: Input, offset: number) =>
|
|
|
72
259
|
)
|
|
73
260
|
|
|
74
261
|
/**
|
|
75
|
-
* Options for {@link parseMany}.
|
|
262
|
+
* Options for {@link parseMany} and {@link parseAll}.
|
|
76
263
|
*
|
|
77
264
|
* @since 0.1.0
|
|
78
265
|
* @category models
|
|
79
266
|
*/
|
|
80
|
-
export interface ParseManyOptions<I> {
|
|
267
|
+
export interface ParseManyOptions<I> extends BufferOptions {
|
|
81
268
|
/** The empty input, used to initialise the buffer (inferred from the first chunk if omitted). */
|
|
82
269
|
readonly empty?: Source<I> | undefined
|
|
83
270
|
}
|
|
@@ -92,8 +279,34 @@ export interface ParseManyOptions<I> {
|
|
|
92
279
|
*
|
|
93
280
|
* A parser compiled with the `Compiler` module runs compiled, with the
|
|
94
281
|
* build that handles partial input (see `Parser.run`); a value that fails
|
|
95
|
-
*
|
|
96
|
-
* `Incomplete`
|
|
282
|
+
* is parsed again interpreted, so errors are the interpreter's. A value
|
|
283
|
+
* cut short by its chunk is not: compiled code reports `Incomplete` where
|
|
284
|
+
* the interpreter does.
|
|
285
|
+
*
|
|
286
|
+
* **Untrusted streams.** The input not yet consumed (one unfinished value)
|
|
287
|
+
* is limited to `maxBufferSize` tokens, 64 MiB by default
|
|
288
|
+
* ({@link defaultMaxBufferSize}); past it the stream fails with a
|
|
289
|
+
* {@link BufferLimitError}, so a peer that never sends the delimiter cannot
|
|
290
|
+
* grow the buffer without bound. An unfinished value is parsed again from
|
|
291
|
+
* its start when more input arrives. Up to 64 KiB that happens on every
|
|
292
|
+
* chunk; a larger one is retried only once the buffer has grown by an
|
|
293
|
+
* eighth, so the work stays linear in its size. The cost: the last chunk of
|
|
294
|
+
* a value larger than 64 KiB may not complete it in the output until more
|
|
295
|
+
* input arrives or the stream ends, when everything left is parsed.
|
|
296
|
+
*
|
|
297
|
+
* @example
|
|
298
|
+
* ```ts
|
|
299
|
+
* import { Effect, Stream } from "effect"
|
|
300
|
+
* import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
|
|
301
|
+
*
|
|
302
|
+
* const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
|
|
303
|
+
*
|
|
304
|
+
* const program = Stream.make("1\n2", "3\n", "4\n").pipe(
|
|
305
|
+
* ParserStream.parseMany(line, { maxBufferSize: 1024 }),
|
|
306
|
+
* Stream.runCollect
|
|
307
|
+
* )
|
|
308
|
+
* // Effect.runSync(program) => [1, 23, 4]
|
|
309
|
+
* ```
|
|
97
310
|
*
|
|
98
311
|
* @since 0.1.0
|
|
99
312
|
* @category combinators
|
|
@@ -102,51 +315,64 @@ export const parseMany: {
|
|
|
102
315
|
<A, E, R, I extends Chunkable>(
|
|
103
316
|
parser: Parser<A, E, R, I>,
|
|
104
317
|
options?: ParseManyOptions<I>
|
|
105
|
-
): <E0, R0>(
|
|
318
|
+
): <E0, R0>(
|
|
319
|
+
self: Stream.Stream<Source<I>, E0, R0>
|
|
320
|
+
) => Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
106
321
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
107
322
|
self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
|
|
108
323
|
parser: Parser<A, E, R, I>,
|
|
109
324
|
options?: ParseManyOptions<I>
|
|
110
|
-
): Stream.Stream<A, E0 | ParseError.ParseError<E
|
|
325
|
+
): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
111
326
|
} = /*#__PURE__*/ dual(
|
|
112
327
|
(args) => Stream.isStream(args[0]),
|
|
113
328
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
114
329
|
self: Stream.Stream<Source<I>, E0, R0>,
|
|
115
330
|
parser: Parser<A, E, R, I>,
|
|
116
331
|
options?: ParseManyOptions<I>
|
|
117
|
-
): Stream.Stream<A, E0 | ParseError.ParseError<E
|
|
118
|
-
|
|
332
|
+
): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
|
|
333
|
+
const limit = limitOf(options)
|
|
334
|
+
return Stream.suspend(() => {
|
|
119
335
|
const step = parser.step
|
|
120
|
-
|
|
336
|
+
const pending = new Pending<I>()
|
|
337
|
+
if (options?.empty !== undefined) pending.append(fromSource<I>(options.empty))
|
|
121
338
|
|
|
122
|
-
const drain = (
|
|
339
|
+
const drain = (
|
|
340
|
+
context: any,
|
|
341
|
+
partial: boolean
|
|
342
|
+
): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
|
|
123
343
|
const out: Array<A> = []
|
|
124
|
-
if (
|
|
125
|
-
const input =
|
|
344
|
+
if (pending.isEmpty) return Effect.succeed(out)
|
|
345
|
+
const input = pending.input()
|
|
126
346
|
// One state for the whole buffer, reset before each value
|
|
127
347
|
const state = new State(input)
|
|
128
348
|
let offset = 0
|
|
129
349
|
while (offset < input.length) {
|
|
130
350
|
state.reset(offset, { partial, context })
|
|
131
|
-
|
|
351
|
+
// A compiled `Incomplete` is the interpreter's: only an error needs the interpreted run
|
|
352
|
+
const a = Run.runOn(parser, step, state, false)
|
|
132
353
|
if (a === Fail) {
|
|
133
354
|
if (partial && state.mode === "Incomplete") break
|
|
355
|
+
state.reset(offset, { partial, context })
|
|
356
|
+
Run.runOn(parser, step, state)
|
|
134
357
|
return Effect.fail(ParseError.fromFailure<E>(input, state.reportedFailure(), state.mode, state.encoding))
|
|
135
358
|
}
|
|
136
359
|
if (state.pos === offset) return Effect.fail(noProgress(input, offset))
|
|
137
360
|
out.push(a)
|
|
138
361
|
offset = state.pos
|
|
139
362
|
}
|
|
140
|
-
|
|
141
|
-
|
|
363
|
+
pending.consume(offset)
|
|
364
|
+
pending.attemptedWith(offset < input.length)
|
|
365
|
+
// Only what no value consumed counts: an unfinished value larger than the limit
|
|
366
|
+
const size = input.length - offset
|
|
367
|
+
return size > limit ? Effect.fail(new BufferLimitError({ limit, size })) : Effect.succeed(out)
|
|
142
368
|
}
|
|
143
369
|
|
|
144
370
|
const feed = (source: Source<I>) =>
|
|
145
371
|
Effect.flatMap(Effect.context<R>(), (context) =>
|
|
146
|
-
Effect.suspend(() => {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
return drain(context, true)
|
|
372
|
+
Effect.suspend((): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
|
|
373
|
+
pending.append(fromSource<I>(source))
|
|
374
|
+
// Over the limit, parse whatever the back-off says, so that complete values are not counted
|
|
375
|
+
return pending.due() || pending.size > limit ? drain(context, true) : Effect.succeed([])
|
|
150
376
|
})
|
|
151
377
|
)
|
|
152
378
|
|
|
@@ -158,6 +384,7 @@ export const parseMany: {
|
|
|
158
384
|
Stream.concat(Stream.unwrap(Effect.map(finish, Stream.fromIterable)))
|
|
159
385
|
)
|
|
160
386
|
})
|
|
387
|
+
}
|
|
161
388
|
)
|
|
162
389
|
|
|
163
390
|
/**
|
|
@@ -213,15 +440,24 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
213
440
|
}
|
|
214
441
|
|
|
215
442
|
private input(): I {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
443
|
+
if (this.kind === 1) {
|
|
444
|
+
// SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
|
|
445
|
+
return this.bytes.subarray(0, this.length) as I
|
|
446
|
+
}
|
|
447
|
+
if (this.kind === 2) {
|
|
448
|
+
// Items are only appended between attempts, after each state has read its `end`
|
|
449
|
+
const items: Chunkable = this.items
|
|
450
|
+
// SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
|
|
451
|
+
return items as I
|
|
224
452
|
}
|
|
453
|
+
// SAFETY: `kind` 0 means the first chunk was a string, so `I` is `string`; while `kind` is unset (no
|
|
454
|
+
// chunk arrived) `I` is unknown and the empty string stands for the empty input
|
|
455
|
+
return this.text as I
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/** The tokens held: the whole input so far. */
|
|
459
|
+
get size(): number {
|
|
460
|
+
return this.length
|
|
225
461
|
}
|
|
226
462
|
|
|
227
463
|
push(chunk: I): void {
|
|
@@ -247,8 +483,9 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
247
483
|
* The step attempts run: the parser's compiled code if it was compiled
|
|
248
484
|
* (the resumable build, unless `resumable` is false), else the interpreter.
|
|
249
485
|
*/
|
|
250
|
-
private step(state: State<I>, resumable = true):
|
|
251
|
-
|
|
486
|
+
private step(state: State<I>, resumable = true): A | Fail {
|
|
487
|
+
// SAFETY: a compiled step (typed `any` by the compilers) returns what the parser's own step returns
|
|
488
|
+
return Stack.run(Registry.stepFor(this.parser, state, resumable) ?? this.parser.step, state) as A | Fail
|
|
252
489
|
}
|
|
253
490
|
|
|
254
491
|
private attempt(): void {
|
|
@@ -257,7 +494,7 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
257
494
|
const a = this.step(state)
|
|
258
495
|
this.attempted = this.length
|
|
259
496
|
if (a !== Fail) {
|
|
260
|
-
this.settled = { value: a
|
|
497
|
+
this.settled = { value: a, pos: state.pos }
|
|
261
498
|
} else if (state.mode === "Incomplete") {
|
|
262
499
|
this.restart = state.failure.offset
|
|
263
500
|
} else {
|
|
@@ -276,7 +513,7 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
276
513
|
const resumable = this.attempted > 0
|
|
277
514
|
if (resumable) state.resume = this.resume
|
|
278
515
|
const a = this.step(state, resumable)
|
|
279
|
-
if (a !== Fail && state.pos === state.end) return Effect.succeed(a
|
|
516
|
+
if (a !== Fail && state.pos === state.end) return Effect.succeed(a)
|
|
280
517
|
}
|
|
281
518
|
// The error comes from an ordinary parse of the whole input, so it is the
|
|
282
519
|
// same as if the input had been parsed in one go
|
|
@@ -308,6 +545,11 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
308
545
|
* callbacks should be pure: the elements parsed before a checkpoint are
|
|
309
546
|
* not parsed again, and on failure everything is parsed again.
|
|
310
547
|
*
|
|
548
|
+
* The whole input is held until the stream ends, so it is limited to
|
|
549
|
+
* `maxBufferSize` tokens (64 MiB by default, see {@link BufferOptions}):
|
|
550
|
+
* a stream that would exceed it fails with a {@link BufferLimitError}
|
|
551
|
+
* before the chunk that crosses the limit is buffered.
|
|
552
|
+
*
|
|
311
553
|
* @since 0.1.0
|
|
312
554
|
* @category combinators
|
|
313
555
|
*/
|
|
@@ -315,29 +557,42 @@ export const parseAll: {
|
|
|
315
557
|
<A, E, R, I extends Chunkable>(
|
|
316
558
|
parser: Parser<A, E, R, I>,
|
|
317
559
|
options?: ParseManyOptions<I>
|
|
318
|
-
): <E0, R0>(
|
|
560
|
+
): <E0, R0>(
|
|
561
|
+
self: Stream.Stream<Source<I>, E0, R0>
|
|
562
|
+
) => Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
319
563
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
320
564
|
self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
|
|
321
565
|
parser: Parser<A, E, R, I>,
|
|
322
566
|
options?: ParseManyOptions<I>
|
|
323
|
-
): Effect.Effect<A, E0 | ParseError.ParseError<E
|
|
567
|
+
): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
324
568
|
} = /*#__PURE__*/ dual(
|
|
325
569
|
(args) => Stream.isStream(args[0]),
|
|
326
570
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
327
571
|
self: Stream.Stream<Source<I>, E0, R0>,
|
|
328
572
|
parser: Parser<A, E, R, I>,
|
|
329
573
|
options?: ParseManyOptions<I>
|
|
330
|
-
): Effect.Effect<A, E0 | ParseError.ParseError<E
|
|
331
|
-
|
|
574
|
+
): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
|
|
575
|
+
const limit = limitOf(options)
|
|
576
|
+
return Effect.flatMap(Effect.context<R>(), (context) =>
|
|
332
577
|
Effect.suspend(() => {
|
|
333
578
|
const empty = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
|
|
334
|
-
const incremental = new Incremental(parser, context
|
|
579
|
+
const incremental = new Incremental(parser, context, empty)
|
|
335
580
|
return Effect.flatMap(
|
|
336
|
-
Stream.runForEach(self, (
|
|
581
|
+
Stream.runForEach(self, (source) =>
|
|
582
|
+
Effect.suspend(() => {
|
|
583
|
+
const chunk = fromSource<I>(source)
|
|
584
|
+
// Checked before the chunk is copied in: the whole input is held, so all of it counts
|
|
585
|
+
const size = incremental.size + chunk.length
|
|
586
|
+
if (size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
|
|
587
|
+
incremental.push(chunk)
|
|
588
|
+
return Effect.void
|
|
589
|
+
})
|
|
590
|
+
),
|
|
337
591
|
() => Effect.suspend(() => incremental.finish())
|
|
338
592
|
)
|
|
339
593
|
})
|
|
340
594
|
)
|
|
595
|
+
}
|
|
341
596
|
)
|
|
342
597
|
|
|
343
598
|
/**
|
|
@@ -349,11 +604,17 @@ export const parseAll: {
|
|
|
349
604
|
*/
|
|
350
605
|
export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ readonly cause: unknown }> {}
|
|
351
606
|
|
|
607
|
+
/** Whether a web-stream runner was called with its stream first: the other form starts with the parser. */
|
|
608
|
+
const isDataFirst = (args: IArguments): boolean => !isParser(args[0])
|
|
609
|
+
|
|
352
610
|
/**
|
|
353
611
|
* Parses a web `ReadableStream` of bytes, such as the body of a `fetch`
|
|
354
612
|
* response, incrementally (see {@link parseAll}): the bytes are parsed as
|
|
355
613
|
* they arrive, without waiting for the end of the stream or decoding them
|
|
356
|
-
* first. Use a byte parser, e.g. one built from `Text.utf8`.
|
|
614
|
+
* first. Use a byte parser, e.g. one built from `Text.utf8`. The stream is
|
|
615
|
+
* limited to `maxBufferSize` bytes (64 MiB by default, see
|
|
616
|
+
* {@link BufferOptions}), past which it fails with a
|
|
617
|
+
* {@link BufferLimitError}.
|
|
357
618
|
*
|
|
358
619
|
* @example
|
|
359
620
|
* ```ts
|
|
@@ -372,30 +633,52 @@ export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ rea
|
|
|
372
633
|
*/
|
|
373
634
|
export const parseReadable: {
|
|
374
635
|
<A, E, R>(
|
|
375
|
-
parser: Parser<A, E, R, Uint8Array
|
|
376
|
-
|
|
636
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
637
|
+
options?: BufferOptions
|
|
638
|
+
): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
377
639
|
<A, E, R>(
|
|
378
640
|
self: ReadableStream<BinaryInput>,
|
|
379
|
-
parser: Parser<A, E, R, Uint8Array
|
|
380
|
-
|
|
641
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
642
|
+
options?: BufferOptions
|
|
643
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
381
644
|
} = /*#__PURE__*/ dual(
|
|
382
|
-
|
|
645
|
+
isDataFirst,
|
|
383
646
|
<A, E, R>(
|
|
384
647
|
self: ReadableStream<BinaryInput>,
|
|
385
|
-
parser: Parser<A, E, R, Uint8Array
|
|
386
|
-
|
|
648
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
649
|
+
options?: BufferOptions
|
|
650
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
|
|
387
651
|
parseAll(
|
|
388
652
|
Stream.fromReadableStream({ evaluate: () => self, onError: (cause) => new ReadError({ cause }) }),
|
|
389
653
|
parser,
|
|
390
|
-
{ empty: new Uint8Array(0) }
|
|
654
|
+
{ empty: new Uint8Array(0), maxBufferSize: options?.maxBufferSize }
|
|
391
655
|
)
|
|
392
656
|
)
|
|
393
657
|
|
|
658
|
+
/**
|
|
659
|
+
* The length a response declares for the body as it will be read, if it
|
|
660
|
+
* declares one: `Content-Length` counts the encoded body, so a response
|
|
661
|
+
* with a `Content-Encoding` (which `fetch` decodes) declares nothing about
|
|
662
|
+
* the bytes parsed.
|
|
663
|
+
*/
|
|
664
|
+
const declaredLength = (response: Response): number | undefined => {
|
|
665
|
+
if (response.headers.has("content-encoding")) return undefined
|
|
666
|
+
const header = response.headers.get("content-length")
|
|
667
|
+
if (header === null || !/^[0-9]+$/.test(header)) return undefined
|
|
668
|
+
return Number(header)
|
|
669
|
+
}
|
|
670
|
+
|
|
394
671
|
/**
|
|
395
672
|
* Parses the body of a `fetch` `Response` incrementally, as it downloads
|
|
396
673
|
* (see {@link parseReadable}). A response without a body is parsed as empty
|
|
397
674
|
* input. The status is not checked.
|
|
398
675
|
*
|
|
676
|
+
* The body is limited to `maxBufferSize` bytes (64 MiB by default, see
|
|
677
|
+
* {@link BufferOptions}). A response whose `Content-Length` already exceeds
|
|
678
|
+
* the limit fails with a {@link BufferLimitError} before any of the body is
|
|
679
|
+
* read; the limit is enforced on the bytes read as well, since the header
|
|
680
|
+
* may be absent or wrong.
|
|
681
|
+
*
|
|
399
682
|
* @example
|
|
400
683
|
* ```ts
|
|
401
684
|
* import { Effect } from "effect"
|
|
@@ -404,7 +687,9 @@ export const parseReadable: {
|
|
|
404
687
|
* const lines = Parser.separated(Text.utf8.tillLineEnding, Text.utf8.lineEnding)
|
|
405
688
|
*
|
|
406
689
|
* const response = await fetch("https://example.com/log.txt")
|
|
407
|
-
* const value = await Effect.runPromise(
|
|
690
|
+
* const value = await Effect.runPromise(
|
|
691
|
+
* ParserStream.parseResponse(response, lines, { maxBufferSize: 8 * 1024 * 1024 })
|
|
692
|
+
* )
|
|
408
693
|
* ```
|
|
409
694
|
*
|
|
410
695
|
* @since 0.1.0
|
|
@@ -412,17 +697,25 @@ export const parseReadable: {
|
|
|
412
697
|
*/
|
|
413
698
|
export const parseResponse: {
|
|
414
699
|
<A, E, R>(
|
|
415
|
-
parser: Parser<A, E, R, Uint8Array
|
|
416
|
-
|
|
700
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
701
|
+
options?: BufferOptions
|
|
702
|
+
): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
417
703
|
<A, E, R>(
|
|
418
704
|
self: Response,
|
|
419
|
-
parser: Parser<A, E, R, Uint8Array
|
|
420
|
-
|
|
705
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
706
|
+
options?: BufferOptions
|
|
707
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
421
708
|
} = /*#__PURE__*/ dual(
|
|
422
|
-
|
|
709
|
+
isDataFirst,
|
|
423
710
|
<A, E, R>(
|
|
424
711
|
self: Response,
|
|
425
|
-
parser: Parser<A, E, R, Uint8Array
|
|
426
|
-
|
|
427
|
-
|
|
712
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
713
|
+
options?: BufferOptions
|
|
714
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
|
|
715
|
+
Effect.suspend(() => {
|
|
716
|
+
const limit = limitOf(options)
|
|
717
|
+
const size = declaredLength(self)
|
|
718
|
+
if (size !== undefined && size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
|
|
719
|
+
return parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser, options)
|
|
720
|
+
})
|
|
428
721
|
)
|