@thomasfosterau/effect-parse 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/README.md +19 -1
- package/dist/Ascii.d.ts.map +1 -1
- package/dist/Ascii.js +5 -3
- package/dist/Ascii.js.map +1 -1
- package/dist/Binary.d.ts +5 -2
- package/dist/Binary.d.ts.map +1 -1
- package/dist/Binary.js +20 -3
- package/dist/Binary.js.map +1 -1
- package/dist/Bits.d.ts.map +1 -1
- package/dist/Bits.js +4 -0
- package/dist/Bits.js.map +1 -1
- package/dist/Input.d.ts +21 -3
- package/dist/Input.d.ts.map +1 -1
- package/dist/Input.js +31 -4
- package/dist/Input.js.map +1 -1
- package/dist/Nom.d.ts.map +1 -1
- package/dist/ParseError.d.ts +69 -11
- package/dist/ParseError.d.ts.map +1 -1
- package/dist/ParseError.js +129 -28
- package/dist/ParseError.js.map +1 -1
- package/dist/Parser.d.ts +10 -4
- package/dist/Parser.d.ts.map +1 -1
- package/dist/Parser.js +10 -24
- package/dist/Parser.js.map +1 -1
- package/dist/ParserSchema.d.ts.map +1 -1
- package/dist/ParserSchema.js +1 -2
- package/dist/ParserSchema.js.map +1 -1
- package/dist/ParserStream.d.ts +119 -12
- package/dist/ParserStream.d.ts.map +1 -1
- package/dist/ParserStream.js +282 -55
- package/dist/ParserStream.js.map +1 -1
- package/dist/Syntax.js +1 -1
- package/dist/Text.d.ts.map +1 -1
- package/dist/Text.js +13 -7
- package/dist/Text.js.map +1 -1
- package/dist/Token.d.ts +10 -1
- package/dist/Token.d.ts.map +1 -1
- package/dist/Token.js +14 -5
- package/dist/Token.js.map +1 -1
- package/dist/internal/compiler/closure.d.ts +1 -1
- package/dist/internal/compiler/closure.d.ts.map +1 -1
- package/dist/internal/compiler/closure.js +7 -17
- package/dist/internal/compiler/closure.js.map +1 -1
- package/dist/internal/compiler/first.d.ts.map +1 -1
- package/dist/internal/compiler/first.js +2 -1
- package/dist/internal/compiler/first.js.map +1 -1
- package/dist/internal/compiler/graph.d.ts +23 -1
- package/dist/internal/compiler/graph.d.ts.map +1 -1
- package/dist/internal/compiler/graph.js +27 -5
- package/dist/internal/compiler/graph.js.map +1 -1
- package/dist/internal/compiler/jit.d.ts.map +1 -1
- package/dist/internal/compiler/jit.js +26 -13
- package/dist/internal/compiler/jit.js.map +1 -1
- package/dist/internal/compiler/registry.d.ts +1 -1
- package/dist/internal/compiler/wasm.d.ts.map +1 -1
- package/dist/internal/compiler/wasm.js +105 -34
- package/dist/internal/compiler/wasm.js.map +1 -1
- package/dist/internal/node.d.ts +19 -1
- package/dist/internal/node.d.ts.map +1 -1
- package/dist/internal/node.js +42 -8
- package/dist/internal/node.js.map +1 -1
- package/dist/internal/run.d.ts +3 -1
- package/dist/internal/run.d.ts.map +1 -1
- package/dist/internal/run.js +4 -2
- package/dist/internal/run.js.map +1 -1
- package/dist/internal/stack.d.ts +8 -0
- package/dist/internal/stack.d.ts.map +1 -1
- package/dist/internal/stack.js +17 -1
- package/dist/internal/stack.js.map +1 -1
- package/dist/internal/tokens.d.ts +10 -3
- package/dist/internal/tokens.d.ts.map +1 -1
- package/dist/internal/tokens.js +22 -9
- package/dist/internal/tokens.js.map +1 -1
- package/dist/internal/tuple.d.ts +6 -0
- package/dist/internal/tuple.d.ts.map +1 -0
- package/dist/internal/tuple.js +170 -0
- package/dist/internal/tuple.js.map +1 -0
- package/package.json +17 -10
- package/src/Ascii.ts +5 -3
- package/src/Binary.ts +24 -3
- package/src/Bits.ts +4 -0
- package/src/Input.ts +37 -4
- package/src/ParseError.ts +206 -52
- package/src/Parser.ts +19 -29
- package/src/ParserSchema.ts +1 -2
- package/src/ParserStream.ts +336 -62
- package/src/Text.ts +16 -12
- package/src/Token.ts +14 -5
- package/src/internal/compiler/closure.ts +9 -18
- package/src/internal/compiler/first.ts +6 -0
- package/src/internal/compiler/graph.ts +32 -4
- package/src/internal/compiler/jit.ts +37 -17
- package/src/internal/compiler/wasm.ts +141 -68
- package/src/internal/node.ts +49 -3
- package/src/internal/run.ts +4 -2
- package/src/internal/stack.ts +17 -0
- package/src/internal/tokens.ts +20 -10
- package/src/internal/tuple.ts +141 -0
package/src/ParserStream.ts
CHANGED
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
* retried once more data arrives. When the stream ends, the leftover input
|
|
8
8
|
* is parsed in complete mode.
|
|
9
9
|
*
|
|
10
|
+
* Every runner bounds what it buffers: `maxBufferSize` (see
|
|
11
|
+
* {@link BufferOptions}) defaults to 64 MiB ({@link defaultMaxBufferSize}),
|
|
12
|
+
* and a stream that exceeds it fails with a {@link BufferLimitError}, so a
|
|
13
|
+
* socket or an HTTP body can be parsed without wrapping the stream first.
|
|
14
|
+
* Pass `Infinity` for input that is trusted to be large.
|
|
15
|
+
*
|
|
10
16
|
* @example
|
|
11
17
|
* ```ts
|
|
12
18
|
* import { Effect, Stream } from "effect"
|
|
@@ -38,6 +44,7 @@ import * as Run from "./internal/run.js"
|
|
|
38
44
|
import * as Stack from "./internal/stack.js"
|
|
39
45
|
import { isEmptyUnits } from "./internal/text.js"
|
|
40
46
|
import type { Parser } from "./Parser.js"
|
|
47
|
+
import { isParser } from "./Parser.js"
|
|
41
48
|
|
|
42
49
|
/**
|
|
43
50
|
* Input kinds that can be streamed in chunks.
|
|
@@ -47,32 +54,202 @@ import type { Parser } from "./Parser.js"
|
|
|
47
54
|
*/
|
|
48
55
|
export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
|
|
49
56
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
57
|
+
/**
|
|
58
|
+
* The default `maxBufferSize` of {@link parseMany}, {@link parseAll},
|
|
59
|
+
* {@link parseReadable} and {@link parseResponse}: 64 MiB, counted in tokens
|
|
60
|
+
* of the input (UTF-16 code units of a string, bytes of a `Uint8Array`,
|
|
61
|
+
* elements of an array).
|
|
62
|
+
*
|
|
63
|
+
* @since 0.6.0
|
|
64
|
+
* @category constants
|
|
65
|
+
*/
|
|
66
|
+
export const defaultMaxBufferSize: number = 64 * 1024 * 1024
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* A stream runner would have buffered more than its `maxBufferSize`: `size`
|
|
70
|
+
* tokens against a limit of `limit`. It is a different failure from a
|
|
71
|
+
* `ParseError`: the input is too large, not malformed (an HTTP server
|
|
72
|
+
* answers 413 for the one and 400 for the other), and nothing of the input
|
|
73
|
+
* is attached to it.
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```ts
|
|
77
|
+
* import { Effect, Stream } from "effect"
|
|
78
|
+
* import { Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
|
|
79
|
+
*
|
|
80
|
+
* const line = Parser.terminated(Token.takeTill("\n"), Token.literal("\n"))
|
|
81
|
+
*
|
|
82
|
+
* Stream.make("no newline, ", "ever").pipe(
|
|
83
|
+
* ParserStream.parseMany(line, { maxBufferSize: 8 }),
|
|
84
|
+
* Stream.runCollect,
|
|
85
|
+
* Effect.catchTag("BufferLimitError", (e) => Effect.succeed(`too large: ${e.size} > ${e.limit}`))
|
|
86
|
+
* )
|
|
87
|
+
* ```
|
|
88
|
+
*
|
|
89
|
+
* @since 0.6.0
|
|
90
|
+
* @category errors
|
|
91
|
+
*/
|
|
92
|
+
export class BufferLimitError
|
|
93
|
+
extends /*#__PURE__*/ Data.TaggedError("BufferLimitError")<{
|
|
94
|
+
/** The `maxBufferSize` in force, in tokens. */
|
|
95
|
+
readonly limit: number
|
|
96
|
+
/** The tokens the runner held, or was about to hold (for `parseResponse`, the declared `Content-Length`). */
|
|
97
|
+
readonly size: number
|
|
98
|
+
}>
|
|
99
|
+
{
|
|
100
|
+
override get message(): string {
|
|
101
|
+
return `the input is larger than maxBufferSize: ${this.size} tokens, limit ${this.limit}`
|
|
66
102
|
}
|
|
67
|
-
// SAFETY: both are chunks of one stream of `I`, so `b` is an array when `a` is
|
|
68
|
-
const joined: Chunkable = [...a, ...(b as ReadonlyArray<unknown>)]
|
|
69
|
-
// SAFETY: `I` is `a`'s type, an array here, and `joined` is a new array of both chunks' elements
|
|
70
|
-
return joined as I
|
|
71
103
|
}
|
|
72
104
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
105
|
+
/**
|
|
106
|
+
* The limit on buffered input that the stream runners share.
|
|
107
|
+
*
|
|
108
|
+
* @since 0.6.0
|
|
109
|
+
* @category models
|
|
110
|
+
*/
|
|
111
|
+
export interface BufferOptions {
|
|
112
|
+
/**
|
|
113
|
+
* The most tokens a runner buffers (UTF-16 code units of a string, bytes
|
|
114
|
+
* of a `Uint8Array`, elements of an array) before it fails with a
|
|
115
|
+
* {@link BufferLimitError}. Defaults to {@link defaultMaxBufferSize}
|
|
116
|
+
* (64 MiB); `Infinity` turns the limit off.
|
|
117
|
+
*
|
|
118
|
+
* {@link parseMany} counts the input it has not consumed yet, which is one
|
|
119
|
+
* unfinished value: complete values in a chunk are parsed and emitted
|
|
120
|
+
* first, whatever the chunk's size. {@link parseAll},
|
|
121
|
+
* {@link parseReadable} and {@link parseResponse} hold the whole input, so
|
|
122
|
+
* they count all of it.
|
|
123
|
+
*/
|
|
124
|
+
readonly maxBufferSize?: number | undefined
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The limit `options` asks for: a positive number of tokens, or `Infinity`. */
|
|
128
|
+
const limitOf = (options: BufferOptions | undefined): number => {
|
|
129
|
+
const limit = options?.maxBufferSize ?? defaultMaxBufferSize
|
|
130
|
+
if (!(limit > 0)) throw new RangeError("maxBufferSize must be a positive number of tokens, or Infinity")
|
|
131
|
+
return limit
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Above this many unconsumed tokens, {@link parseMany} stops retrying its
|
|
136
|
+
* unfinished value on every chunk (see {@link Pending.due}).
|
|
137
|
+
*/
|
|
138
|
+
const BackoffThreshold = 64 * 1024
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The input `parseMany` has not consumed yet: what is left of the chunks
|
|
142
|
+
* after the values parsed from them, which is one unfinished value.
|
|
143
|
+
*
|
|
144
|
+
* Appending is amortized constant time per token: text is concatenated (a
|
|
145
|
+
* rope until it is parsed), bytes go into a buffer that doubles, and
|
|
146
|
+
* elements are pushed. Bytes that a parse has seen are never overwritten,
|
|
147
|
+
* since the values parsed from them may be views of the buffer
|
|
148
|
+
* (`Token.take` returns a `subarray`): the buffer is only ever appended to
|
|
149
|
+
* past its end, or replaced by a larger one.
|
|
150
|
+
*/
|
|
151
|
+
class Pending<I extends Chunkable> {
|
|
152
|
+
private kind: 0 | 1 | 2 | undefined
|
|
153
|
+
private text = ""
|
|
154
|
+
private bytes: Uint8Array = new Uint8Array(0)
|
|
155
|
+
/** Where the unconsumed bytes start and end in `bytes`, and how far `bytes` may be written. */
|
|
156
|
+
private start = 0
|
|
157
|
+
private end = 0
|
|
158
|
+
private capacity = 0
|
|
159
|
+
private items: Array<unknown> = []
|
|
160
|
+
/** The unconsumed size at the last attempt that ran out of input, or 0 when none is outstanding. */
|
|
161
|
+
private attempted = 0
|
|
162
|
+
|
|
163
|
+
/** The number of unconsumed tokens. */
|
|
164
|
+
get size(): number {
|
|
165
|
+
return this.kind === 1 ? this.end - this.start : this.kind === 2 ? this.items.length : this.text.length
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
get isEmpty(): boolean {
|
|
169
|
+
return this.kind === undefined
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
append(chunk: I): void {
|
|
173
|
+
if (Predicate.isString(chunk)) {
|
|
174
|
+
this.kind ??= 0
|
|
175
|
+
this.text += chunk
|
|
176
|
+
} else if (chunk instanceof Uint8Array) {
|
|
177
|
+
this.kind ??= 1
|
|
178
|
+
this.appendBytes(chunk)
|
|
179
|
+
} else {
|
|
180
|
+
this.kind ??= 2
|
|
181
|
+
for (let i = 0; i < chunk.length; i++) this.items.push(chunk[i])
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
private appendBytes(chunk: Uint8Array): void {
|
|
186
|
+
const held = this.end - this.start
|
|
187
|
+
if (held === 0) {
|
|
188
|
+
// Nothing is pending: the chunk is the buffer, uncopied. It is the
|
|
189
|
+
// caller's, so it is never written to (`capacity` is its end)
|
|
190
|
+
this.bytes = chunk
|
|
191
|
+
this.start = 0
|
|
192
|
+
this.end = this.capacity = chunk.length
|
|
193
|
+
return
|
|
194
|
+
}
|
|
195
|
+
const needed = held + chunk.length
|
|
196
|
+
if (this.end + chunk.length > this.capacity) {
|
|
197
|
+
// Twice what is needed, so appending costs amortized constant time per byte
|
|
198
|
+
const grown = new Uint8Array(Math.max(needed * 2, 4096))
|
|
199
|
+
grown.set(this.bytes.subarray(this.start, this.end))
|
|
200
|
+
this.bytes = grown
|
|
201
|
+
this.start = 0
|
|
202
|
+
this.end = held
|
|
203
|
+
this.capacity = grown.length
|
|
204
|
+
}
|
|
205
|
+
this.bytes.set(chunk, this.end)
|
|
206
|
+
this.end += chunk.length
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** The unconsumed input. */
|
|
210
|
+
input(): I {
|
|
211
|
+
if (this.kind === 1) {
|
|
212
|
+
// SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
|
|
213
|
+
return this.bytes.subarray(this.start, this.end) as I
|
|
214
|
+
}
|
|
215
|
+
if (this.kind === 2) {
|
|
216
|
+
const items: Chunkable = this.items
|
|
217
|
+
// SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
|
|
218
|
+
return items as I
|
|
219
|
+
}
|
|
220
|
+
// SAFETY: `kind` 0 is set by the first chunk being a string, so `I` is `string`
|
|
221
|
+
return this.text as I
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Drops the first `count` tokens, which values were parsed from. */
|
|
225
|
+
consume(count: number): void {
|
|
226
|
+
if (count === 0) return
|
|
227
|
+
if (this.kind === 1) this.start += count
|
|
228
|
+
// A new array: the one parsed is the `input` of the state that read it
|
|
229
|
+
else if (this.kind === 2) this.items = this.items.slice(count)
|
|
230
|
+
else this.text = this.text.slice(count)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Records whether the attempt just made left an unfinished value. */
|
|
234
|
+
attemptedWith(unfinished: boolean): void {
|
|
235
|
+
this.attempted = unfinished ? this.size : 0
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Whether to parse now. A value that is still unfinished is parsed again
|
|
240
|
+
* from its start, so retrying on every chunk costs the square of its
|
|
241
|
+
* size: a megabyte with no delimiter, in 1 KiB chunks, took seconds. Past
|
|
242
|
+
* {@link BackoffThreshold} tokens, the next attempt therefore waits until
|
|
243
|
+
* the buffer has grown by an eighth, which keeps the total work linear
|
|
244
|
+
* (each token is re-read at most about nine times). Smaller values are
|
|
245
|
+
* retried on every chunk, as cheaply as before, so a record is emitted
|
|
246
|
+
* as soon as its last chunk arrives.
|
|
247
|
+
*/
|
|
248
|
+
due(): boolean {
|
|
249
|
+
const attempted = this.attempted
|
|
250
|
+
return attempted <= BackoffThreshold || this.size - attempted >= attempted >>> 3
|
|
251
|
+
}
|
|
252
|
+
}
|
|
76
253
|
|
|
77
254
|
const noProgress = (input: Input, offset: number) =>
|
|
78
255
|
ParseError.fromFailure<never>(
|
|
@@ -82,12 +259,12 @@ const noProgress = (input: Input, offset: number) =>
|
|
|
82
259
|
)
|
|
83
260
|
|
|
84
261
|
/**
|
|
85
|
-
* Options for {@link parseMany}.
|
|
262
|
+
* Options for {@link parseMany} and {@link parseAll}.
|
|
86
263
|
*
|
|
87
264
|
* @since 0.1.0
|
|
88
265
|
* @category models
|
|
89
266
|
*/
|
|
90
|
-
export interface ParseManyOptions<I> {
|
|
267
|
+
export interface ParseManyOptions<I> extends BufferOptions {
|
|
91
268
|
/** The empty input, used to initialise the buffer (inferred from the first chunk if omitted). */
|
|
92
269
|
readonly empty?: Source<I> | undefined
|
|
93
270
|
}
|
|
@@ -106,6 +283,31 @@ export interface ParseManyOptions<I> {
|
|
|
106
283
|
* cut short by its chunk is not: compiled code reports `Incomplete` where
|
|
107
284
|
* the interpreter does.
|
|
108
285
|
*
|
|
286
|
+
* **Untrusted streams.** The input not yet consumed (one unfinished value)
|
|
287
|
+
* is limited to `maxBufferSize` tokens, 64 MiB by default
|
|
288
|
+
* ({@link defaultMaxBufferSize}); past it the stream fails with a
|
|
289
|
+
* {@link BufferLimitError}, so a peer that never sends the delimiter cannot
|
|
290
|
+
* grow the buffer without bound. An unfinished value is parsed again from
|
|
291
|
+
* its start when more input arrives. Up to 64 KiB that happens on every
|
|
292
|
+
* chunk; a larger one is retried only once the buffer has grown by an
|
|
293
|
+
* eighth, so the work stays linear in its size. The cost: the last chunk of
|
|
294
|
+
* a value larger than 64 KiB may not complete it in the output until more
|
|
295
|
+
* input arrives or the stream ends, when everything left is parsed.
|
|
296
|
+
*
|
|
297
|
+
* @example
|
|
298
|
+
* ```ts
|
|
299
|
+
* import { Effect, Stream } from "effect"
|
|
300
|
+
* import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
|
|
301
|
+
*
|
|
302
|
+
* const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
|
|
303
|
+
*
|
|
304
|
+
* const program = Stream.make("1\n2", "3\n", "4\n").pipe(
|
|
305
|
+
* ParserStream.parseMany(line, { maxBufferSize: 1024 }),
|
|
306
|
+
* Stream.runCollect
|
|
307
|
+
* )
|
|
308
|
+
* // Effect.runSync(program) => [1, 23, 4]
|
|
309
|
+
* ```
|
|
310
|
+
*
|
|
109
311
|
* @since 0.1.0
|
|
110
312
|
* @category combinators
|
|
111
313
|
*/
|
|
@@ -113,27 +315,34 @@ export const parseMany: {
|
|
|
113
315
|
<A, E, R, I extends Chunkable>(
|
|
114
316
|
parser: Parser<A, E, R, I>,
|
|
115
317
|
options?: ParseManyOptions<I>
|
|
116
|
-
): <E0, R0>(
|
|
318
|
+
): <E0, R0>(
|
|
319
|
+
self: Stream.Stream<Source<I>, E0, R0>
|
|
320
|
+
) => Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
117
321
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
118
322
|
self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
|
|
119
323
|
parser: Parser<A, E, R, I>,
|
|
120
324
|
options?: ParseManyOptions<I>
|
|
121
|
-
): Stream.Stream<A, E0 | ParseError.ParseError<E
|
|
325
|
+
): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
122
326
|
} = /*#__PURE__*/ dual(
|
|
123
327
|
(args) => Stream.isStream(args[0]),
|
|
124
328
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
125
329
|
self: Stream.Stream<Source<I>, E0, R0>,
|
|
126
330
|
parser: Parser<A, E, R, I>,
|
|
127
331
|
options?: ParseManyOptions<I>
|
|
128
|
-
): Stream.Stream<A, E0 | ParseError.ParseError<E
|
|
129
|
-
|
|
332
|
+
): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
|
|
333
|
+
const limit = limitOf(options)
|
|
334
|
+
return Stream.suspend(() => {
|
|
130
335
|
const step = parser.step
|
|
131
|
-
|
|
336
|
+
const pending = new Pending<I>()
|
|
337
|
+
if (options?.empty !== undefined) pending.append(fromSource<I>(options.empty))
|
|
132
338
|
|
|
133
|
-
const drain = (
|
|
339
|
+
const drain = (
|
|
340
|
+
context: any,
|
|
341
|
+
partial: boolean
|
|
342
|
+
): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
|
|
134
343
|
const out: Array<A> = []
|
|
135
|
-
if (
|
|
136
|
-
const input =
|
|
344
|
+
if (pending.isEmpty) return Effect.succeed(out)
|
|
345
|
+
const input = pending.input()
|
|
137
346
|
// One state for the whole buffer, reset before each value
|
|
138
347
|
const state = new State(input)
|
|
139
348
|
let offset = 0
|
|
@@ -151,16 +360,19 @@ export const parseMany: {
|
|
|
151
360
|
out.push(a)
|
|
152
361
|
offset = state.pos
|
|
153
362
|
}
|
|
154
|
-
|
|
155
|
-
|
|
363
|
+
pending.consume(offset)
|
|
364
|
+
pending.attemptedWith(offset < input.length)
|
|
365
|
+
// Only what no value consumed counts: an unfinished value larger than the limit
|
|
366
|
+
const size = input.length - offset
|
|
367
|
+
return size > limit ? Effect.fail(new BufferLimitError({ limit, size })) : Effect.succeed(out)
|
|
156
368
|
}
|
|
157
369
|
|
|
158
370
|
const feed = (source: Source<I>) =>
|
|
159
371
|
Effect.flatMap(Effect.context<R>(), (context) =>
|
|
160
|
-
Effect.suspend(() => {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
return drain(context, true)
|
|
372
|
+
Effect.suspend((): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
|
|
373
|
+
pending.append(fromSource<I>(source))
|
|
374
|
+
// Over the limit, parse whatever the back-off says, so that complete values are not counted
|
|
375
|
+
return pending.due() || pending.size > limit ? drain(context, true) : Effect.succeed([])
|
|
164
376
|
})
|
|
165
377
|
)
|
|
166
378
|
|
|
@@ -172,6 +384,7 @@ export const parseMany: {
|
|
|
172
384
|
Stream.concat(Stream.unwrap(Effect.map(finish, Stream.fromIterable)))
|
|
173
385
|
)
|
|
174
386
|
})
|
|
387
|
+
}
|
|
175
388
|
)
|
|
176
389
|
|
|
177
390
|
/**
|
|
@@ -242,6 +455,11 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
242
455
|
return this.text as I
|
|
243
456
|
}
|
|
244
457
|
|
|
458
|
+
/** The tokens held: the whole input so far. */
|
|
459
|
+
get size(): number {
|
|
460
|
+
return this.length
|
|
461
|
+
}
|
|
462
|
+
|
|
245
463
|
push(chunk: I): void {
|
|
246
464
|
if (isEmptyUnits(chunk)) return
|
|
247
465
|
this.append(chunk)
|
|
@@ -327,6 +545,11 @@ class Incremental<A, E, I extends Chunkable> {
|
|
|
327
545
|
* callbacks should be pure: the elements parsed before a checkpoint are
|
|
328
546
|
* not parsed again, and on failure everything is parsed again.
|
|
329
547
|
*
|
|
548
|
+
* The whole input is held until the stream ends, so it is limited to
|
|
549
|
+
* `maxBufferSize` tokens (64 MiB by default, see {@link BufferOptions}):
|
|
550
|
+
* a stream that would exceed it fails with a {@link BufferLimitError}
|
|
551
|
+
* before the chunk that crosses the limit is buffered.
|
|
552
|
+
*
|
|
330
553
|
* @since 0.1.0
|
|
331
554
|
* @category combinators
|
|
332
555
|
*/
|
|
@@ -334,29 +557,42 @@ export const parseAll: {
|
|
|
334
557
|
<A, E, R, I extends Chunkable>(
|
|
335
558
|
parser: Parser<A, E, R, I>,
|
|
336
559
|
options?: ParseManyOptions<I>
|
|
337
|
-
): <E0, R0>(
|
|
560
|
+
): <E0, R0>(
|
|
561
|
+
self: Stream.Stream<Source<I>, E0, R0>
|
|
562
|
+
) => Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
338
563
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
339
564
|
self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
|
|
340
565
|
parser: Parser<A, E, R, I>,
|
|
341
566
|
options?: ParseManyOptions<I>
|
|
342
|
-
): Effect.Effect<A, E0 | ParseError.ParseError<E
|
|
567
|
+
): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
|
|
343
568
|
} = /*#__PURE__*/ dual(
|
|
344
569
|
(args) => Stream.isStream(args[0]),
|
|
345
570
|
<I extends Chunkable, E0, R0, A, E, R>(
|
|
346
571
|
self: Stream.Stream<Source<I>, E0, R0>,
|
|
347
572
|
parser: Parser<A, E, R, I>,
|
|
348
573
|
options?: ParseManyOptions<I>
|
|
349
|
-
): Effect.Effect<A, E0 | ParseError.ParseError<E
|
|
350
|
-
|
|
574
|
+
): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
|
|
575
|
+
const limit = limitOf(options)
|
|
576
|
+
return Effect.flatMap(Effect.context<R>(), (context) =>
|
|
351
577
|
Effect.suspend(() => {
|
|
352
578
|
const empty = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
|
|
353
579
|
const incremental = new Incremental(parser, context, empty)
|
|
354
580
|
return Effect.flatMap(
|
|
355
|
-
Stream.runForEach(self, (
|
|
581
|
+
Stream.runForEach(self, (source) =>
|
|
582
|
+
Effect.suspend(() => {
|
|
583
|
+
const chunk = fromSource<I>(source)
|
|
584
|
+
// Checked before the chunk is copied in: the whole input is held, so all of it counts
|
|
585
|
+
const size = incremental.size + chunk.length
|
|
586
|
+
if (size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
|
|
587
|
+
incremental.push(chunk)
|
|
588
|
+
return Effect.void
|
|
589
|
+
})
|
|
590
|
+
),
|
|
356
591
|
() => Effect.suspend(() => incremental.finish())
|
|
357
592
|
)
|
|
358
593
|
})
|
|
359
594
|
)
|
|
595
|
+
}
|
|
360
596
|
)
|
|
361
597
|
|
|
362
598
|
/**
|
|
@@ -368,11 +604,17 @@ export const parseAll: {
|
|
|
368
604
|
*/
|
|
369
605
|
export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ readonly cause: unknown }> {}
|
|
370
606
|
|
|
607
|
+
/** Whether a web-stream runner was called with its stream first: the other form starts with the parser. */
|
|
608
|
+
const isDataFirst = (args: IArguments): boolean => !isParser(args[0])
|
|
609
|
+
|
|
371
610
|
/**
|
|
372
611
|
* Parses a web `ReadableStream` of bytes, such as the body of a `fetch`
|
|
373
612
|
* response, incrementally (see {@link parseAll}): the bytes are parsed as
|
|
374
613
|
* they arrive, without waiting for the end of the stream or decoding them
|
|
375
|
-
* first. Use a byte parser, e.g. one built from `Text.utf8`.
|
|
614
|
+
* first. Use a byte parser, e.g. one built from `Text.utf8`. The stream is
|
|
615
|
+
* limited to `maxBufferSize` bytes (64 MiB by default, see
|
|
616
|
+
* {@link BufferOptions}), past which it fails with a
|
|
617
|
+
* {@link BufferLimitError}.
|
|
376
618
|
*
|
|
377
619
|
* @example
|
|
378
620
|
* ```ts
|
|
@@ -391,30 +633,52 @@ export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ rea
|
|
|
391
633
|
*/
|
|
392
634
|
export const parseReadable: {
|
|
393
635
|
<A, E, R>(
|
|
394
|
-
parser: Parser<A, E, R, Uint8Array
|
|
395
|
-
|
|
636
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
637
|
+
options?: BufferOptions
|
|
638
|
+
): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
396
639
|
<A, E, R>(
|
|
397
640
|
self: ReadableStream<BinaryInput>,
|
|
398
|
-
parser: Parser<A, E, R, Uint8Array
|
|
399
|
-
|
|
641
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
642
|
+
options?: BufferOptions
|
|
643
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
400
644
|
} = /*#__PURE__*/ dual(
|
|
401
|
-
|
|
645
|
+
isDataFirst,
|
|
402
646
|
<A, E, R>(
|
|
403
647
|
self: ReadableStream<BinaryInput>,
|
|
404
|
-
parser: Parser<A, E, R, Uint8Array
|
|
405
|
-
|
|
648
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
649
|
+
options?: BufferOptions
|
|
650
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
|
|
406
651
|
parseAll(
|
|
407
652
|
Stream.fromReadableStream({ evaluate: () => self, onError: (cause) => new ReadError({ cause }) }),
|
|
408
653
|
parser,
|
|
409
|
-
{ empty: new Uint8Array(0) }
|
|
654
|
+
{ empty: new Uint8Array(0), maxBufferSize: options?.maxBufferSize }
|
|
410
655
|
)
|
|
411
656
|
)
|
|
412
657
|
|
|
658
|
+
/**
|
|
659
|
+
* The length a response declares for the body as it will be read, if it
|
|
660
|
+
* declares one: `Content-Length` counts the encoded body, so a response
|
|
661
|
+
* with a `Content-Encoding` (which `fetch` decodes) declares nothing about
|
|
662
|
+
* the bytes parsed.
|
|
663
|
+
*/
|
|
664
|
+
const declaredLength = (response: Response): number | undefined => {
|
|
665
|
+
if (response.headers.has("content-encoding")) return undefined
|
|
666
|
+
const header = response.headers.get("content-length")
|
|
667
|
+
if (header === null || !/^[0-9]+$/.test(header)) return undefined
|
|
668
|
+
return Number(header)
|
|
669
|
+
}
|
|
670
|
+
|
|
413
671
|
/**
|
|
414
672
|
* Parses the body of a `fetch` `Response` incrementally, as it downloads
|
|
415
673
|
* (see {@link parseReadable}). A response without a body is parsed as empty
|
|
416
674
|
* input. The status is not checked.
|
|
417
675
|
*
|
|
676
|
+
* The body is limited to `maxBufferSize` bytes (64 MiB by default, see
|
|
677
|
+
* {@link BufferOptions}). A response whose `Content-Length` already exceeds
|
|
678
|
+
* the limit fails with a {@link BufferLimitError} before any of the body is
|
|
679
|
+
* read; the limit is enforced on the bytes read as well, since the header
|
|
680
|
+
* may be absent or wrong.
|
|
681
|
+
*
|
|
418
682
|
* @example
|
|
419
683
|
* ```ts
|
|
420
684
|
* import { Effect } from "effect"
|
|
@@ -423,7 +687,9 @@ export const parseReadable: {
|
|
|
423
687
|
* const lines = Parser.separated(Text.utf8.tillLineEnding, Text.utf8.lineEnding)
|
|
424
688
|
*
|
|
425
689
|
* const response = await fetch("https://example.com/log.txt")
|
|
426
|
-
* const value = await Effect.runPromise(
|
|
690
|
+
* const value = await Effect.runPromise(
|
|
691
|
+
* ParserStream.parseResponse(response, lines, { maxBufferSize: 8 * 1024 * 1024 })
|
|
692
|
+
* )
|
|
427
693
|
* ```
|
|
428
694
|
*
|
|
429
695
|
* @since 0.1.0
|
|
@@ -431,17 +697,25 @@ export const parseReadable: {
|
|
|
431
697
|
*/
|
|
432
698
|
export const parseResponse: {
|
|
433
699
|
<A, E, R>(
|
|
434
|
-
parser: Parser<A, E, R, Uint8Array
|
|
435
|
-
|
|
700
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
701
|
+
options?: BufferOptions
|
|
702
|
+
): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
436
703
|
<A, E, R>(
|
|
437
704
|
self: Response,
|
|
438
|
-
parser: Parser<A, E, R, Uint8Array
|
|
439
|
-
|
|
705
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
706
|
+
options?: BufferOptions
|
|
707
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
|
|
440
708
|
} = /*#__PURE__*/ dual(
|
|
441
|
-
|
|
709
|
+
isDataFirst,
|
|
442
710
|
<A, E, R>(
|
|
443
711
|
self: Response,
|
|
444
|
-
parser: Parser<A, E, R, Uint8Array
|
|
445
|
-
|
|
446
|
-
|
|
712
|
+
parser: Parser<A, E, R, Uint8Array>,
|
|
713
|
+
options?: BufferOptions
|
|
714
|
+
): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
|
|
715
|
+
Effect.suspend(() => {
|
|
716
|
+
const limit = limitOf(options)
|
|
717
|
+
const size = declaredLength(self)
|
|
718
|
+
if (size !== undefined && size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
|
|
719
|
+
return parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser, options)
|
|
720
|
+
})
|
|
447
721
|
)
|
package/src/Text.ts
CHANGED
|
@@ -46,6 +46,7 @@ import { codePointPredicate, compile, indexOf, matchLiteral } from "./internal/t
|
|
|
46
46
|
import { floatText as markFloatText } from "./internal/numbers.js"
|
|
47
47
|
import * as internal from "./internal/text.js"
|
|
48
48
|
import type { Node } from "./internal/node.js"
|
|
49
|
+
import { bounds, checkCount, controlCode } from "./internal/node.js"
|
|
49
50
|
import type { ErrorKind } from "./ParseError.js"
|
|
50
51
|
import type { Parser, Range } from "./Parser.js"
|
|
51
52
|
import * as P from "./Parser.js"
|
|
@@ -354,9 +355,8 @@ export interface TextParsers<I extends string | Uint8Array> {
|
|
|
354
355
|
}
|
|
355
356
|
|
|
356
357
|
const normalize = (r: Range | undefined): [number, number] => {
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
return [r.min ?? 0, r.max ?? Infinity]
|
|
358
|
+
const { max, min } = bounds(r)
|
|
359
|
+
return [min, max]
|
|
360
360
|
}
|
|
361
361
|
|
|
362
362
|
const isAsciiDigit = /*#__PURE__*/ codePointPredicate("isAsciiDigit", (c: number): boolean => c >= 48 && c <= 57, [
|
|
@@ -547,6 +547,7 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
|
|
|
547
547
|
}
|
|
548
548
|
|
|
549
549
|
const take = (count: number): Parser<string, never, never, I> => {
|
|
550
|
+
checkCount(count)
|
|
550
551
|
const expected = [count === 1 ? "1 character" : `${count} characters`]
|
|
551
552
|
return P.described<string, never, never, I>(
|
|
552
553
|
(state: State<any>) => {
|
|
@@ -719,8 +720,7 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
|
|
|
719
720
|
)
|
|
720
721
|
const floatText = markFloatText(taken(P.alt(caseless("nan"), P.seq(sign, unsignedFloat))).pipe(P.expected("number")))
|
|
721
722
|
|
|
722
|
-
const matchControl = (
|
|
723
|
-
const ctrl = control.codePointAt(0)
|
|
723
|
+
const matchControl = (ctrl: number) => {
|
|
724
724
|
return (state: State<any>): boolean => {
|
|
725
725
|
if (internal.next(enc, state) !== ctrl) return false
|
|
726
726
|
state.pos += internal.width
|
|
@@ -779,17 +779,21 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
|
|
|
779
779
|
P.named("hexUint")
|
|
780
780
|
),
|
|
781
781
|
float: floatText.pipe(P.map(toFloat), P.named("float")),
|
|
782
|
-
takeEscaped: (normal, control, escapable) =>
|
|
783
|
-
|
|
782
|
+
takeEscaped: (normal, control, escapable) => {
|
|
783
|
+
const ctrl = controlCode(control, internal.firstCodePoint)
|
|
784
|
+
return takeEscapedImpl(
|
|
784
785
|
normal,
|
|
785
|
-
matchControl(
|
|
786
|
+
matchControl(ctrl),
|
|
786
787
|
escapable,
|
|
787
788
|
(state, start) => internal.textIn(enc, state, start, state.pos),
|
|
788
|
-
|
|
789
|
+
ctrl,
|
|
789
790
|
enc
|
|
790
|
-
)
|
|
791
|
-
|
|
792
|
-
|
|
791
|
+
)
|
|
792
|
+
},
|
|
793
|
+
escaped: (normal, control, escape) => {
|
|
794
|
+
const ctrl = controlCode(control, internal.firstCodePoint)
|
|
795
|
+
return escapedImpl(normal, matchControl(ctrl), escape, ctrl, enc)
|
|
796
|
+
}
|
|
793
797
|
}
|
|
794
798
|
}
|
|
795
799
|
|
package/src/Token.ts
CHANGED
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
splitSurrogate
|
|
39
39
|
} from "./internal/tokens.js"
|
|
40
40
|
import type { ErrorKind } from "./ParseError.js"
|
|
41
|
-
import { bounds } from "./internal/node.js"
|
|
41
|
+
import { bounds, checkCount } from "./internal/node.js"
|
|
42
42
|
import type { Parser, Range } from "./Parser.js"
|
|
43
43
|
import { described } from "./Parser.js"
|
|
44
44
|
|
|
@@ -353,12 +353,22 @@ export const noneOf = <I extends Input = string>(
|
|
|
353
353
|
}
|
|
354
354
|
|
|
355
355
|
/**
|
|
356
|
-
* Takes exactly `count` tokens (winnow's and nom's `take`).
|
|
356
|
+
* Takes exactly `count` tokens (winnow's and nom's `take`). `count` must be
|
|
357
|
+
* a non-negative integer: anything else throws a `RangeError` here, when
|
|
358
|
+
* the parser is built.
|
|
359
|
+
*
|
|
360
|
+
* @example
|
|
361
|
+
* ```ts
|
|
362
|
+
* import { Parser, Token } from "@thomasfosterau/effect-parse"
|
|
363
|
+
*
|
|
364
|
+
* Parser.parsePeek(Token.take(2), "abc") // Result.succeed(["c", "ab"])
|
|
365
|
+
* ```
|
|
357
366
|
*
|
|
358
367
|
* @since 0.1.0
|
|
359
368
|
* @category slices
|
|
360
369
|
*/
|
|
361
370
|
export const take = <I extends Input = string>(count: number): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {
|
|
371
|
+
checkCount(count)
|
|
362
372
|
const expected = [count === 1 ? "1 token" : `${count} tokens`]
|
|
363
373
|
return described(
|
|
364
374
|
(state) => {
|
|
@@ -374,9 +384,8 @@ export const take = <I extends Input = string>(count: number): Parser<Slice<NoIn
|
|
|
374
384
|
}
|
|
375
385
|
|
|
376
386
|
const normalize = (r: Range | undefined): [number, number] => {
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
return [r.min ?? 0, r.max ?? Infinity]
|
|
387
|
+
const { max, min } = bounds(r)
|
|
388
|
+
return [min, max]
|
|
380
389
|
}
|
|
381
390
|
|
|
382
391
|
/** @internal */
|