@thomasfosterau/effect-parse 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/CHANGELOG.md +202 -0
  2. package/README.md +19 -1
  3. package/dist/Ascii.d.ts.map +1 -1
  4. package/dist/Ascii.js +5 -3
  5. package/dist/Ascii.js.map +1 -1
  6. package/dist/Binary.d.ts +5 -2
  7. package/dist/Binary.d.ts.map +1 -1
  8. package/dist/Binary.js +20 -3
  9. package/dist/Binary.js.map +1 -1
  10. package/dist/Bits.d.ts.map +1 -1
  11. package/dist/Bits.js +4 -0
  12. package/dist/Bits.js.map +1 -1
  13. package/dist/Input.d.ts +21 -3
  14. package/dist/Input.d.ts.map +1 -1
  15. package/dist/Input.js +31 -4
  16. package/dist/Input.js.map +1 -1
  17. package/dist/Nom.d.ts.map +1 -1
  18. package/dist/ParseError.d.ts +69 -11
  19. package/dist/ParseError.d.ts.map +1 -1
  20. package/dist/ParseError.js +129 -28
  21. package/dist/ParseError.js.map +1 -1
  22. package/dist/Parser.d.ts +10 -4
  23. package/dist/Parser.d.ts.map +1 -1
  24. package/dist/Parser.js +10 -24
  25. package/dist/Parser.js.map +1 -1
  26. package/dist/ParserSchema.d.ts.map +1 -1
  27. package/dist/ParserSchema.js +1 -2
  28. package/dist/ParserSchema.js.map +1 -1
  29. package/dist/ParserStream.d.ts +119 -12
  30. package/dist/ParserStream.d.ts.map +1 -1
  31. package/dist/ParserStream.js +282 -55
  32. package/dist/ParserStream.js.map +1 -1
  33. package/dist/Syntax.js +1 -1
  34. package/dist/Text.d.ts.map +1 -1
  35. package/dist/Text.js +13 -7
  36. package/dist/Text.js.map +1 -1
  37. package/dist/Token.d.ts +10 -1
  38. package/dist/Token.d.ts.map +1 -1
  39. package/dist/Token.js +14 -5
  40. package/dist/Token.js.map +1 -1
  41. package/dist/internal/compiler/closure.d.ts +1 -1
  42. package/dist/internal/compiler/closure.d.ts.map +1 -1
  43. package/dist/internal/compiler/closure.js +7 -17
  44. package/dist/internal/compiler/closure.js.map +1 -1
  45. package/dist/internal/compiler/first.d.ts.map +1 -1
  46. package/dist/internal/compiler/first.js +2 -1
  47. package/dist/internal/compiler/first.js.map +1 -1
  48. package/dist/internal/compiler/graph.d.ts +23 -1
  49. package/dist/internal/compiler/graph.d.ts.map +1 -1
  50. package/dist/internal/compiler/graph.js +27 -5
  51. package/dist/internal/compiler/graph.js.map +1 -1
  52. package/dist/internal/compiler/jit.d.ts.map +1 -1
  53. package/dist/internal/compiler/jit.js +26 -13
  54. package/dist/internal/compiler/jit.js.map +1 -1
  55. package/dist/internal/compiler/registry.d.ts +1 -1
  56. package/dist/internal/compiler/wasm.d.ts.map +1 -1
  57. package/dist/internal/compiler/wasm.js +105 -34
  58. package/dist/internal/compiler/wasm.js.map +1 -1
  59. package/dist/internal/node.d.ts +19 -1
  60. package/dist/internal/node.d.ts.map +1 -1
  61. package/dist/internal/node.js +42 -8
  62. package/dist/internal/node.js.map +1 -1
  63. package/dist/internal/run.d.ts +3 -1
  64. package/dist/internal/run.d.ts.map +1 -1
  65. package/dist/internal/run.js +4 -2
  66. package/dist/internal/run.js.map +1 -1
  67. package/dist/internal/stack.d.ts +8 -0
  68. package/dist/internal/stack.d.ts.map +1 -1
  69. package/dist/internal/stack.js +17 -1
  70. package/dist/internal/stack.js.map +1 -1
  71. package/dist/internal/tokens.d.ts +10 -3
  72. package/dist/internal/tokens.d.ts.map +1 -1
  73. package/dist/internal/tokens.js +22 -9
  74. package/dist/internal/tokens.js.map +1 -1
  75. package/dist/internal/tuple.d.ts +6 -0
  76. package/dist/internal/tuple.d.ts.map +1 -0
  77. package/dist/internal/tuple.js +170 -0
  78. package/dist/internal/tuple.js.map +1 -0
  79. package/package.json +17 -10
  80. package/src/Ascii.ts +5 -3
  81. package/src/Binary.ts +24 -3
  82. package/src/Bits.ts +4 -0
  83. package/src/Input.ts +37 -4
  84. package/src/ParseError.ts +206 -52
  85. package/src/Parser.ts +19 -29
  86. package/src/ParserSchema.ts +1 -2
  87. package/src/ParserStream.ts +336 -62
  88. package/src/Text.ts +16 -12
  89. package/src/Token.ts +14 -5
  90. package/src/internal/compiler/closure.ts +9 -18
  91. package/src/internal/compiler/first.ts +6 -0
  92. package/src/internal/compiler/graph.ts +32 -4
  93. package/src/internal/compiler/jit.ts +37 -17
  94. package/src/internal/compiler/wasm.ts +141 -68
  95. package/src/internal/node.ts +49 -3
  96. package/src/internal/run.ts +4 -2
  97. package/src/internal/stack.ts +17 -0
  98. package/src/internal/tokens.ts +20 -10
  99. package/src/internal/tuple.ts +141 -0
@@ -7,6 +7,12 @@
7
7
  * retried once more data arrives. When the stream ends, the leftover input
8
8
  * is parsed in complete mode.
9
9
  *
10
+ * Every runner bounds what it buffers: `maxBufferSize` (see
11
+ * {@link BufferOptions}) defaults to 64 MiB ({@link defaultMaxBufferSize}),
12
+ * and a stream that exceeds it fails with a {@link BufferLimitError}, so a
13
+ * socket or an HTTP body can be parsed without wrapping the stream first.
14
+ * Pass `Infinity` for input that is trusted to be large.
15
+ *
10
16
  * @example
11
17
  * ```ts
12
18
  * import { Effect, Stream } from "effect"
@@ -38,6 +44,7 @@ import * as Run from "./internal/run.js"
38
44
  import * as Stack from "./internal/stack.js"
39
45
  import { isEmptyUnits } from "./internal/text.js"
40
46
  import type { Parser } from "./Parser.js"
47
+ import { isParser } from "./Parser.js"
41
48
 
42
49
  /**
43
50
  * Input kinds that can be streamed in chunks.
@@ -47,32 +54,202 @@ import type { Parser } from "./Parser.js"
47
54
  */
48
55
  export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
49
56
 
50
- const concat = <I extends Chunkable>(first: I, b: I): I => {
51
- // Widened so the checks below narrow it (a type parameter does not narrow).
52
- const a: Chunkable = first
53
- if (Predicate.isString(a)) {
54
- // SAFETY: both are chunks of one stream of `I`, so `b` is a string when `a` is, and so is their concatenation
55
- return (a + (b as string)) as I
56
- }
57
- if (a instanceof Uint8Array) {
58
- if (a.byteLength === 0) return b
59
- // SAFETY: both are chunks of one stream of `I`, so `b` is a `Uint8Array` when `a` is
60
- const out = new Uint8Array(a.length + (b as Uint8Array).length)
61
- out.set(a)
62
- // SAFETY: both are chunks of one stream of `I`, so `b` is a `Uint8Array` when `a` is
63
- out.set(b as Uint8Array, a.length)
64
- // SAFETY: `I` is `a`'s type, a `Uint8Array` here, and `out` is a new `Uint8Array`
65
- return out as I
57
+ /**
58
+ * The default `maxBufferSize` of {@link parseMany}, {@link parseAll},
59
+ * {@link parseReadable} and {@link parseResponse}: 64 MiB, counted in tokens
60
+ * of the input (UTF-16 code units of a string, bytes of a `Uint8Array`,
61
+ * elements of an array).
62
+ *
63
+ * @since 0.6.0
64
+ * @category constants
65
+ */
66
+ export const defaultMaxBufferSize: number = 64 * 1024 * 1024
67
+
68
+ /**
69
+ * A stream runner would have buffered more than its `maxBufferSize`: `size`
70
+ * tokens against a limit of `limit`. It is a different failure from a
71
+ * `ParseError`: the input is too large, not malformed (an HTTP server
72
+ * answers 413 for the one and 400 for the other), and nothing of the input
73
+ * is attached to it.
74
+ *
75
+ * @example
76
+ * ```ts
77
+ * import { Effect, Stream } from "effect"
78
+ * import { Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
79
+ *
80
+ * const line = Parser.terminated(Token.takeTill("\n"), Token.literal("\n"))
81
+ *
82
+ * Stream.make("no newline, ", "ever").pipe(
83
+ * ParserStream.parseMany(line, { maxBufferSize: 8 }),
84
+ * Stream.runCollect,
85
+ * Effect.catchTag("BufferLimitError", (e) => Effect.succeed(`too large: ${e.size} > ${e.limit}`))
86
+ * )
87
+ * ```
88
+ *
89
+ * @since 0.6.0
90
+ * @category errors
91
+ */
92
+ export class BufferLimitError
93
+ extends /*#__PURE__*/ Data.TaggedError("BufferLimitError")<{
94
+ /** The `maxBufferSize` in force, in tokens. */
95
+ readonly limit: number
96
+ /** The tokens the runner held, or was about to hold (for `parseResponse`, the declared `Content-Length`). */
97
+ readonly size: number
98
+ }>
99
+ {
100
+ override get message(): string {
101
+ return `the input is larger than maxBufferSize: ${this.size} tokens, limit ${this.limit}`
66
102
  }
67
- // SAFETY: both are chunks of one stream of `I`, so `b` is an array when `a` is
68
- const joined: Chunkable = [...a, ...(b as ReadonlyArray<unknown>)]
69
- // SAFETY: `I` is `a`'s type, an array here, and `joined` is a new array of both chunks' elements
70
- return joined as I
71
103
  }
72
104
 
73
- const sliceFrom = <I extends Chunkable>(input: I, offset: number): I =>
74
- // SAFETY: slicing a string, `Uint8Array` or array returns a value of the same kind
75
- (Predicate.isString(input) ? input.slice(offset) : input.slice(offset)) as I
105
+ /**
106
+ * The limit on buffered input that the stream runners share.
107
+ *
108
+ * @since 0.6.0
109
+ * @category models
110
+ */
111
+ export interface BufferOptions {
112
+ /**
113
+ * The most tokens a runner buffers (UTF-16 code units of a string, bytes
114
+ * of a `Uint8Array`, elements of an array) before it fails with a
115
+ * {@link BufferLimitError}. Defaults to {@link defaultMaxBufferSize}
116
+ * (64 MiB); `Infinity` turns the limit off.
117
+ *
118
+ * {@link parseMany} counts the input it has not consumed yet, which is one
119
+ * unfinished value: complete values in a chunk are parsed and emitted
120
+ * first, whatever the chunk's size. {@link parseAll},
121
+ * {@link parseReadable} and {@link parseResponse} hold the whole input, so
122
+ * they count all of it.
123
+ */
124
+ readonly maxBufferSize?: number | undefined
125
+ }
126
+
127
+ /** The limit `options` asks for: a positive number of tokens, or `Infinity`. */
128
+ const limitOf = (options: BufferOptions | undefined): number => {
129
+ const limit = options?.maxBufferSize ?? defaultMaxBufferSize
130
+ if (!(limit > 0)) throw new RangeError("maxBufferSize must be a positive number of tokens, or Infinity")
131
+ return limit
132
+ }
133
+
134
+ /**
135
+ * Above this many unconsumed tokens, {@link parseMany} stops retrying its
136
+ * unfinished value on every chunk (see {@link Pending.due}).
137
+ */
138
+ const BackoffThreshold = 64 * 1024
139
+
140
+ /**
141
+ * The input `parseMany` has not consumed yet: what is left of the chunks
142
+ * after the values parsed from them, which is one unfinished value.
143
+ *
144
+ * Appending is amortized constant time per token: text is concatenated (a
145
+ * rope until it is parsed), bytes go into a buffer that doubles, and
146
+ * elements are pushed. Bytes that a parse has seen are never overwritten,
147
+ * since the values parsed from them may be views of the buffer
148
+ * (`Token.take` returns a `subarray`): the buffer is only ever appended to
149
+ * past its end, or replaced by a larger one.
150
+ */
151
+ class Pending<I extends Chunkable> {
152
+ private kind: 0 | 1 | 2 | undefined
153
+ private text = ""
154
+ private bytes: Uint8Array = new Uint8Array(0)
155
+ /** Where the unconsumed bytes start and end in `bytes`, and how far `bytes` may be written. */
156
+ private start = 0
157
+ private end = 0
158
+ private capacity = 0
159
+ private items: Array<unknown> = []
160
+ /** The unconsumed size at the last attempt that ran out of input, or 0 when none is outstanding. */
161
+ private attempted = 0
162
+
163
+ /** The number of unconsumed tokens. */
164
+ get size(): number {
165
+ return this.kind === 1 ? this.end - this.start : this.kind === 2 ? this.items.length : this.text.length
166
+ }
167
+
168
+ get isEmpty(): boolean {
169
+ return this.kind === undefined
170
+ }
171
+
172
+ append(chunk: I): void {
173
+ if (Predicate.isString(chunk)) {
174
+ this.kind ??= 0
175
+ this.text += chunk
176
+ } else if (chunk instanceof Uint8Array) {
177
+ this.kind ??= 1
178
+ this.appendBytes(chunk)
179
+ } else {
180
+ this.kind ??= 2
181
+ for (let i = 0; i < chunk.length; i++) this.items.push(chunk[i])
182
+ }
183
+ }
184
+
185
+ private appendBytes(chunk: Uint8Array): void {
186
+ const held = this.end - this.start
187
+ if (held === 0) {
188
+ // Nothing is pending: the chunk is the buffer, uncopied. It is the
189
+ // caller's, so it is never written to (`capacity` is its end)
190
+ this.bytes = chunk
191
+ this.start = 0
192
+ this.end = this.capacity = chunk.length
193
+ return
194
+ }
195
+ const needed = held + chunk.length
196
+ if (this.end + chunk.length > this.capacity) {
197
+ // Twice what is needed, so appending costs amortized constant time per byte
198
+ const grown = new Uint8Array(Math.max(needed * 2, 4096))
199
+ grown.set(this.bytes.subarray(this.start, this.end))
200
+ this.bytes = grown
201
+ this.start = 0
202
+ this.end = held
203
+ this.capacity = grown.length
204
+ }
205
+ this.bytes.set(chunk, this.end)
206
+ this.end += chunk.length
207
+ }
208
+
209
+ /** The unconsumed input. */
210
+ input(): I {
211
+ if (this.kind === 1) {
212
+ // SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
213
+ return this.bytes.subarray(this.start, this.end) as I
214
+ }
215
+ if (this.kind === 2) {
216
+ const items: Chunkable = this.items
217
+ // SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
218
+ return items as I
219
+ }
220
+ // SAFETY: `kind` 0 is set by the first chunk being a string, so `I` is `string`
221
+ return this.text as I
222
+ }
223
+
224
+ /** Drops the first `count` tokens, which values were parsed from. */
225
+ consume(count: number): void {
226
+ if (count === 0) return
227
+ if (this.kind === 1) this.start += count
228
+ // A new array: the one parsed is the `input` of the state that read it
229
+ else if (this.kind === 2) this.items = this.items.slice(count)
230
+ else this.text = this.text.slice(count)
231
+ }
232
+
233
+ /** Records whether the attempt just made left an unfinished value. */
234
+ attemptedWith(unfinished: boolean): void {
235
+ this.attempted = unfinished ? this.size : 0
236
+ }
237
+
238
+ /**
239
+ * Whether to parse now. A value that is still unfinished is parsed again
240
+ * from its start, so retrying on every chunk costs the square of its
241
+ * size: a megabyte with no delimiter, in 1 KiB chunks, took seconds. Past
242
+ * {@link BackoffThreshold} tokens, the next attempt therefore waits until
243
+ * the buffer has grown by an eighth, which keeps the total work linear
244
+ * (each token is re-read at most about nine times). Smaller values are
245
+ * retried on every chunk, as cheaply as before, so a record is emitted
246
+ * as soon as its last chunk arrives.
247
+ */
248
+ due(): boolean {
249
+ const attempted = this.attempted
250
+ return attempted <= BackoffThreshold || this.size - attempted >= attempted >>> 3
251
+ }
252
+ }
76
253
 
77
254
  const noProgress = (input: Input, offset: number) =>
78
255
  ParseError.fromFailure<never>(
@@ -82,12 +259,12 @@ const noProgress = (input: Input, offset: number) =>
82
259
  )
83
260
 
84
261
  /**
85
- * Options for {@link parseMany}.
262
+ * Options for {@link parseMany} and {@link parseAll}.
86
263
  *
87
264
  * @since 0.1.0
88
265
  * @category models
89
266
  */
90
- export interface ParseManyOptions<I> {
267
+ export interface ParseManyOptions<I> extends BufferOptions {
91
268
  /** The empty input, used to initialise the buffer (inferred from the first chunk if omitted). */
92
269
  readonly empty?: Source<I> | undefined
93
270
  }
@@ -106,6 +283,31 @@ export interface ParseManyOptions<I> {
106
283
  * cut short by its chunk is not: compiled code reports `Incomplete` where
107
284
  * the interpreter does.
108
285
  *
286
+ * **Untrusted streams.** The input not yet consumed (one unfinished value)
287
+ * is limited to `maxBufferSize` tokens, 64 MiB by default
288
+ * ({@link defaultMaxBufferSize}); past it the stream fails with a
289
+ * {@link BufferLimitError}, so a peer that never sends the delimiter cannot
290
+ * grow the buffer without bound. An unfinished value is parsed again from
291
+ * its start when more input arrives. Up to 64 KiB that happens on every
292
+ * chunk; a larger one is retried only once the buffer has grown by an
293
+ * eighth, so the work stays linear in its size. The cost: the last chunk of
294
+ * a value larger than 64 KiB may not complete it in the output until more
295
+ * input arrives or the stream ends, when everything left is parsed.
296
+ *
297
+ * @example
298
+ * ```ts
299
+ * import { Effect, Stream } from "effect"
300
+ * import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
301
+ *
302
+ * const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
303
+ *
304
+ * const program = Stream.make("1\n2", "3\n", "4\n").pipe(
305
+ * ParserStream.parseMany(line, { maxBufferSize: 1024 }),
306
+ * Stream.runCollect
307
+ * )
308
+ * // Effect.runSync(program) => [1, 23, 4]
309
+ * ```
310
+ *
109
311
  * @since 0.1.0
110
312
  * @category combinators
111
313
  */
@@ -113,27 +315,34 @@ export const parseMany: {
113
315
  <A, E, R, I extends Chunkable>(
114
316
  parser: Parser<A, E, R, I>,
115
317
  options?: ParseManyOptions<I>
116
- ): <E0, R0>(self: Stream.Stream<Source<I>, E0, R0>) => Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R>
318
+ ): <E0, R0>(
319
+ self: Stream.Stream<Source<I>, E0, R0>
320
+ ) => Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
117
321
  <I extends Chunkable, E0, R0, A, E, R>(
118
322
  self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
119
323
  parser: Parser<A, E, R, I>,
120
324
  options?: ParseManyOptions<I>
121
- ): Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R>
325
+ ): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
122
326
  } = /*#__PURE__*/ dual(
123
327
  (args) => Stream.isStream(args[0]),
124
328
  <I extends Chunkable, E0, R0, A, E, R>(
125
329
  self: Stream.Stream<Source<I>, E0, R0>,
126
330
  parser: Parser<A, E, R, I>,
127
331
  options?: ParseManyOptions<I>
128
- ): Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R> =>
129
- Stream.suspend(() => {
332
+ ): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
333
+ const limit = limitOf(options)
334
+ return Stream.suspend(() => {
130
335
  const step = parser.step
131
- let buffer: I | undefined = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
336
+ const pending = new Pending<I>()
337
+ if (options?.empty !== undefined) pending.append(fromSource<I>(options.empty))
132
338
 
133
- const drain = (context: any, partial: boolean): Effect.Effect<Array<A>, ParseError.ParseError<E>> => {
339
+ const drain = (
340
+ context: any,
341
+ partial: boolean
342
+ ): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
134
343
  const out: Array<A> = []
135
- if (buffer === undefined) return Effect.succeed(out)
136
- const input = buffer
344
+ if (pending.isEmpty) return Effect.succeed(out)
345
+ const input = pending.input()
137
346
  // One state for the whole buffer, reset before each value
138
347
  const state = new State(input)
139
348
  let offset = 0
@@ -151,16 +360,19 @@ export const parseMany: {
151
360
  out.push(a)
152
361
  offset = state.pos
153
362
  }
154
- buffer = sliceFrom(input, offset)
155
- return Effect.succeed(out)
363
+ pending.consume(offset)
364
+ pending.attemptedWith(offset < input.length)
365
+ // Only what no value consumed counts: an unfinished value larger than the limit
366
+ const size = input.length - offset
367
+ return size > limit ? Effect.fail(new BufferLimitError({ limit, size })) : Effect.succeed(out)
156
368
  }
157
369
 
158
370
  const feed = (source: Source<I>) =>
159
371
  Effect.flatMap(Effect.context<R>(), (context) =>
160
- Effect.suspend(() => {
161
- const chunk = fromSource<I>(source)
162
- buffer = buffer === undefined ? chunk : concat(buffer, chunk)
163
- return drain(context, true)
372
+ Effect.suspend((): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
373
+ pending.append(fromSource<I>(source))
374
+ // Over the limit, parse whatever the back-off says, so that complete values are not counted
375
+ return pending.due() || pending.size > limit ? drain(context, true) : Effect.succeed([])
164
376
  })
165
377
  )
166
378
 
@@ -172,6 +384,7 @@ export const parseMany: {
172
384
  Stream.concat(Stream.unwrap(Effect.map(finish, Stream.fromIterable)))
173
385
  )
174
386
  })
387
+ }
175
388
  )
176
389
 
177
390
  /**
@@ -242,6 +455,11 @@ class Incremental<A, E, I extends Chunkable> {
242
455
  return this.text as I
243
456
  }
244
457
 
458
+ /** The tokens held: the whole input so far. */
459
+ get size(): number {
460
+ return this.length
461
+ }
462
+
245
463
  push(chunk: I): void {
246
464
  if (isEmptyUnits(chunk)) return
247
465
  this.append(chunk)
@@ -327,6 +545,11 @@ class Incremental<A, E, I extends Chunkable> {
327
545
  * callbacks should be pure: the elements parsed before a checkpoint are
328
546
  * not parsed again, and on failure everything is parsed again.
329
547
  *
548
+ * The whole input is held until the stream ends, so it is limited to
549
+ * `maxBufferSize` tokens (64 MiB by default, see {@link BufferOptions}):
550
+ * a stream that would exceed it fails with a {@link BufferLimitError}
551
+ * before the chunk that crosses the limit is buffered.
552
+ *
330
553
  * @since 0.1.0
331
554
  * @category combinators
332
555
  */
@@ -334,29 +557,42 @@ export const parseAll: {
334
557
  <A, E, R, I extends Chunkable>(
335
558
  parser: Parser<A, E, R, I>,
336
559
  options?: ParseManyOptions<I>
337
- ): <E0, R0>(self: Stream.Stream<Source<I>, E0, R0>) => Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R>
560
+ ): <E0, R0>(
561
+ self: Stream.Stream<Source<I>, E0, R0>
562
+ ) => Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
338
563
  <I extends Chunkable, E0, R0, A, E, R>(
339
564
  self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
340
565
  parser: Parser<A, E, R, I>,
341
566
  options?: ParseManyOptions<I>
342
- ): Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R>
567
+ ): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
343
568
  } = /*#__PURE__*/ dual(
344
569
  (args) => Stream.isStream(args[0]),
345
570
  <I extends Chunkable, E0, R0, A, E, R>(
346
571
  self: Stream.Stream<Source<I>, E0, R0>,
347
572
  parser: Parser<A, E, R, I>,
348
573
  options?: ParseManyOptions<I>
349
- ): Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R> =>
350
- Effect.flatMap(Effect.context<R>(), (context) =>
574
+ ): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
575
+ const limit = limitOf(options)
576
+ return Effect.flatMap(Effect.context<R>(), (context) =>
351
577
  Effect.suspend(() => {
352
578
  const empty = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
353
579
  const incremental = new Incremental(parser, context, empty)
354
580
  return Effect.flatMap(
355
- Stream.runForEach(self, (chunk) => Effect.sync(() => incremental.push(fromSource<I>(chunk)))),
581
+ Stream.runForEach(self, (source) =>
582
+ Effect.suspend(() => {
583
+ const chunk = fromSource<I>(source)
584
+ // Checked before the chunk is copied in: the whole input is held, so all of it counts
585
+ const size = incremental.size + chunk.length
586
+ if (size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
587
+ incremental.push(chunk)
588
+ return Effect.void
589
+ })
590
+ ),
356
591
  () => Effect.suspend(() => incremental.finish())
357
592
  )
358
593
  })
359
594
  )
595
+ }
360
596
  )
361
597
 
362
598
  /**
@@ -368,11 +604,17 @@ export const parseAll: {
368
604
  */
369
605
  export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ readonly cause: unknown }> {}
370
606
 
607
+ /** Whether a web-stream runner was called with its stream first: the other form starts with the parser. */
608
+ const isDataFirst = (args: IArguments): boolean => !isParser(args[0])
609
+
371
610
  /**
372
611
  * Parses a web `ReadableStream` of bytes, such as the body of a `fetch`
373
612
  * response, incrementally (see {@link parseAll}): the bytes are parsed as
374
613
  * they arrive, without waiting for the end of the stream or decoding them
375
- * first. Use a byte parser, e.g. one built from `Text.utf8`.
614
+ * first. Use a byte parser, e.g. one built from `Text.utf8`. The stream is
615
+ * limited to `maxBufferSize` bytes (64 MiB by default, see
616
+ * {@link BufferOptions}), past which it fails with a
617
+ * {@link BufferLimitError}.
376
618
  *
377
619
  * @example
378
620
  * ```ts
@@ -391,30 +633,52 @@ export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ rea
391
633
  */
392
634
  export const parseReadable: {
393
635
  <A, E, R>(
394
- parser: Parser<A, E, R, Uint8Array>
395
- ): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
636
+ parser: Parser<A, E, R, Uint8Array>,
637
+ options?: BufferOptions
638
+ ): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
396
639
  <A, E, R>(
397
640
  self: ReadableStream<BinaryInput>,
398
- parser: Parser<A, E, R, Uint8Array>
399
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
641
+ parser: Parser<A, E, R, Uint8Array>,
642
+ options?: BufferOptions
643
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
400
644
  } = /*#__PURE__*/ dual(
401
- 2,
645
+ isDataFirst,
402
646
  <A, E, R>(
403
647
  self: ReadableStream<BinaryInput>,
404
- parser: Parser<A, E, R, Uint8Array>
405
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R> =>
648
+ parser: Parser<A, E, R, Uint8Array>,
649
+ options?: BufferOptions
650
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
406
651
  parseAll(
407
652
  Stream.fromReadableStream({ evaluate: () => self, onError: (cause) => new ReadError({ cause }) }),
408
653
  parser,
409
- { empty: new Uint8Array(0) }
654
+ { empty: new Uint8Array(0), maxBufferSize: options?.maxBufferSize }
410
655
  )
411
656
  )
412
657
 
658
+ /**
659
+ * The length a response declares for the body as it will be read, if it
660
+ * declares one: `Content-Length` counts the encoded body, so a response
661
+ * with a `Content-Encoding` (which `fetch` decodes) declares nothing about
662
+ * the bytes parsed.
663
+ */
664
+ const declaredLength = (response: Response): number | undefined => {
665
+ if (response.headers.has("content-encoding")) return undefined
666
+ const header = response.headers.get("content-length")
667
+ if (header === null || !/^[0-9]+$/.test(header)) return undefined
668
+ return Number(header)
669
+ }
670
+
413
671
  /**
414
672
  * Parses the body of a `fetch` `Response` incrementally, as it downloads
415
673
  * (see {@link parseReadable}). A response without a body is parsed as empty
416
674
  * input. The status is not checked.
417
675
  *
676
+ * The body is limited to `maxBufferSize` bytes (64 MiB by default, see
677
+ * {@link BufferOptions}). A response whose `Content-Length` already exceeds
678
+ * the limit fails with a {@link BufferLimitError} before any of the body is
679
+ * read; the limit is enforced on the bytes read as well, since the header
680
+ * may be absent or wrong.
681
+ *
418
682
  * @example
419
683
  * ```ts
420
684
  * import { Effect } from "effect"
@@ -423,7 +687,9 @@ export const parseReadable: {
423
687
  * const lines = Parser.separated(Text.utf8.tillLineEnding, Text.utf8.lineEnding)
424
688
  *
425
689
  * const response = await fetch("https://example.com/log.txt")
426
- * const value = await Effect.runPromise(ParserStream.parseResponse(response, lines))
690
+ * const value = await Effect.runPromise(
691
+ * ParserStream.parseResponse(response, lines, { maxBufferSize: 8 * 1024 * 1024 })
692
+ * )
427
693
  * ```
428
694
  *
429
695
  * @since 0.1.0
@@ -431,17 +697,25 @@ export const parseReadable: {
431
697
  */
432
698
  export const parseResponse: {
433
699
  <A, E, R>(
434
- parser: Parser<A, E, R, Uint8Array>
435
- ): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
700
+ parser: Parser<A, E, R, Uint8Array>,
701
+ options?: BufferOptions
702
+ ): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
436
703
  <A, E, R>(
437
704
  self: Response,
438
- parser: Parser<A, E, R, Uint8Array>
439
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
705
+ parser: Parser<A, E, R, Uint8Array>,
706
+ options?: BufferOptions
707
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
440
708
  } = /*#__PURE__*/ dual(
441
- 2,
709
+ isDataFirst,
442
710
  <A, E, R>(
443
711
  self: Response,
444
- parser: Parser<A, E, R, Uint8Array>
445
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R> =>
446
- parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser)
712
+ parser: Parser<A, E, R, Uint8Array>,
713
+ options?: BufferOptions
714
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
715
+ Effect.suspend(() => {
716
+ const limit = limitOf(options)
717
+ const size = declaredLength(self)
718
+ if (size !== undefined && size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
719
+ return parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser, options)
720
+ })
447
721
  )
package/src/Text.ts CHANGED
@@ -46,6 +46,7 @@ import { codePointPredicate, compile, indexOf, matchLiteral } from "./internal/t
46
46
  import { floatText as markFloatText } from "./internal/numbers.js"
47
47
  import * as internal from "./internal/text.js"
48
48
  import type { Node } from "./internal/node.js"
49
+ import { bounds, checkCount, controlCode } from "./internal/node.js"
49
50
  import type { ErrorKind } from "./ParseError.js"
50
51
  import type { Parser, Range } from "./Parser.js"
51
52
  import * as P from "./Parser.js"
@@ -354,9 +355,8 @@ export interface TextParsers<I extends string | Uint8Array> {
354
355
  }
355
356
 
356
357
  const normalize = (r: Range | undefined): [number, number] => {
357
- if (r === undefined) return [0, Infinity]
358
- if (Predicate.isNumber(r)) return [r, r]
359
- return [r.min ?? 0, r.max ?? Infinity]
358
+ const { max, min } = bounds(r)
359
+ return [min, max]
360
360
  }
361
361
 
362
362
  const isAsciiDigit = /*#__PURE__*/ codePointPredicate("isAsciiDigit", (c: number): boolean => c >= 48 && c <= 57, [
@@ -547,6 +547,7 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
547
547
  }
548
548
 
549
549
  const take = (count: number): Parser<string, never, never, I> => {
550
+ checkCount(count)
550
551
  const expected = [count === 1 ? "1 character" : `${count} characters`]
551
552
  return P.described<string, never, never, I>(
552
553
  (state: State<any>) => {
@@ -719,8 +720,7 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
719
720
  )
720
721
  const floatText = markFloatText(taken(P.alt(caseless("nan"), P.seq(sign, unsignedFloat))).pipe(P.expected("number")))
721
722
 
722
- const matchControl = (control: string) => {
723
- const ctrl = control.codePointAt(0)
723
+ const matchControl = (ctrl: number) => {
724
724
  return (state: State<any>): boolean => {
725
725
  if (internal.next(enc, state) !== ctrl) return false
726
726
  state.pos += internal.width
@@ -779,17 +779,21 @@ export const make = <I extends string | Uint8Array>(encoding: Encoding<I>): Text
779
779
  P.named("hexUint")
780
780
  ),
781
781
  float: floatText.pipe(P.map(toFloat), P.named("float")),
782
- takeEscaped: (normal, control, escapable) =>
783
- takeEscapedImpl(
782
+ takeEscaped: (normal, control, escapable) => {
783
+ const ctrl = controlCode(control, internal.firstCodePoint)
784
+ return takeEscapedImpl(
784
785
  normal,
785
- matchControl(control),
786
+ matchControl(ctrl),
786
787
  escapable,
787
788
  (state, start) => internal.textIn(enc, state, start, state.pos),
788
- internal.firstCodePoint(control),
789
+ ctrl,
789
790
  enc
790
- ),
791
- escaped: (normal, control, escape) =>
792
- escapedImpl(normal, matchControl(control), escape, internal.firstCodePoint(control), enc)
791
+ )
792
+ },
793
+ escaped: (normal, control, escape) => {
794
+ const ctrl = controlCode(control, internal.firstCodePoint)
795
+ return escapedImpl(normal, matchControl(ctrl), escape, ctrl, enc)
796
+ }
793
797
  }
794
798
  }
795
799
 
package/src/Token.ts CHANGED
@@ -38,7 +38,7 @@ import {
38
38
  splitSurrogate
39
39
  } from "./internal/tokens.js"
40
40
  import type { ErrorKind } from "./ParseError.js"
41
- import { bounds } from "./internal/node.js"
41
+ import { bounds, checkCount } from "./internal/node.js"
42
42
  import type { Parser, Range } from "./Parser.js"
43
43
  import { described } from "./Parser.js"
44
44
 
@@ -353,12 +353,22 @@ export const noneOf = <I extends Input = string>(
353
353
  }
354
354
 
355
355
  /**
356
- * Takes exactly `count` tokens (winnow's and nom's `take`).
356
+ * Takes exactly `count` tokens (winnow's and nom's `take`). `count` must be
357
+ * a non-negative integer: anything else throws a `RangeError` here, when
358
+ * the parser is built.
359
+ *
360
+ * @example
361
+ * ```ts
362
+ * import { Parser, Token } from "@thomasfosterau/effect-parse"
363
+ *
364
+ * Parser.parsePeek(Token.take(2), "abc") // Result.succeed(["c", "ab"])
365
+ * ```
357
366
  *
358
367
  * @since 0.1.0
359
368
  * @category slices
360
369
  */
361
370
  export const take = <I extends Input = string>(count: number): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {
371
+ checkCount(count)
362
372
  const expected = [count === 1 ? "1 token" : `${count} tokens`]
363
373
  return described(
364
374
  (state) => {
@@ -374,9 +384,8 @@ export const take = <I extends Input = string>(count: number): Parser<Slice<NoIn
374
384
  }
375
385
 
376
386
  const normalize = (r: Range | undefined): [number, number] => {
377
- if (r === undefined) return [0, Infinity]
378
- if (Predicate.isNumber(r)) return [r, r]
379
- return [r.min ?? 0, r.max ?? Infinity]
387
+ const { max, min } = bounds(r)
388
+ return [min, max]
380
389
  }
381
390
 
382
391
  /** @internal */