@thomasfosterau/effect-parse 0.4.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/CHANGELOG.md +202 -0
  2. package/README.md +56 -14
  3. package/dist/Ascii.d.ts +11 -4
  4. package/dist/Ascii.d.ts.map +1 -1
  5. package/dist/Ascii.js +12 -4
  6. package/dist/Ascii.js.map +1 -1
  7. package/dist/Binary.d.ts +5 -2
  8. package/dist/Binary.d.ts.map +1 -1
  9. package/dist/Binary.js +83 -37
  10. package/dist/Binary.js.map +1 -1
  11. package/dist/Bits.d.ts.map +1 -1
  12. package/dist/Bits.js +11 -4
  13. package/dist/Bits.js.map +1 -1
  14. package/dist/Compiler.d.ts +74 -2
  15. package/dist/Compiler.d.ts.map +1 -1
  16. package/dist/Compiler.js +78 -1
  17. package/dist/Compiler.js.map +1 -1
  18. package/dist/Expression.d.ts.map +1 -1
  19. package/dist/Expression.js +86 -74
  20. package/dist/Expression.js.map +1 -1
  21. package/dist/Input.d.ts +56 -5
  22. package/dist/Input.d.ts.map +1 -1
  23. package/dist/Input.js +70 -1
  24. package/dist/Input.js.map +1 -1
  25. package/dist/Nom.d.ts +5 -15
  26. package/dist/Nom.d.ts.map +1 -1
  27. package/dist/Nom.js.map +1 -1
  28. package/dist/ParseError.d.ts +69 -11
  29. package/dist/ParseError.d.ts.map +1 -1
  30. package/dist/ParseError.js +129 -28
  31. package/dist/ParseError.js.map +1 -1
  32. package/dist/Parser.d.ts +375 -29
  33. package/dist/Parser.d.ts.map +1 -1
  34. package/dist/Parser.js +394 -37
  35. package/dist/Parser.js.map +1 -1
  36. package/dist/ParserSchema.d.ts.map +1 -1
  37. package/dist/ParserSchema.js +2 -3
  38. package/dist/ParserSchema.js.map +1 -1
  39. package/dist/ParserStream.d.ts +122 -14
  40. package/dist/ParserStream.d.ts.map +1 -1
  41. package/dist/ParserStream.js +293 -62
  42. package/dist/ParserStream.js.map +1 -1
  43. package/dist/Syntax.d.ts +23 -5
  44. package/dist/Syntax.d.ts.map +1 -1
  45. package/dist/Syntax.js +1 -1
  46. package/dist/Syntax.js.map +1 -1
  47. package/dist/Text.d.ts.map +1 -1
  48. package/dist/Text.js +21 -15
  49. package/dist/Text.js.map +1 -1
  50. package/dist/Token.d.ts +65 -11
  51. package/dist/Token.d.ts.map +1 -1
  52. package/dist/Token.js +73 -26
  53. package/dist/Token.js.map +1 -1
  54. package/dist/internal/compiler/closure.d.ts +1 -1
  55. package/dist/internal/compiler/closure.d.ts.map +1 -1
  56. package/dist/internal/compiler/closure.js +263 -113
  57. package/dist/internal/compiler/closure.js.map +1 -1
  58. package/dist/internal/compiler/entries.d.ts +3 -1
  59. package/dist/internal/compiler/entries.d.ts.map +1 -1
  60. package/dist/internal/compiler/entries.js +3 -2
  61. package/dist/internal/compiler/entries.js.map +1 -1
  62. package/dist/internal/compiler/first.d.ts.map +1 -1
  63. package/dist/internal/compiler/first.js +9 -2
  64. package/dist/internal/compiler/first.js.map +1 -1
  65. package/dist/internal/compiler/graph.d.ts +39 -0
  66. package/dist/internal/compiler/graph.d.ts.map +1 -0
  67. package/dist/internal/compiler/graph.js +78 -0
  68. package/dist/internal/compiler/graph.js.map +1 -0
  69. package/dist/internal/compiler/jit.d.ts +5 -0
  70. package/dist/internal/compiler/jit.d.ts.map +1 -1
  71. package/dist/internal/compiler/jit.js +208 -106
  72. package/dist/internal/compiler/jit.js.map +1 -1
  73. package/dist/internal/compiler/registry.d.ts +11 -3
  74. package/dist/internal/compiler/registry.d.ts.map +1 -1
  75. package/dist/internal/compiler/registry.js.map +1 -1
  76. package/dist/internal/compiler/search.d.ts +32 -0
  77. package/dist/internal/compiler/search.d.ts.map +1 -0
  78. package/dist/internal/compiler/search.js +68 -0
  79. package/dist/internal/compiler/search.js.map +1 -0
  80. package/dist/internal/compiler/sets.d.ts.map +1 -1
  81. package/dist/internal/compiler/sets.js +1 -1
  82. package/dist/internal/compiler/sets.js.map +1 -1
  83. package/dist/internal/compiler/wasm.d.ts +8 -0
  84. package/dist/internal/compiler/wasm.d.ts.map +1 -0
  85. package/dist/internal/compiler/wasm.js +2840 -0
  86. package/dist/internal/compiler/wasm.js.map +1 -0
  87. package/dist/internal/compiler/wasmAsm.d.ts +199 -0
  88. package/dist/internal/compiler/wasmAsm.d.ts.map +1 -0
  89. package/dist/internal/compiler/wasmAsm.js +427 -0
  90. package/dist/internal/compiler/wasmAsm.js.map +1 -0
  91. package/dist/internal/entries.d.ts +3 -1
  92. package/dist/internal/entries.d.ts.map +1 -1
  93. package/dist/internal/entries.js.map +1 -1
  94. package/dist/internal/node.d.ts +77 -3
  95. package/dist/internal/node.d.ts.map +1 -1
  96. package/dist/internal/node.js +54 -10
  97. package/dist/internal/node.js.map +1 -1
  98. package/dist/internal/numbers.d.ts.map +1 -1
  99. package/dist/internal/numbers.js.map +1 -1
  100. package/dist/internal/run.d.ts +14 -0
  101. package/dist/internal/run.d.ts.map +1 -1
  102. package/dist/internal/run.js +53 -11
  103. package/dist/internal/run.js.map +1 -1
  104. package/dist/internal/stack.d.ts +14 -0
  105. package/dist/internal/stack.d.ts.map +1 -1
  106. package/dist/internal/stack.js +24 -2
  107. package/dist/internal/stack.js.map +1 -1
  108. package/dist/internal/syntax.d.ts +4 -4
  109. package/dist/internal/syntax.d.ts.map +1 -1
  110. package/dist/internal/syntax.js +18 -10
  111. package/dist/internal/syntax.js.map +1 -1
  112. package/dist/internal/text.d.ts +32 -0
  113. package/dist/internal/text.d.ts.map +1 -1
  114. package/dist/internal/text.js +68 -14
  115. package/dist/internal/text.js.map +1 -1
  116. package/dist/internal/tokens.d.ts +20 -0
  117. package/dist/internal/tokens.d.ts.map +1 -1
  118. package/dist/internal/tokens.js +72 -39
  119. package/dist/internal/tokens.js.map +1 -1
  120. package/dist/internal/tuple.d.ts +6 -0
  121. package/dist/internal/tuple.d.ts.map +1 -0
  122. package/dist/internal/tuple.js +170 -0
  123. package/dist/internal/tuple.js.map +1 -0
  124. package/package.json +18 -10
  125. package/src/Ascii.ts +47 -12
  126. package/src/Binary.ts +174 -73
  127. package/src/Bits.ts +16 -6
  128. package/src/Compiler.ts +101 -10
  129. package/src/Expression.ts +94 -81
  130. package/src/Input.ts +99 -7
  131. package/src/Nom.ts +15 -5
  132. package/src/ParseError.ts +209 -52
  133. package/src/Parser.ts +684 -143
  134. package/src/ParserSchema.ts +1 -2
  135. package/src/ParserStream.ts +361 -68
  136. package/src/Syntax.ts +60 -36
  137. package/src/Text.ts +32 -27
  138. package/src/Token.ts +152 -34
  139. package/src/internal/compiler/closure.ts +371 -153
  140. package/src/internal/compiler/entries.ts +4 -2
  141. package/src/internal/compiler/first.ts +16 -0
  142. package/src/internal/compiler/graph.ts +96 -0
  143. package/src/internal/compiler/jit.ts +319 -169
  144. package/src/internal/compiler/registry.ts +10 -2
  145. package/src/internal/compiler/search.ts +68 -0
  146. package/src/internal/compiler/sets.ts +7 -3
  147. package/src/internal/compiler/wasm.ts +3276 -0
  148. package/src/internal/compiler/wasmAsm.ts +484 -0
  149. package/src/internal/entries.ts +8 -2
  150. package/src/internal/node.ts +103 -7
  151. package/src/internal/numbers.ts +3 -1
  152. package/src/internal/run.ts +73 -12
  153. package/src/internal/stack.ts +25 -1
  154. package/src/internal/syntax.ts +39 -18
  155. package/src/internal/text.ts +97 -9
  156. package/src/internal/tokens.ts +106 -14
  157. package/src/internal/tuple.ts +141 -0
@@ -120,8 +120,7 @@ const decodeWith = <T extends Schema.Top>(
120
120
  if (encode === undefined) {
121
121
  return Effect.fail(new SchemaIssue.Forbidden({ message: "Encoding is not supported" }, value, parseOptions))
122
122
  }
123
- const result = encode(value)
124
- return Result.isSuccess(result) ? Effect.succeed(result.success) : Effect.fail(result.failure)
123
+ return Effect.fromResult(encode(value))
125
124
  }
126
125
  })
127
126
  )
@@ -7,6 +7,12 @@
7
7
  * retried once more data arrives. When the stream ends, the leftover input
8
8
  * is parsed in complete mode.
9
9
  *
10
+ * Every runner bounds what it buffers: `maxBufferSize` (see
11
+ * {@link BufferOptions}) defaults to 64 MiB ({@link defaultMaxBufferSize}),
12
+ * and a stream that exceeds it fails with a {@link BufferLimitError}, so a
13
+ * socket or an HTTP body can be parsed without wrapping the stream first.
14
+ * Pass `Infinity` for input that is trusted to be large.
15
+ *
10
16
  * @example
11
17
  * ```ts
12
18
  * import { Effect, Stream } from "effect"
@@ -38,6 +44,7 @@ import * as Run from "./internal/run.js"
38
44
  import * as Stack from "./internal/stack.js"
39
45
  import { isEmptyUnits } from "./internal/text.js"
40
46
  import type { Parser } from "./Parser.js"
47
+ import { isParser } from "./Parser.js"
41
48
 
42
49
  /**
43
50
  * Input kinds that can be streamed in chunks.
@@ -47,22 +54,202 @@ import type { Parser } from "./Parser.js"
47
54
  */
48
55
  export type Chunkable = string | Uint8Array | ReadonlyArray<unknown>
49
56
 
50
- const concat = <I extends Chunkable>(first: I, b: I): I => {
51
- // Widened so the checks below narrow it (a type parameter does not narrow).
52
- const a: Chunkable = first
53
- if (Predicate.isString(a)) return (a + (b as string)) as I
54
- if (a instanceof Uint8Array) {
55
- if (a.byteLength === 0) return b
56
- const out = new Uint8Array(a.length + (b as Uint8Array).length)
57
- out.set(a)
58
- out.set(b as Uint8Array, a.length)
59
- return out as I
57
+ /**
58
+ * The default `maxBufferSize` of {@link parseMany}, {@link parseAll},
59
+ * {@link parseReadable} and {@link parseResponse}: 64 MiB, counted in tokens
60
+ * of the input (UTF-16 code units of a string, bytes of a `Uint8Array`,
61
+ * elements of an array).
62
+ *
63
+ * @since 0.6.0
64
+ * @category constants
65
+ */
66
+ export const defaultMaxBufferSize: number = 64 * 1024 * 1024
67
+
68
+ /**
69
+ * A stream runner would have buffered more than its `maxBufferSize`: `size`
70
+ * tokens against a limit of `limit`. It is a different failure from a
71
+ * `ParseError`: the input is too large, not malformed (an HTTP server
72
+ * answers 413 for the one and 400 for the other), and nothing of the input
73
+ * is attached to it.
74
+ *
75
+ * @example
76
+ * ```ts
77
+ * import { Effect, Stream } from "effect"
78
+ * import { Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
79
+ *
80
+ * const line = Parser.terminated(Token.takeTill("\n"), Token.literal("\n"))
81
+ *
82
+ * Stream.make("no newline, ", "ever").pipe(
83
+ * ParserStream.parseMany(line, { maxBufferSize: 8 }),
84
+ * Stream.runCollect,
85
+ * Effect.catchTag("BufferLimitError", (e) => Effect.succeed(`too large: ${e.size} > ${e.limit}`))
86
+ * )
87
+ * ```
88
+ *
89
+ * @since 0.6.0
90
+ * @category errors
91
+ */
92
+ export class BufferLimitError
93
+ extends /*#__PURE__*/ Data.TaggedError("BufferLimitError")<{
94
+ /** The `maxBufferSize` in force, in tokens. */
95
+ readonly limit: number
96
+ /** The tokens the runner held, or was about to hold (for `parseResponse`, the declared `Content-Length`). */
97
+ readonly size: number
98
+ }>
99
+ {
100
+ override get message(): string {
101
+ return `the input is larger than maxBufferSize: ${this.size} tokens, limit ${this.limit}`
60
102
  }
61
- return [...a, ...(b as ReadonlyArray<unknown>)] as unknown as I
62
103
  }
63
104
 
64
- const sliceFrom = <I extends Chunkable>(input: I, offset: number): I =>
65
- (Predicate.isString(input) ? input.slice(offset) : input.slice(offset)) as I
105
+ /**
106
+ * The limit on buffered input that the stream runners share.
107
+ *
108
+ * @since 0.6.0
109
+ * @category models
110
+ */
111
+ export interface BufferOptions {
112
+ /**
113
+ * The most tokens a runner buffers (UTF-16 code units of a string, bytes
114
+ * of a `Uint8Array`, elements of an array) before it fails with a
115
+ * {@link BufferLimitError}. Defaults to {@link defaultMaxBufferSize}
116
+ * (64 MiB); `Infinity` turns the limit off.
117
+ *
118
+ * {@link parseMany} counts the input it has not consumed yet, which is one
119
+ * unfinished value: complete values in a chunk are parsed and emitted
120
+ * first, whatever the chunk's size. {@link parseAll},
121
+ * {@link parseReadable} and {@link parseResponse} hold the whole input, so
122
+ * they count all of it.
123
+ */
124
+ readonly maxBufferSize?: number | undefined
125
+ }
126
+
127
+ /** The limit `options` asks for: a positive number of tokens, or `Infinity`. */
128
+ const limitOf = (options: BufferOptions | undefined): number => {
129
+ const limit = options?.maxBufferSize ?? defaultMaxBufferSize
130
+ if (!(limit > 0)) throw new RangeError("maxBufferSize must be a positive number of tokens, or Infinity")
131
+ return limit
132
+ }
133
+
134
+ /**
135
+ * Above this many unconsumed tokens, {@link parseMany} stops retrying its
136
+ * unfinished value on every chunk (see {@link Pending.due}).
137
+ */
138
+ const BackoffThreshold = 64 * 1024
139
+
140
+ /**
141
+ * The input `parseMany` has not consumed yet: what is left of the chunks
142
+ * after the values parsed from them, which is one unfinished value.
143
+ *
144
+ * Appending is amortized constant time per token: text is concatenated (a
145
+ * rope until it is parsed), bytes go into a buffer that doubles, and
146
+ * elements are pushed. Bytes that a parse has seen are never overwritten,
147
+ * since the values parsed from them may be views of the buffer
148
+ * (`Token.take` returns a `subarray`): the buffer is only ever appended to
149
+ * past its end, or replaced by a larger one.
150
+ */
151
+ class Pending<I extends Chunkable> {
152
+ private kind: 0 | 1 | 2 | undefined
153
+ private text = ""
154
+ private bytes: Uint8Array = new Uint8Array(0)
155
+ /** Where the unconsumed bytes start and end in `bytes`, and how far `bytes` may be written. */
156
+ private start = 0
157
+ private end = 0
158
+ private capacity = 0
159
+ private items: Array<unknown> = []
160
+ /** The unconsumed size at the last attempt that ran out of input, or 0 when none is outstanding. */
161
+ private attempted = 0
162
+
163
+ /** The number of unconsumed tokens. */
164
+ get size(): number {
165
+ return this.kind === 1 ? this.end - this.start : this.kind === 2 ? this.items.length : this.text.length
166
+ }
167
+
168
+ get isEmpty(): boolean {
169
+ return this.kind === undefined
170
+ }
171
+
172
+ append(chunk: I): void {
173
+ if (Predicate.isString(chunk)) {
174
+ this.kind ??= 0
175
+ this.text += chunk
176
+ } else if (chunk instanceof Uint8Array) {
177
+ this.kind ??= 1
178
+ this.appendBytes(chunk)
179
+ } else {
180
+ this.kind ??= 2
181
+ for (let i = 0; i < chunk.length; i++) this.items.push(chunk[i])
182
+ }
183
+ }
184
+
185
+ private appendBytes(chunk: Uint8Array): void {
186
+ const held = this.end - this.start
187
+ if (held === 0) {
188
+ // Nothing is pending: the chunk is the buffer, uncopied. It is the
189
+ // caller's, so it is never written to (`capacity` is its end)
190
+ this.bytes = chunk
191
+ this.start = 0
192
+ this.end = this.capacity = chunk.length
193
+ return
194
+ }
195
+ const needed = held + chunk.length
196
+ if (this.end + chunk.length > this.capacity) {
197
+ // Twice what is needed, so appending costs amortized constant time per byte
198
+ const grown = new Uint8Array(Math.max(needed * 2, 4096))
199
+ grown.set(this.bytes.subarray(this.start, this.end))
200
+ this.bytes = grown
201
+ this.start = 0
202
+ this.end = held
203
+ this.capacity = grown.length
204
+ }
205
+ this.bytes.set(chunk, this.end)
206
+ this.end += chunk.length
207
+ }
208
+
209
+ /** The unconsumed input. */
210
+ input(): I {
211
+ if (this.kind === 1) {
212
+ // SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
213
+ return this.bytes.subarray(this.start, this.end) as I
214
+ }
215
+ if (this.kind === 2) {
216
+ const items: Chunkable = this.items
217
+ // SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
218
+ return items as I
219
+ }
220
+ // SAFETY: `kind` 0 is set by the first chunk being a string, so `I` is `string`
221
+ return this.text as I
222
+ }
223
+
224
+ /** Drops the first `count` tokens, which values were parsed from. */
225
+ consume(count: number): void {
226
+ if (count === 0) return
227
+ if (this.kind === 1) this.start += count
228
+ // A new array: the one parsed is the `input` of the state that read it
229
+ else if (this.kind === 2) this.items = this.items.slice(count)
230
+ else this.text = this.text.slice(count)
231
+ }
232
+
233
+ /** Records whether the attempt just made left an unfinished value. */
234
+ attemptedWith(unfinished: boolean): void {
235
+ this.attempted = unfinished ? this.size : 0
236
+ }
237
+
238
+ /**
239
+ * Whether to parse now. A value that is still unfinished is parsed again
240
+ * from its start, so retrying on every chunk costs the square of its
241
+ * size: a megabyte with no delimiter, in 1 KiB chunks, took seconds. Past
242
+ * {@link BackoffThreshold} tokens, the next attempt therefore waits until
243
+ * the buffer has grown by an eighth, which keeps the total work linear
244
+ * (each token is re-read at most about nine times). Smaller values are
245
+ * retried on every chunk, as cheaply as before, so a record is emitted
246
+ * as soon as its last chunk arrives.
247
+ */
248
+ due(): boolean {
249
+ const attempted = this.attempted
250
+ return attempted <= BackoffThreshold || this.size - attempted >= attempted >>> 3
251
+ }
252
+ }
66
253
 
67
254
  const noProgress = (input: Input, offset: number) =>
68
255
  ParseError.fromFailure<never>(
@@ -72,12 +259,12 @@ const noProgress = (input: Input, offset: number) =>
72
259
  )
73
260
 
74
261
  /**
75
- * Options for {@link parseMany}.
262
+ * Options for {@link parseMany} and {@link parseAll}.
76
263
  *
77
264
  * @since 0.1.0
78
265
  * @category models
79
266
  */
80
- export interface ParseManyOptions<I> {
267
+ export interface ParseManyOptions<I> extends BufferOptions {
81
268
  /** The empty input, used to initialise the buffer (inferred from the first chunk if omitted). */
82
269
  readonly empty?: Source<I> | undefined
83
270
  }
@@ -92,8 +279,34 @@ export interface ParseManyOptions<I> {
92
279
  *
93
280
  * A parser compiled with the `Compiler` module runs compiled, with the
94
281
  * build that handles partial input (see `Parser.run`); a value that fails
95
- * or needs more input is parsed again interpreted, so errors and
96
- * `Incomplete` are the interpreter's.
282
+ * is parsed again interpreted, so errors are the interpreter's. A value
283
+ * cut short by its chunk is not: compiled code reports `Incomplete` where
284
+ * the interpreter does.
285
+ *
286
+ * **Untrusted streams.** The input not yet consumed (one unfinished value)
287
+ * is limited to `maxBufferSize` tokens, 64 MiB by default
288
+ * ({@link defaultMaxBufferSize}); past it the stream fails with a
289
+ * {@link BufferLimitError}, so a peer that never sends the delimiter cannot
290
+ * grow the buffer without bound. An unfinished value is parsed again from
291
+ * its start when more input arrives. Up to 64 KiB that happens on every
292
+ * chunk; a larger one is retried only once the buffer has grown by an
293
+ * eighth, so the work stays linear in its size. The cost: the last chunk of
294
+ * a value larger than 64 KiB may not complete it in the output until more
295
+ * input arrives or the stream ends, when everything left is parsed.
296
+ *
297
+ * @example
298
+ * ```ts
299
+ * import { Effect, Stream } from "effect"
300
+ * import { Ascii, Parser, ParserStream, Token } from "@thomasfosterau/effect-parse"
301
+ *
302
+ * const line = Parser.terminated(Ascii.decInt, Token.literal("\n"))
303
+ *
304
+ * const program = Stream.make("1\n2", "3\n", "4\n").pipe(
305
+ * ParserStream.parseMany(line, { maxBufferSize: 1024 }),
306
+ * Stream.runCollect
307
+ * )
308
+ * // Effect.runSync(program) => [1, 23, 4]
309
+ * ```
97
310
  *
98
311
  * @since 0.1.0
99
312
  * @category combinators
@@ -102,51 +315,64 @@ export const parseMany: {
102
315
  <A, E, R, I extends Chunkable>(
103
316
  parser: Parser<A, E, R, I>,
104
317
  options?: ParseManyOptions<I>
105
- ): <E0, R0>(self: Stream.Stream<Source<I>, E0, R0>) => Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R>
318
+ ): <E0, R0>(
319
+ self: Stream.Stream<Source<I>, E0, R0>
320
+ ) => Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
106
321
  <I extends Chunkable, E0, R0, A, E, R>(
107
322
  self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
108
323
  parser: Parser<A, E, R, I>,
109
324
  options?: ParseManyOptions<I>
110
- ): Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R>
325
+ ): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
111
326
  } = /*#__PURE__*/ dual(
112
327
  (args) => Stream.isStream(args[0]),
113
328
  <I extends Chunkable, E0, R0, A, E, R>(
114
329
  self: Stream.Stream<Source<I>, E0, R0>,
115
330
  parser: Parser<A, E, R, I>,
116
331
  options?: ParseManyOptions<I>
117
- ): Stream.Stream<A, E0 | ParseError.ParseError<E>, R0 | R> =>
118
- Stream.suspend(() => {
332
+ ): Stream.Stream<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
333
+ const limit = limitOf(options)
334
+ return Stream.suspend(() => {
119
335
  const step = parser.step
120
- let buffer: I | undefined = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
336
+ const pending = new Pending<I>()
337
+ if (options?.empty !== undefined) pending.append(fromSource<I>(options.empty))
121
338
 
122
- const drain = (context: any, partial: boolean): Effect.Effect<Array<A>, ParseError.ParseError<E>> => {
339
+ const drain = (
340
+ context: any,
341
+ partial: boolean
342
+ ): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
123
343
  const out: Array<A> = []
124
- if (buffer === undefined) return Effect.succeed(out)
125
- const input = buffer
344
+ if (pending.isEmpty) return Effect.succeed(out)
345
+ const input = pending.input()
126
346
  // One state for the whole buffer, reset before each value
127
347
  const state = new State(input)
128
348
  let offset = 0
129
349
  while (offset < input.length) {
130
350
  state.reset(offset, { partial, context })
131
- const a = Run.runOn(parser, step, state)
351
+ // A compiled `Incomplete` is the interpreter's: only an error needs the interpreted run
352
+ const a = Run.runOn(parser, step, state, false)
132
353
  if (a === Fail) {
133
354
  if (partial && state.mode === "Incomplete") break
355
+ state.reset(offset, { partial, context })
356
+ Run.runOn(parser, step, state)
134
357
  return Effect.fail(ParseError.fromFailure<E>(input, state.reportedFailure(), state.mode, state.encoding))
135
358
  }
136
359
  if (state.pos === offset) return Effect.fail(noProgress(input, offset))
137
360
  out.push(a)
138
361
  offset = state.pos
139
362
  }
140
- buffer = sliceFrom(input, offset)
141
- return Effect.succeed(out)
363
+ pending.consume(offset)
364
+ pending.attemptedWith(offset < input.length)
365
+ // Only what no value consumed counts: an unfinished value larger than the limit
366
+ const size = input.length - offset
367
+ return size > limit ? Effect.fail(new BufferLimitError({ limit, size })) : Effect.succeed(out)
142
368
  }
143
369
 
144
370
  const feed = (source: Source<I>) =>
145
371
  Effect.flatMap(Effect.context<R>(), (context) =>
146
- Effect.suspend(() => {
147
- const chunk = fromSource<I>(source)
148
- buffer = buffer === undefined ? chunk : concat(buffer, chunk)
149
- return drain(context, true)
372
+ Effect.suspend((): Effect.Effect<Array<A>, ParseError.ParseError<E> | BufferLimitError> => {
373
+ pending.append(fromSource<I>(source))
374
+ // Over the limit, parse whatever the back-off says, so that complete values are not counted
375
+ return pending.due() || pending.size > limit ? drain(context, true) : Effect.succeed([])
150
376
  })
151
377
  )
152
378
 
@@ -158,6 +384,7 @@ export const parseMany: {
158
384
  Stream.concat(Stream.unwrap(Effect.map(finish, Stream.fromIterable)))
159
385
  )
160
386
  })
387
+ }
161
388
  )
162
389
 
163
390
  /**
@@ -213,15 +440,24 @@ class Incremental<A, E, I extends Chunkable> {
213
440
  }
214
441
 
215
442
  private input(): I {
216
- switch (this.kind) {
217
- case 1:
218
- return this.bytes.subarray(0, this.length) as I
219
- case 2:
220
- // Items are only appended between attempts, after each state has read its `end`
221
- return this.items as unknown as I
222
- default:
223
- return this.text as I
443
+ if (this.kind === 1) {
444
+ // SAFETY: `kind` 1 is set by the first chunk being a `Uint8Array`, so `I` is `Uint8Array`
445
+ return this.bytes.subarray(0, this.length) as I
446
+ }
447
+ if (this.kind === 2) {
448
+ // Items are only appended between attempts, after each state has read its `end`
449
+ const items: Chunkable = this.items
450
+ // SAFETY: `kind` 2 is set by the first chunk being an array, so `I` is an array of the items' type
451
+ return items as I
224
452
  }
453
+ // SAFETY: `kind` 0 means the first chunk was a string, so `I` is `string`; while `kind` is unset (no
454
+ // chunk arrived) `I` is unknown and the empty string stands for the empty input
455
+ return this.text as I
456
+ }
457
+
458
+ /** The tokens held: the whole input so far. */
459
+ get size(): number {
460
+ return this.length
225
461
  }
226
462
 
227
463
  push(chunk: I): void {
@@ -247,8 +483,9 @@ class Incremental<A, E, I extends Chunkable> {
247
483
  * The step attempts run: the parser's compiled code if it was compiled
248
484
  * (the resumable build, unless `resumable` is false), else the interpreter.
249
485
  */
250
- private step(state: State<I>, resumable = true): unknown {
251
- return Stack.run(Registry.stepFor(this.parser, state, resumable) ?? this.parser.step, state)
486
+ private step(state: State<I>, resumable = true): A | Fail {
487
+ // SAFETY: a compiled step (typed `any` by the compilers) returns what the parser's own step returns
488
+ return Stack.run(Registry.stepFor(this.parser, state, resumable) ?? this.parser.step, state) as A | Fail
252
489
  }
253
490
 
254
491
  private attempt(): void {
@@ -257,7 +494,7 @@ class Incremental<A, E, I extends Chunkable> {
257
494
  const a = this.step(state)
258
495
  this.attempted = this.length
259
496
  if (a !== Fail) {
260
- this.settled = { value: a as A, pos: state.pos }
497
+ this.settled = { value: a, pos: state.pos }
261
498
  } else if (state.mode === "Incomplete") {
262
499
  this.restart = state.failure.offset
263
500
  } else {
@@ -276,7 +513,7 @@ class Incremental<A, E, I extends Chunkable> {
276
513
  const resumable = this.attempted > 0
277
514
  if (resumable) state.resume = this.resume
278
515
  const a = this.step(state, resumable)
279
- if (a !== Fail && state.pos === state.end) return Effect.succeed(a as A)
516
+ if (a !== Fail && state.pos === state.end) return Effect.succeed(a)
280
517
  }
281
518
  // The error comes from an ordinary parse of the whole input, so it is the
282
519
  // same as if the input had been parsed in one go
@@ -308,6 +545,11 @@ class Incremental<A, E, I extends Chunkable> {
308
545
  * callbacks should be pure: the elements parsed before a checkpoint are
309
546
  * not parsed again, and on failure everything is parsed again.
310
547
  *
548
+ * The whole input is held until the stream ends, so it is limited to
549
+ * `maxBufferSize` tokens (64 MiB by default, see {@link BufferOptions}):
550
+ * a stream that would exceed it fails with a {@link BufferLimitError}
551
+ * before the chunk that crosses the limit is buffered.
552
+ *
311
553
  * @since 0.1.0
312
554
  * @category combinators
313
555
  */
@@ -315,29 +557,42 @@ export const parseAll: {
315
557
  <A, E, R, I extends Chunkable>(
316
558
  parser: Parser<A, E, R, I>,
317
559
  options?: ParseManyOptions<I>
318
- ): <E0, R0>(self: Stream.Stream<Source<I>, E0, R0>) => Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R>
560
+ ): <E0, R0>(
561
+ self: Stream.Stream<Source<I>, E0, R0>
562
+ ) => Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
319
563
  <I extends Chunkable, E0, R0, A, E, R>(
320
564
  self: Stream.Stream<Source<NoInfer<I>>, E0, R0>,
321
565
  parser: Parser<A, E, R, I>,
322
566
  options?: ParseManyOptions<I>
323
- ): Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R>
567
+ ): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R>
324
568
  } = /*#__PURE__*/ dual(
325
569
  (args) => Stream.isStream(args[0]),
326
570
  <I extends Chunkable, E0, R0, A, E, R>(
327
571
  self: Stream.Stream<Source<I>, E0, R0>,
328
572
  parser: Parser<A, E, R, I>,
329
573
  options?: ParseManyOptions<I>
330
- ): Effect.Effect<A, E0 | ParseError.ParseError<E>, R0 | R> =>
331
- Effect.flatMap(Effect.context<R>(), (context) =>
574
+ ): Effect.Effect<A, E0 | ParseError.ParseError<E> | BufferLimitError, R0 | R> => {
575
+ const limit = limitOf(options)
576
+ return Effect.flatMap(Effect.context<R>(), (context) =>
332
577
  Effect.suspend(() => {
333
578
  const empty = options?.empty === undefined ? undefined : fromSource<I>(options.empty)
334
- const incremental = new Incremental(parser, context as Context.Context<never>, empty)
579
+ const incremental = new Incremental(parser, context, empty)
335
580
  return Effect.flatMap(
336
- Stream.runForEach(self, (chunk) => Effect.sync(() => incremental.push(fromSource<I>(chunk)))),
581
+ Stream.runForEach(self, (source) =>
582
+ Effect.suspend(() => {
583
+ const chunk = fromSource<I>(source)
584
+ // Checked before the chunk is copied in: the whole input is held, so all of it counts
585
+ const size = incremental.size + chunk.length
586
+ if (size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
587
+ incremental.push(chunk)
588
+ return Effect.void
589
+ })
590
+ ),
337
591
  () => Effect.suspend(() => incremental.finish())
338
592
  )
339
593
  })
340
594
  )
595
+ }
341
596
  )
342
597
 
343
598
  /**
@@ -349,11 +604,17 @@ export const parseAll: {
349
604
  */
350
605
  export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ readonly cause: unknown }> {}
351
606
 
607
+ /** Whether a web-stream runner was called with its stream first: the other form starts with the parser. */
608
+ const isDataFirst = (args: IArguments): boolean => !isParser(args[0])
609
+
352
610
  /**
353
611
  * Parses a web `ReadableStream` of bytes, such as the body of a `fetch`
354
612
  * response, incrementally (see {@link parseAll}): the bytes are parsed as
355
613
  * they arrive, without waiting for the end of the stream or decoding them
356
- * first. Use a byte parser, e.g. one built from `Text.utf8`.
614
+ * first. Use a byte parser, e.g. one built from `Text.utf8`. The stream is
615
+ * limited to `maxBufferSize` bytes (64 MiB by default, see
616
+ * {@link BufferOptions}), past which it fails with a
617
+ * {@link BufferLimitError}.
357
618
  *
358
619
  * @example
359
620
  * ```ts
@@ -372,30 +633,52 @@ export class ReadError extends /*#__PURE__*/ Data.TaggedError("ReadError")<{ rea
372
633
  */
373
634
  export const parseReadable: {
374
635
  <A, E, R>(
375
- parser: Parser<A, E, R, Uint8Array>
376
- ): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
636
+ parser: Parser<A, E, R, Uint8Array>,
637
+ options?: BufferOptions
638
+ ): (self: ReadableStream<BinaryInput>) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
377
639
  <A, E, R>(
378
640
  self: ReadableStream<BinaryInput>,
379
- parser: Parser<A, E, R, Uint8Array>
380
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
641
+ parser: Parser<A, E, R, Uint8Array>,
642
+ options?: BufferOptions
643
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
381
644
  } = /*#__PURE__*/ dual(
382
- 2,
645
+ isDataFirst,
383
646
  <A, E, R>(
384
647
  self: ReadableStream<BinaryInput>,
385
- parser: Parser<A, E, R, Uint8Array>
386
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R> =>
648
+ parser: Parser<A, E, R, Uint8Array>,
649
+ options?: BufferOptions
650
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
387
651
  parseAll(
388
652
  Stream.fromReadableStream({ evaluate: () => self, onError: (cause) => new ReadError({ cause }) }),
389
653
  parser,
390
- { empty: new Uint8Array(0) }
654
+ { empty: new Uint8Array(0), maxBufferSize: options?.maxBufferSize }
391
655
  )
392
656
  )
393
657
 
658
+ /**
659
+ * The length a response declares for the body as it will be read, if it
660
+ * declares one: `Content-Length` counts the encoded body, so a response
661
+ * with a `Content-Encoding` (which `fetch` decodes) declares nothing about
662
+ * the bytes parsed.
663
+ */
664
+ const declaredLength = (response: Response): number | undefined => {
665
+ if (response.headers.has("content-encoding")) return undefined
666
+ const header = response.headers.get("content-length")
667
+ if (header === null || !/^[0-9]+$/.test(header)) return undefined
668
+ return Number(header)
669
+ }
670
+
394
671
  /**
395
672
  * Parses the body of a `fetch` `Response` incrementally, as it downloads
396
673
  * (see {@link parseReadable}). A response without a body is parsed as empty
397
674
  * input. The status is not checked.
398
675
  *
676
+ * The body is limited to `maxBufferSize` bytes (64 MiB by default, see
677
+ * {@link BufferOptions}). A response whose `Content-Length` already exceeds
678
+ * the limit fails with a {@link BufferLimitError} before any of the body is
679
+ * read; the limit is enforced on the bytes read as well, since the header
680
+ * may be absent or wrong.
681
+ *
399
682
  * @example
400
683
  * ```ts
401
684
  * import { Effect } from "effect"
@@ -404,7 +687,9 @@ export const parseReadable: {
404
687
  * const lines = Parser.separated(Text.utf8.tillLineEnding, Text.utf8.lineEnding)
405
688
  *
406
689
  * const response = await fetch("https://example.com/log.txt")
407
- * const value = await Effect.runPromise(ParserStream.parseResponse(response, lines))
690
+ * const value = await Effect.runPromise(
691
+ * ParserStream.parseResponse(response, lines, { maxBufferSize: 8 * 1024 * 1024 })
692
+ * )
408
693
  * ```
409
694
  *
410
695
  * @since 0.1.0
@@ -412,17 +697,25 @@ export const parseReadable: {
412
697
  */
413
698
  export const parseResponse: {
414
699
  <A, E, R>(
415
- parser: Parser<A, E, R, Uint8Array>
416
- ): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
700
+ parser: Parser<A, E, R, Uint8Array>,
701
+ options?: BufferOptions
702
+ ): (self: Response) => Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
417
703
  <A, E, R>(
418
704
  self: Response,
419
- parser: Parser<A, E, R, Uint8Array>
420
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R>
705
+ parser: Parser<A, E, R, Uint8Array>,
706
+ options?: BufferOptions
707
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R>
421
708
  } = /*#__PURE__*/ dual(
422
- 2,
709
+ isDataFirst,
423
710
  <A, E, R>(
424
711
  self: Response,
425
- parser: Parser<A, E, R, Uint8Array>
426
- ): Effect.Effect<A, ParseError.ParseError<E> | ReadError, R> =>
427
- parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser)
712
+ parser: Parser<A, E, R, Uint8Array>,
713
+ options?: BufferOptions
714
+ ): Effect.Effect<A, ParseError.ParseError<E> | ReadError | BufferLimitError, R> =>
715
+ Effect.suspend(() => {
716
+ const limit = limitOf(options)
717
+ const size = declaredLength(self)
718
+ if (size !== undefined && size > limit) return Effect.fail(new BufferLimitError({ limit, size }))
719
+ return parseReadable(self.body ?? new ReadableStream<Uint8Array>({ start: (c) => c.close() }), parser, options)
720
+ })
428
721
  )