@zio.dev/zio-blocks 0.0.51 → 0.0.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adr/2026-07-18-data-migration.md +123 -0
- package/guides/async-getting-started.md +687 -0
- package/guides/compile-time-resource-safety-with-scope.md +6 -0
- package/guides/getting-started-with-mux.md +0 -112
- package/guides/query-dsl-extending.md +1 -1
- package/guides/query-dsl-fluent-builder.md +1 -1
- package/guides/query-dsl-reified-optics.md +1 -1
- package/guides/query-dsl-sql.md +395 -1
- package/guides/sql-checked-interpolation.md +173 -0
- package/guides/sql-transactions.md +286 -0
- package/guides/telemetry-guide.md +131 -70
- package/guides/zio-schema-migration.md +6 -6
- package/index.md +200 -583
- package/package.json +1 -1
- package/reference/async.md +1379 -531
- package/reference/chunk.md +3 -3
- package/reference/codegen/index.md +1 -1
- package/reference/combinators.md +4 -4
- package/reference/config/config-decoder.md +460 -0
- package/reference/config/config-source.md +489 -0
- package/reference/config/errors.md +278 -0
- package/reference/config/flags.md +369 -0
- package/reference/config/formats.md +314 -0
- package/reference/config/index.md +304 -0
- package/reference/config/rollout.md +336 -0
- package/reference/context.md +6 -49
- package/reference/data-migration.md +269 -0
- package/reference/datastar/attributes.md +302 -0
- package/reference/datastar/events.md +234 -0
- package/reference/datastar/index.md +256 -0
- package/reference/datastar/signals.md +230 -0
- package/reference/datastar/sse.md +295 -0
- package/reference/datastar.md +2 -2
- package/reference/docs.md +2 -2
- package/reference/endpoint/bulk-creation.md +96 -0
- package/reference/endpoint/index.md +9 -89
- package/reference/endpoint/path-codec.md +12 -24
- package/reference/endpoint/route-pattern.md +4 -6
- package/reference/endpoint/segment-codec.md +19 -32
- package/reference/html.md +313 -9
- package/reference/htmx/index.md +4 -52
- package/reference/htmx/response-headers.md +240 -0
- package/reference/http-model/headers.md +735 -0
- package/reference/http-model/index.md +3 -1
- package/reference/http-model/model.md +107 -71
- package/reference/http-model/schema-codecs.md +522 -0
- package/reference/http-model/schema.md +6 -3
- package/reference/http-model/server-sent-event.md +341 -0
- package/reference/jwt.md +195 -0
- package/reference/maybe.md +128 -11
- package/reference/media-type.md +2 -2
- package/reference/mux.md +254 -0
- package/reference/mux.mdx +7 -2
- package/reference/openapi.md +3 -3
- package/reference/projection.md +654 -0
- package/reference/resource-management/resource.md +2 -98
- package/reference/resource-management/scope.md +1 -209
- package/reference/resource-management/wire.md +4 -50
- package/reference/ringbuffer/advanced.mdx +1 -1
- package/reference/ringbuffer/index.mdx +3 -3
- package/reference/ringbuffer/mpmc.mdx +38 -4
- package/reference/ringbuffer/mpsc.mdx +36 -4
- package/reference/ringbuffer/spmc.mdx +1 -1
- package/reference/ringbuffer/spsc.mdx +87 -15
- package/reference/schema/allows.md +0 -96
- package/reference/schema/binding.md +2 -2
- package/reference/schema/built-in-codecs/avro.md +2 -2
- package/reference/schema/built-in-codecs/bson.md +50 -20
- package/reference/schema/built-in-codecs/csv.md +2 -2
- package/reference/schema/built-in-codecs/index.md +3 -3
- package/reference/schema/built-in-codecs/json/index.md +2 -2
- package/reference/schema/built-in-codecs/messagepack.md +3 -3
- package/reference/schema/built-in-codecs/thrift.md +2 -2
- package/reference/schema/built-in-codecs/toon.md +3 -3
- package/reference/schema/built-in-codecs/yaml.md +2 -2
- package/reference/schema/codec.md +11 -11
- package/reference/schema/dynamic-optic.md +48 -3
- package/reference/schema/dynamic-schema.md +3 -3
- package/reference/schema/index.md +2 -0
- package/reference/schema/path-interpolator.md +2 -0
- package/reference/schema/reflect-transformer.md +140 -0
- package/reference/schema/schema-evolution/as.md +4 -4
- package/reference/schema/schema-evolution/into.md +2 -2
- package/reference/schema/schema-expr.md +2 -2
- package/reference/schema/schema-search.md +263 -0
- package/reference/schema/schema.md +10 -2
- package/reference/schema/type-class-derivation.md +1 -1
- package/reference/smithy.md +502 -3
- package/reference/sql/db-codec-deriver.md +3 -3
- package/reference/sql/db-codec.md +22 -22
- package/reference/sql/db-con.md +4 -4
- package/reference/sql/db-connection.md +1 -1
- package/reference/sql/db-param.md +1 -1
- package/reference/sql/db-result-reader.md +4 -2
- package/reference/sql/db-tx.md +46 -14
- package/reference/sql/ddl.md +1 -1
- package/reference/sql/frag.md +44 -10
- package/reference/sql/index.md +7 -7
- package/reference/sql/repo.md +15 -15
- package/reference/sql/sql-dialect.md +1 -1
- package/reference/sql/sql-logger.md +1 -1
- package/reference/sql/sql-name-mapper.md +3 -3
- package/reference/sql/table-metadata.md +3 -3
- package/reference/sql/table.md +10 -10
- package/reference/sql/transactor-zio.md +1 -1
- package/reference/sql/transactor.md +21 -11
- package/reference/sql-zio.md +1 -1
- package/reference/streams/core/index.md +32 -0
- package/reference/streams/{pipeline.md → core/pipeline.md} +210 -74
- package/reference/streams/{sink.md → core/sink.md} +331 -353
- package/reference/streams/{stream.md → core/stream.md} +919 -209
- package/reference/streams/execution-and-compatibility/async-execution.md +822 -0
- package/reference/streams/execution-and-compatibility/index.md +35 -0
- package/reference/streams/execution-and-compatibility/platform-differences.md +297 -0
- package/reference/streams/execution-and-compatibility/scala-2-compatibility.md +88 -0
- package/reference/streams/execution-and-compatibility/zero-boxing.md +393 -0
- package/reference/streams/index.md +140 -67
- package/reference/streams/primitives/index.md +30 -0
- package/reference/streams/primitives/reader.md +1992 -0
- package/reference/streams/{writer.md → primitives/writer.md} +254 -98
- package/reference/telemetry/common/any-value.md +90 -0
- package/reference/telemetry/common/attribute-key.md +87 -0
- package/reference/telemetry/common/attributes.md +118 -0
- package/reference/telemetry/common/index.md +39 -0
- package/reference/telemetry/common/instrumentation-scope.md +24 -0
- package/reference/telemetry/common/resource.md +34 -0
- package/reference/telemetry/index.md +311 -0
- package/reference/telemetry/logging/index.md +197 -0
- package/reference/telemetry/logging/log-enrichment.md +72 -0
- package/reference/telemetry/logging/log-formatter.md +100 -0
- package/reference/telemetry/logging/log-record-processor.md +56 -0
- package/reference/telemetry/logging/log-record.md +44 -0
- package/reference/telemetry/logging/log-writer.md +64 -0
- package/reference/telemetry/logging/logger-provider.md +142 -0
- package/reference/telemetry/logging/logger.md +83 -0
- package/reference/telemetry/logging/severity.md +62 -0
- package/reference/telemetry/metrics/index.md +150 -0
- package/reference/telemetry/metrics/instruments.md +183 -0
- package/reference/telemetry/metrics/labeled-instruments.md +74 -0
- package/reference/telemetry/metrics/meter-provider.md +76 -0
- package/reference/telemetry/metrics/meter.md +98 -0
- package/reference/telemetry/metrics/metric-data.md +57 -0
- package/reference/telemetry/otel/custom-exporter.md +216 -0
- package/reference/telemetry/otel/index.md +212 -0
- package/reference/telemetry/tracing/index.md +155 -0
- package/reference/telemetry/tracing/sampler.md +89 -0
- package/reference/telemetry/tracing/span-builder.md +57 -0
- package/reference/telemetry/tracing/span-context.md +39 -0
- package/reference/telemetry/tracing/span-data.md +32 -0
- package/reference/telemetry/tracing/span-kind.md +55 -0
- package/reference/telemetry/tracing/span-processor.md +53 -0
- package/reference/telemetry/tracing/span-status.md +47 -0
- package/reference/telemetry/tracing/span.md +117 -0
- package/reference/telemetry/tracing/tracer-provider.md +91 -0
- package/reference/telemetry/tracing/tracer.md +52 -0
- package/reference/typeid.md +0 -64
- package/sidebars.js +150 -12
- package/undocumented-report.md +528 -270
- package/reference/config.md +0 -158
- package/reference/streams/concurrent-operators.md +0 -106
- package/reference/streams/reader.md +0 -1284
- package/reference/streams/scala-2-compatibility.md +0 -55
- package/reference/streams/zero-boxing.md +0 -275
- package/reference/telemetry.md +0 -693
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
---
|
|
2
2
|
id: pipeline
|
|
3
3
|
title: "Pipeline"
|
|
4
|
+
sidebar_label: "Pipeline"
|
|
5
|
+
description: "The reusable stream transformation: its synchronous and asynchronous factories, andThen composition, and the two routes for applying one."
|
|
6
|
+
keywords:
|
|
7
|
+
- "Stream Transformation"
|
|
8
|
+
- "Pipeline Composition"
|
|
9
|
+
- "Async Pipeline Stages"
|
|
10
|
+
- "Primitive Specialization"
|
|
11
|
+
- "Pipeline"
|
|
4
12
|
---
|
|
5
13
|
|
|
6
14
|
`Pipeline[-In, +Out]` is a **reusable, composable stream transformation** that converts elements of type `In` into elements of type `Out`. Pipelines are first-class values: you can define them once, compose them with `andThen`, and apply them to any [Stream](./stream.md) via `stream.via(pipe)` or to any `Sink` via `pipe.andThenSink(sink)`.
|
|
@@ -96,7 +104,11 @@ These laws guarantee that pipelines compose predictably, regardless of how you p
|
|
|
96
104
|
|
|
97
105
|
## Construction
|
|
98
106
|
|
|
99
|
-
Pipelines are built using factory methods on the `Pipeline` companion object. Each factory creates a pipeline that performs a specific transformation: mapping elements, filtering, collecting, or controlling flow.
|
|
107
|
+
Pipelines are built using factory methods on the `Pipeline` companion object. Each factory creates a pipeline that performs a specific transformation: mapping elements, filtering, collecting, or controlling flow. A factory that applies a user-supplied function to produce a new element type asks for `JvmType.Infer` evidence for the **result** type; the one exception is the `map` overload for a function returning `Nothing`, which takes a `DummyImplicit` and supplies the boxed lane itself. The input representation already belongs to the stream or sink at application time; callers do not supply redundant input evidence. Type-preserving factories, and the structural factories `buffer` and `chunked`, retain or rebuild that representation without call-site evidence.
|
|
108
|
+
|
|
109
|
+
Three of those factories have asynchronous twins, and exactly three: `mapAsync`, `filterAsync`, and `collectAsync`. They are cross-platform, their callbacks return [`Async`](../../async.md), they are evaluated sequentially as the downstream pulls, and they work through both application routes. Other `*Async` operators such as `tapEachAsync` and `distinctByAsync` exist on `Stream` rather than on `Pipeline`; see [Asynchronous Stream Execution](../execution-and-compatibility/async-execution.md#async-operators) for the full stream-level list. `Writer`'s similarly named deferred methods are something else again — adaptation wrappers around its synchronous operations, described in [Writer — Asynchronous Writes](../primitives/writer.md#asynchronous-writes).
|
|
110
|
+
|
|
111
|
+
Factory callbacks are protected as user callbacks: synchronous throws and failed callback effects are defects, not typed stream errors. When a pipeline is applied to a sink, both drain paths close the derived reader. Cancellation closes upstream and downstream state, and a cleanup failure is attached to an existing failure rather than hiding it.
|
|
100
112
|
|
|
101
113
|
### `Pipeline.map[A, B]` — Transform Each Element
|
|
102
114
|
|
|
@@ -104,7 +116,7 @@ Applies a function to every element, producing a new element type. Here is the s
|
|
|
104
116
|
|
|
105
117
|
```scala
|
|
106
118
|
object Pipeline {
|
|
107
|
-
def map[A, B](f: A => B)(implicit
|
|
119
|
+
def map[A, B](f: A => B)(implicit jtB: JvmType.Infer[B]): Pipeline[A, B]
|
|
108
120
|
}
|
|
109
121
|
```
|
|
110
122
|
|
|
@@ -114,21 +126,44 @@ This is the most common pipeline constructor:
|
|
|
114
126
|
import zio.blocks.streams.*
|
|
115
127
|
|
|
116
128
|
val doubler = Pipeline.map[Int, Int](_ * 2)
|
|
117
|
-
// doubler: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$MapPipeline@
|
|
129
|
+
// doubler: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$MapPipeline@2bf42ef7
|
|
118
130
|
val toStr = Pipeline.map[Int, String](_.toString)
|
|
119
|
-
// toStr: Pipeline[Int, String] = zio.blocks.streams.Pipeline$MapPipeline@
|
|
131
|
+
// toStr: Pipeline[Int, String] = zio.blocks.streams.Pipeline$MapPipeline@29332a5b
|
|
120
132
|
|
|
121
133
|
val result = Stream(1, 2, 3).via(doubler).runCollect
|
|
122
134
|
// result: Either[Nothing, Chunk[Int]] = Right(IndexedSeq(2, 4, 6))
|
|
123
135
|
```
|
|
124
136
|
|
|
137
|
+
### `Pipeline.mapAsync[A, B]` — Transform Each Element Asynchronously
|
|
138
|
+
|
|
139
|
+
The asynchronous twin of `map`. The callback returns an `Async`, and each one is awaited before the next element is pulled, so at most one invocation is in flight and input order is preserved. Here is the signature:
|
|
140
|
+
|
|
141
|
+
```scala
|
|
142
|
+
object Pipeline {
|
|
143
|
+
def mapAsync[A, B](f: A => Async[B])(implicit jtB: JvmType.Infer[B]): Pipeline[A, B]
|
|
144
|
+
}
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Like `map`, it asks for `JvmType.Infer` evidence for its result type only. A failed or defective `Async` fails the stream as a defect rather than through its typed error channel, and cancellation of a suspended callback is propagated:
|
|
148
|
+
|
|
149
|
+
```scala
|
|
150
|
+
import zio.blocks.async.*
|
|
151
|
+
import zio.blocks.streams.*
|
|
152
|
+
|
|
153
|
+
val lookup = Pipeline.mapAsync[Int, String](id => Async.succeed(s"user-$id"))
|
|
154
|
+
// lookup: Pipeline[Int, String] = zio.blocks.streams.Pipeline$$anon$4@117b2316
|
|
155
|
+
|
|
156
|
+
val result = Stream(1, 2, 3).via(lookup).runCollectAsync
|
|
157
|
+
// result: Async[Either[Nothing, Chunk[String]]] = zio.blocks.async.Async$slowPath$FoldCausePollable@2def46ea
|
|
158
|
+
```
|
|
159
|
+
|
|
125
160
|
### `Pipeline.filter[A]` — Keep Matching Elements
|
|
126
161
|
|
|
127
162
|
Keeps only elements that satisfy a predicate. Here is the signature:
|
|
128
163
|
|
|
129
164
|
```scala
|
|
130
165
|
object Pipeline {
|
|
131
|
-
def filter[A](pred: A => Boolean)
|
|
166
|
+
def filter[A](pred: A => Boolean): Pipeline[A, A]
|
|
132
167
|
}
|
|
133
168
|
```
|
|
134
169
|
|
|
@@ -138,19 +173,42 @@ Note that the output type is the same as the input type — filtering does not c
|
|
|
138
173
|
import zio.blocks.streams.*
|
|
139
174
|
|
|
140
175
|
val positives = Pipeline.filter[Int](_ > 0)
|
|
141
|
-
// positives: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$FilterPipeline@
|
|
176
|
+
// positives: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$FilterPipeline@39c3da53
|
|
142
177
|
|
|
143
178
|
val result = Stream(-2, -1, 0, 1, 2).via(positives).runCollect
|
|
144
179
|
// result: Either[Nothing, Chunk[Int]] = Right(IndexedSeq(1, 2))
|
|
145
180
|
```
|
|
146
181
|
|
|
182
|
+
### `Pipeline.filterAsync[A]` — Keep Matching Elements Asynchronously
|
|
183
|
+
|
|
184
|
+
The asynchronous twin of `filter`. Elements are tested sequentially and in input order, and an element is emitted only when its `Async` yields `true`. Here is the signature:
|
|
185
|
+
|
|
186
|
+
```scala
|
|
187
|
+
object Pipeline {
|
|
188
|
+
def filterAsync[A](f: A => Async[Boolean]): Pipeline[A, A]
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Like `filter`, it is type-preserving and therefore takes no `JvmType.Infer` evidence — it keeps whatever representation the stream or sink supplies. A failed predicate effect is a defect, and cancellation of a suspended predicate is propagated:
|
|
193
|
+
|
|
194
|
+
```scala
|
|
195
|
+
import zio.blocks.async.*
|
|
196
|
+
import zio.blocks.streams.*
|
|
197
|
+
|
|
198
|
+
val allowed = Pipeline.filterAsync[Int](n => Async.succeed(n % 3 == 0))
|
|
199
|
+
// allowed: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$$anon$2@7313125c
|
|
200
|
+
|
|
201
|
+
val result = Stream(1, 2, 3, 4, 5, 6).via(allowed).runCollectAsync
|
|
202
|
+
// result: Async[Either[Nothing, Chunk[Int]]] = zio.blocks.async.Async$slowPath$FoldCausePollable@76cf0fc6
|
|
203
|
+
```
|
|
204
|
+
|
|
147
205
|
### `Pipeline.collect[A, B]` — Partial Function Transformation
|
|
148
206
|
|
|
149
207
|
Applies a partial function: only elements for which the function is defined pass through, and they are transformed to the output type. This combines filtering and mapping in one step. Here is the signature:
|
|
150
208
|
|
|
151
209
|
```scala
|
|
152
210
|
object Pipeline {
|
|
153
|
-
def collect[A, B](pf: PartialFunction[A, B])(implicit
|
|
211
|
+
def collect[A, B](pf: PartialFunction[A, B])(implicit jtB: JvmType.Infer[B]): Pipeline[A, B]
|
|
154
212
|
}
|
|
155
213
|
```
|
|
156
214
|
|
|
@@ -160,12 +218,37 @@ Use `collect` when you need to filter and transform simultaneously:
|
|
|
160
218
|
import zio.blocks.streams.*
|
|
161
219
|
|
|
162
220
|
val extractInts = Pipeline.collect[Any, Int] { case n: Int => n }
|
|
163
|
-
// extractInts: Pipeline[Any, Int] = zio.blocks.streams.Pipeline$CollectPipeline@
|
|
221
|
+
// extractInts: Pipeline[Any, Int] = zio.blocks.streams.Pipeline$CollectPipeline@5fa160de
|
|
164
222
|
|
|
165
223
|
val result = Stream(1, "a", 2, "b", 3).via(extractInts).runCollect
|
|
166
224
|
// result: Either[Nothing, Chunk[Int]] = Right(IndexedSeq(1, 2, 3))
|
|
167
225
|
```
|
|
168
226
|
|
|
227
|
+
### `Pipeline.collectAsync[A, B]` — Asynchronous Filtering Transformation
|
|
228
|
+
|
|
229
|
+
The asynchronous twin of `collect`, with one shape difference worth noticing: where `collect` takes a `PartialFunction[A, B]`, `collectAsync` takes a total function returning `Async[Option[B]]`. A `None` drops the element and a `Some` emits its value. Here is the signature:
|
|
230
|
+
|
|
231
|
+
```scala
|
|
232
|
+
object Pipeline {
|
|
233
|
+
def collectAsync[A, B](f: A => Async[Option[B]])(implicit jtB: JvmType.Infer[B]): Pipeline[A, B]
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Elements are evaluated sequentially and in input order, and like `collect` the factory asks for `JvmType.Infer` evidence for its result type:
|
|
238
|
+
|
|
239
|
+
```scala
|
|
240
|
+
import zio.blocks.async.*
|
|
241
|
+
import zio.blocks.streams.*
|
|
242
|
+
|
|
243
|
+
val parsePort = Pipeline.collectAsync[String, Int] { raw =>
|
|
244
|
+
Async.succeed(raw.toIntOption.filter(p => p > 0 && p <= 65535))
|
|
245
|
+
}
|
|
246
|
+
// parsePort: Pipeline[String, Int] = zio.blocks.streams.Pipeline$$anon$1@53159f19
|
|
247
|
+
|
|
248
|
+
val result = Stream("80", "not-a-port", "8080", "99999").via(parsePort).runCollectAsync
|
|
249
|
+
// result: Async[Either[Nothing, Chunk[Int]]] = zio.blocks.async.Async$slowPath$FoldCausePollable@7376af26
|
|
250
|
+
```
|
|
251
|
+
|
|
169
252
|
### `Pipeline.take[A]` — First N Elements
|
|
170
253
|
|
|
171
254
|
Passes through at most the first `n` elements, then stops. Here is the signature:
|
|
@@ -182,7 +265,7 @@ This naturally short-circuits — upstream stops producing once `n` elements hav
|
|
|
182
265
|
import zio.blocks.streams.*
|
|
183
266
|
|
|
184
267
|
val firstFive = Pipeline.take[Int](5)
|
|
185
|
-
// firstFive: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$TakePipeline@
|
|
268
|
+
// firstFive: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$TakePipeline@6dfe0469
|
|
186
269
|
|
|
187
270
|
val result = Stream.range(0, 1000).via(firstFive).runCollect
|
|
188
271
|
// result: Either[Nothing, Chunk[Int]] = Right(IndexedSeq(0, 1, 2, 3, 4))
|
|
@@ -204,7 +287,7 @@ Use `drop` to skip headers, metadata, or warm-up elements:
|
|
|
204
287
|
import zio.blocks.streams.*
|
|
205
288
|
|
|
206
289
|
val skipHeader = Pipeline.drop[String](1)
|
|
207
|
-
// skipHeader: Pipeline[String, String] = zio.blocks.streams.Pipeline$DropPipeline@
|
|
290
|
+
// skipHeader: Pipeline[String, String] = zio.blocks.streams.Pipeline$DropPipeline@586ae719
|
|
208
291
|
|
|
209
292
|
val result = Stream("header", "row1", "row2").via(skipHeader).runCollect
|
|
210
293
|
// result: Either[Nothing, Chunk[String]] = Right(IndexedSeq("row1", "row2"))
|
|
@@ -216,7 +299,7 @@ The identity pipeline that passes all elements through unchanged. This is the ne
|
|
|
216
299
|
|
|
217
300
|
```scala
|
|
218
301
|
object Pipeline {
|
|
219
|
-
def identity[A]
|
|
302
|
+
def identity[A]: Pipeline[A, A]
|
|
220
303
|
}
|
|
221
304
|
```
|
|
222
305
|
|
|
@@ -226,7 +309,7 @@ You rarely construct `identity` explicitly, but it is important as a base case i
|
|
|
226
309
|
import zio.blocks.streams.*
|
|
227
310
|
|
|
228
311
|
val noOp = Pipeline.identity[Int]
|
|
229
|
-
// noOp: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$
|
|
312
|
+
// noOp: Pipeline[Int, Int] = zio.blocks.streams.Pipeline$$anon$3@43ed8717
|
|
230
313
|
|
|
231
314
|
// These are equivalent:
|
|
232
315
|
// stream.via(noOp) == stream
|
|
@@ -243,7 +326,7 @@ Pipelines compose into larger, more complex transformations using `andThen`. Bec
|
|
|
243
326
|
Composes two pipelines into one, applying `this` first and `that` second. Here is the signature:
|
|
244
327
|
|
|
245
328
|
```scala
|
|
246
|
-
|
|
329
|
+
abstract class Pipeline[-In, +Out] {
|
|
247
330
|
def andThen[C](that: Pipeline[Out, C]): Pipeline[In, C]
|
|
248
331
|
}
|
|
249
332
|
```
|
|
@@ -286,17 +369,19 @@ def buildPipeline(limit: Option[Int], onlyPositive: Boolean): Pipeline[Int, Int]
|
|
|
286
369
|
|
|
287
370
|
Apply a pipeline to a stream using `via` to transform its output elements. This is the most direct way to use a pipeline: define it once and apply it to multiple streams without repeating the transformation logic.
|
|
288
371
|
|
|
372
|
+
Applying an asynchronous pipeline stage to a synchronous stream makes the resulting stream asynchronous. There is nothing to annotate and no type that changes: `Stream[E, A]` carries no marker for which way it will materialize, so `syncStream.via(Pipeline.mapAsync(...))` has exactly the type `syncStream.via(Pipeline.map(...))` would have. What changes is how the description compiles — one asynchronous node makes the whole graph compile to an asynchronous reader, with the synchronous source lifted in place. On the JVM a blocking terminal still accepts the result; on Scala.js use an `*Async` terminal. See [Mixing synchronous and asynchronous stages](../execution-and-compatibility/async-execution.md#mixing-synchronous-and-asynchronous-stages) for how that widening works.
|
|
373
|
+
|
|
289
374
|
### `Stream#via[B]` — Apply a Pipeline to a Stream
|
|
290
375
|
|
|
291
376
|
The primary way to use a pipeline is through `Stream.via`. Here is the signature:
|
|
292
377
|
|
|
293
378
|
```scala
|
|
294
|
-
|
|
379
|
+
abstract class Stream[+E, +A] {
|
|
295
380
|
def via[B](pipe: Pipeline[A, B]): Stream[E, B]
|
|
296
381
|
}
|
|
297
382
|
```
|
|
298
383
|
|
|
299
|
-
Under the hood, `via` calls `pipe.applyToStream(this)`. Each pipeline type delegates to a specific `Stream` node — for example, `Pipeline.map` creates a `Stream.Mapped`, and `Pipeline.filter` creates a `Stream.Filtered`.
|
|
384
|
+
Under the hood, `via` calls `pipe.applyToStream(this)`. Each pipeline type delegates to a specific `Stream` node — for example, `Pipeline.map` creates a `Stream.Mapped`, and `Pipeline.filter` creates a `Stream.Filtered`. Type-preserving stages propagate the incoming stream representation; transforming stages attach the `JvmType.Infer[B]` evidence for their result. Composition preserves this metadata stage by stage rather than rediscovering the original input type at the call site.
|
|
300
385
|
|
|
301
386
|
The key advantage of `via` over inline methods is **reuse**: define the pipeline once and apply it to multiple streams:
|
|
302
387
|
|
|
@@ -307,7 +392,7 @@ import zio.blocks.streams.*
|
|
|
307
392
|
val cleanSensorData: Pipeline[Double, Double] =
|
|
308
393
|
Pipeline.filter[Double](d => !d.isNaN && !d.isInfinite)
|
|
309
394
|
.andThen(Pipeline.filter(d => d >= -100.0 && d <= 100.0))
|
|
310
|
-
// cleanSensorData: Pipeline[Double, Double] = zio.blocks.streams.Pipeline$Composed@
|
|
395
|
+
// cleanSensorData: Pipeline[Double, Double] = zio.blocks.streams.Pipeline$Composed@1cfa8a20
|
|
311
396
|
|
|
312
397
|
// Apply to different sensor streams
|
|
313
398
|
val sensorStream1 = Stream(45.5, 67.2, Double.NaN, 23.1)
|
|
@@ -330,7 +415,7 @@ val sensor2Result = sensorStream2.via(cleanSensorData).runCollect
|
|
|
330
415
|
You can also call `applyToStream` directly. This is equivalent to `via` but reads left-to-right from the pipeline's perspective. Here is the signature:
|
|
331
416
|
|
|
332
417
|
```scala
|
|
333
|
-
|
|
418
|
+
abstract class Pipeline[-In, +Out] {
|
|
334
419
|
def applyToStream[E](stream: Stream[E, In]): Stream[E, Out]
|
|
335
420
|
}
|
|
336
421
|
```
|
|
@@ -346,7 +431,7 @@ Apply a pipeline to a sink using `andThenSink` to pre-process the sink's input e
|
|
|
346
431
|
The dual of `via`: instead of transforming a stream's output, you pre-process a sink's input. Here is the signature:
|
|
347
432
|
|
|
348
433
|
```scala
|
|
349
|
-
|
|
434
|
+
abstract class Pipeline[-In, +Out] {
|
|
350
435
|
def andThenSink[E, Z](sink: Sink[E, Out, Z]): Sink[E, In, Z]
|
|
351
436
|
}
|
|
352
437
|
```
|
|
@@ -358,19 +443,19 @@ import zio.blocks.streams.*
|
|
|
358
443
|
|
|
359
444
|
// A pipeline that normalizes strings
|
|
360
445
|
val normalize = Pipeline.map[String, String](_.trim.toLowerCase)
|
|
361
|
-
// normalize: Pipeline[String, String] = zio.blocks.streams.Pipeline$MapPipeline@
|
|
446
|
+
// normalize: Pipeline[String, String] = zio.blocks.streams.Pipeline$MapPipeline@68b9a3a0
|
|
362
447
|
|
|
363
448
|
// Apply to different sinks
|
|
364
449
|
val collectNormalized = normalize.andThenSink(Sink.collectAll[String])
|
|
365
|
-
// collectNormalized: Sink[Nothing, String, Chunk[String]] = zio.blocks.streams.Sink$Contramapped@
|
|
450
|
+
// collectNormalized: Sink[Nothing, String, Chunk[String]] = zio.blocks.streams.Sink$Contramapped@793ea2aa
|
|
366
451
|
val countNormalized = normalize.andThenSink(Sink.count)
|
|
367
|
-
// countNormalized: Sink[Nothing, String, Long] = zio.blocks.streams.Sink$Contramapped@
|
|
452
|
+
// countNormalized: Sink[Nothing, String, Long] = zio.blocks.streams.Sink$Contramapped@1bae43d7
|
|
368
453
|
|
|
369
454
|
val result = Stream(" Hello ", " WORLD ").run(collectNormalized)
|
|
370
455
|
// result: Either[Nothing, Chunk[String]] = Right(IndexedSeq("hello", "world"))
|
|
371
456
|
```
|
|
372
457
|
|
|
373
|
-
### When to Use `andThenSink`
|
|
458
|
+
### When to Use `andThenSink` Vs `via`
|
|
374
459
|
|
|
375
460
|
Both achieve the same result. Choose based on which side you want to reuse:
|
|
376
461
|
|
|
@@ -386,7 +471,7 @@ The laws guarantee equivalence: `stream.via(pipe).run(sink) == stream.run(pipe.a
|
|
|
386
471
|
`andThenSink` is an alias for `applyToSink`. Here is the signature:
|
|
387
472
|
|
|
388
473
|
```scala
|
|
389
|
-
|
|
474
|
+
abstract class Pipeline[-In, +Out] {
|
|
390
475
|
def applyToSink[E, Z](sink: Sink[E, Out, Z]): Sink[E, In, Z]
|
|
391
476
|
}
|
|
392
477
|
```
|
|
@@ -395,9 +480,11 @@ Prefer `andThenSink` for readability.
|
|
|
395
480
|
|
|
396
481
|
## JVM Primitive Specialization
|
|
397
482
|
|
|
398
|
-
`Pipeline.map`, `Pipeline.
|
|
483
|
+
`Pipeline.map`, `Pipeline.collect`, and their asynchronous counterparts `mapAsync` and `collectAsync` require `JvmType.Infer` for the transformed result type, except for the `map` overload whose function returns `Nothing`, which fixes the boxed lane itself and takes a `DummyImplicit` instead. It is resolved automatically and records which of the nine logical lanes that result belongs to: the eight JVM primitives, or the reference fallback. Those nine logical lanes compact into five interpreter storage lanes — int-like (`Boolean`, `Byte`, `Short`, `Char`, `Int`), `Long`, `Float`, `Double`, and reference — with operator tags selecting the identity-specific reads over them. Nine lanes therefore does not mean nine interpreter arrays; see [Zero-Boxing Streams](../execution-and-compatibility/zero-boxing.md) for how one is chosen. None of these factories request call-site evidence for the input type.
|
|
399
484
|
|
|
400
|
-
`
|
|
485
|
+
Type-preserving factories such as `filter`, `filterAsync`, `identity`, `take`, `drop`, and `buffer` do not require `JvmType.Infer`: they preserve the representation supplied when the pipeline is applied.
|
|
486
|
+
|
|
487
|
+
The rules hold identically through both application routes. On the stream route, result evidence is stored on the transformed stream node. On the sink route, mapping passes the same result evidence to the sink's contramap machinery, and structural pipelines use the generic run-via-sink route. So `stream.via(pipe)`, `pipe.andThenSink(sink)`, and `pipe.applyToSink(sink)` preserve the same specialization information as well as the same semantics.
|
|
401
488
|
|
|
402
489
|
## Integration
|
|
403
490
|
|
|
@@ -426,25 +513,9 @@ cd zio-blocks
|
|
|
426
513
|
|
|
427
514
|
### Basic Usage
|
|
428
515
|
|
|
429
|
-
This example demonstrates
|
|
516
|
+
This example demonstrates six of the Pipeline factory methods: `map`, `filter`, `collect`, `take`, `drop`, and `identity`. Here is the source code:
|
|
430
517
|
|
|
431
518
|
```scala title="streams-examples/src/main/scala/pipeline/PipelineBasicUsageExample.scala"
|
|
432
|
-
/*
|
|
433
|
-
* Copyright 2024-2026 John A. De Goes and the ZIO Contributors
|
|
434
|
-
*
|
|
435
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
436
|
-
* you may not use this file except in compliance with the License.
|
|
437
|
-
* You may obtain a copy of the License at
|
|
438
|
-
*
|
|
439
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
440
|
-
*
|
|
441
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
442
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
443
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
444
|
-
* See the License for the specific language governing permissions and
|
|
445
|
-
* limitations under the License.
|
|
446
|
-
*/
|
|
447
|
-
|
|
448
519
|
package pipeline
|
|
449
520
|
|
|
450
521
|
import zio.blocks.streams.*
|
|
@@ -508,22 +579,6 @@ sbt "streams-examples/runMain pipeline.PipelineBasicUsageExample"
|
|
|
508
579
|
This example shows how to compose pipelines with `andThen`, apply them to multiple streams, and build pipelines conditionally. Here is the source code:
|
|
509
580
|
|
|
510
581
|
```scala title="streams-examples/src/main/scala/pipeline/PipelineCompositionExample.scala"
|
|
511
|
-
/*
|
|
512
|
-
* Copyright 2024-2026 John A. De Goes and the ZIO Contributors
|
|
513
|
-
*
|
|
514
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
515
|
-
* you may not use this file except in compliance with the License.
|
|
516
|
-
* You may obtain a copy of the License at
|
|
517
|
-
*
|
|
518
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
519
|
-
*
|
|
520
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
521
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
522
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
523
|
-
* See the License for the specific language governing permissions and
|
|
524
|
-
* limitations under the License.
|
|
525
|
-
*/
|
|
526
|
-
|
|
527
582
|
package pipeline
|
|
528
583
|
|
|
529
584
|
import zio.blocks.streams.*
|
|
@@ -618,22 +673,6 @@ sbt "streams-examples/runMain pipeline.PipelineCompositionExample"
|
|
|
618
673
|
This example demonstrates applying pipelines to sinks with `andThenSink`, showing the equivalence between `stream.via(pipe).run(sink)` and `stream.run(pipe.andThenSink(sink))`. Here is the source code:
|
|
619
674
|
|
|
620
675
|
```scala title="streams-examples/src/main/scala/pipeline/PipelineSinkIntegrationExample.scala"
|
|
621
|
-
/*
|
|
622
|
-
* Copyright 2024-2026 John A. De Goes and the ZIO Contributors
|
|
623
|
-
*
|
|
624
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
625
|
-
* you may not use this file except in compliance with the License.
|
|
626
|
-
* You may obtain a copy of the License at
|
|
627
|
-
*
|
|
628
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
629
|
-
*
|
|
630
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
631
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
632
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
633
|
-
* See the License for the specific language governing permissions and
|
|
634
|
-
* limitations under the License.
|
|
635
|
-
*/
|
|
636
|
-
|
|
637
676
|
package pipeline
|
|
638
677
|
|
|
639
678
|
import zio.blocks.streams.*
|
|
@@ -716,3 +755,100 @@ Run it with:
|
|
|
716
755
|
```bash
|
|
717
756
|
sbt "streams-examples/runMain pipeline.PipelineSinkIntegrationExample"
|
|
718
757
|
```
|
|
758
|
+
|
|
759
|
+
### Asynchronous Stages Through Both Routes
|
|
760
|
+
|
|
761
|
+
This example builds `mapAsync`, `filterAsync`, and `collectAsync` into one composed pipeline value, then applies that same value through `stream.via(pipe)`, `pipe.andThenSink(sink)`, and `pipe.applyToSink(sink)` — showing that all three produce the same elements. Here is the source code:
|
|
762
|
+
|
|
763
|
+
```scala title="streams-examples/src/main/scala/pipeline/PipelineAsyncExample.scala"
|
|
764
|
+
package pipeline
|
|
765
|
+
|
|
766
|
+
import zio.blocks.async.*
|
|
767
|
+
import zio.blocks.chunk.Chunk
|
|
768
|
+
import zio.blocks.streams.*
|
|
769
|
+
|
|
770
|
+
/**
|
|
771
|
+
* The three asynchronous `Pipeline` factories — `mapAsync`, `filterAsync` and
|
|
772
|
+
* `collectAsync` — built once as values and then applied through both routes:
|
|
773
|
+
* `stream.via(pipe)` and `pipe.andThenSink(sink)` (with `pipe.applyToSink` as
|
|
774
|
+
* the third spelling of the second route).
|
|
775
|
+
*
|
|
776
|
+
* The point of running both is that they agree. A pipeline is a description,
|
|
777
|
+
* not a stage bound to one side of the graph, so the same value produces the
|
|
778
|
+
* same elements whichever end it is attached to.
|
|
779
|
+
*
|
|
780
|
+
* JVM only, because it ends in `.block` to turn an `Async` into a value for
|
|
781
|
+
* `main`.
|
|
782
|
+
*/
|
|
783
|
+
object PipelineAsyncExample {
|
|
784
|
+
|
|
785
|
+
/** Each field of a raw record, as a trimmed reading. */
|
|
786
|
+
private val readings: Stream[Nothing, String] =
|
|
787
|
+
Stream(" 17 ", " 4", "31 ", " 8 ", " ", "150")
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* `mapAsync` — one asynchronous callback per element, awaited before the next
|
|
791
|
+
* element is pulled. It asks for `JvmType.Infer` evidence for `Int`, its
|
|
792
|
+
* result type, and nothing for its input.
|
|
793
|
+
*/
|
|
794
|
+
private val parse: Pipeline[String, Int] =
|
|
795
|
+
Pipeline
|
|
796
|
+
.mapAsync[String, String](raw => Async.succeed(raw.trim))
|
|
797
|
+
.andThen(Pipeline.mapAsync[String, Int](t => Async.succeed(if (t.isEmpty) 0 else t.toInt)))
|
|
798
|
+
|
|
799
|
+
/** `filterAsync` — type-preserving, so it needs no result evidence. */
|
|
800
|
+
private val inRange: Pipeline[Int, Int] =
|
|
801
|
+
Pipeline.filterAsync[Int](n => Async.succeed(n > 0 && n <= 100))
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* `collectAsync` — filter and transform in one callback. It returns
|
|
805
|
+
* `Async[Option[B]]` rather than a `PartialFunction`: a `None` drops the
|
|
806
|
+
* element, a `Some` emits its value.
|
|
807
|
+
*/
|
|
808
|
+
private val tensDigit: Pipeline[Int, Int] =
|
|
809
|
+
Pipeline.collectAsync[Int, Int](n => Async.succeed(if (n >= 10) Some(n / 10) else None))
|
|
810
|
+
|
|
811
|
+
/** One composed value, applied unchanged through both routes below. */
|
|
812
|
+
private val pipe: Pipeline[String, Int] =
|
|
813
|
+
parse.andThen(inRange).andThen(tensDigit)
|
|
814
|
+
|
|
815
|
+
def main(args: Array[String]): Unit = {
|
|
816
|
+
val sink: Sink[Nothing, Int, Chunk[Int]] = Sink.collectAll[Int]
|
|
817
|
+
|
|
818
|
+
// Route 1 — attach the pipeline to the stream, then run a plain sink.
|
|
819
|
+
val viaStream = readings.via(pipe).runAsync(sink).block
|
|
820
|
+
|
|
821
|
+
// Route 2 — attach the pipeline to the sink, then run the plain stream.
|
|
822
|
+
val viaSink = readings.runAsync(pipe.andThenSink(sink)).block
|
|
823
|
+
|
|
824
|
+
// The third spelling: `andThenSink` is an alias for `applyToSink`.
|
|
825
|
+
val viaApplyToSink = readings.runAsync(pipe.applyToSink(sink)).block
|
|
826
|
+
|
|
827
|
+
report("stream.via(pipe)", viaStream)
|
|
828
|
+
report("pipe.andThenSink(sink)", viaSink)
|
|
829
|
+
report("pipe.applyToSink(sink)", viaApplyToSink)
|
|
830
|
+
println(s"routes agree: ${viaStream == viaSink && viaSink == viaApplyToSink}")
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
private def report[E, Z](label: String, result: Either[E, Z]): Unit =
|
|
834
|
+
result match {
|
|
835
|
+
case Right(value) => println(s"$label -> $value")
|
|
836
|
+
case Left(error) => println(s"$label -> typed error: $error")
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
```
|
|
840
|
+
|
|
841
|
+
([source](https://github.com/zio/zio-blocks/blob/main/streams-examples/src/main/scala/pipeline/PipelineAsyncExample.scala))
|
|
842
|
+
|
|
843
|
+
Run it with:
|
|
844
|
+
|
|
845
|
+
```bash
|
|
846
|
+
sbt "streams-examples/runMain pipeline.PipelineAsyncExample"
|
|
847
|
+
```
|
|
848
|
+
|
|
849
|
+
## See Also
|
|
850
|
+
|
|
851
|
+
- [Asynchronous Stream Execution](../execution-and-compatibility/async-execution.md#async-operators) — the stream-level `*Async` operators the three asynchronous factories delegate to, and how a mixed graph compiles
|
|
852
|
+
- [Stream](./stream.md#integration-with-pipeline-and-sink) — the producer side of `via`
|
|
853
|
+
- [Sink](./sink.md) — the consumer a pipeline can be attached to instead
|
|
854
|
+
- [Zero-Boxing Streams](../execution-and-compatibility/zero-boxing.md) — the lanes `JvmType.Infer` selects between, and why transforming factories need the evidence
|