fiber_stream 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2401496eff99cd4792deda8fa927688fc3ba0e97e8a35637db4395895ab04cd9
-  data.tar.gz: 9f9674e2bc7dc9ce49b899727a02b9158ce6c9629c744ec6d626b16923e14160
+  metadata.gz: 1cc93666d0610e659313a12dc756fca935579e62711972a9ea65d9ad818f6020
+  data.tar.gz: 97f315765ba573a5c047752fd083006db89404eff06e92bda5afbfa0f1933ed1
 SHA512:
-  metadata.gz: e7237c7c15b66105b09cdccf1e52980f3c030c74acbd2dc7cc3b5cfb720a716836925b5801bdc59351131dcd7ecab72ebdedb6b50593f5258a1cff2d0ebb39a9
-  data.tar.gz: d9bc7c85992c344bef4a36b0f234840c154c06274c450ef0696246d3042a017f8764c22ab1a2df43e3059420c52af7827852e63b4a5e91d668f4c9bb6a191d57
+  metadata.gz: 5e635531f9e34510ef0eab76254c33e70f36124be779f48d2f9692c351a2f26ed36c7c14fa0d1d596c5645b511bb4fa1abe2bb773fc434c1f4da39a6e19dbefc
+  data.tar.gz: 67eb50ae0a1d727c65fd172be572f7bf08ba82b3ee0cf4d9094c7e9ad579aac327155cc772bcd4e94b778cef53f63decf64a08e12e7ea5ef6949a6d813336b04

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 0.4.0 - 2026-06-09
+
+### Added
+
+- `Flow.parallel_unordered_map(concurrency:)` and
+  `Source#parallel_unordered_map(concurrency:)` for scheduler-backed mapping
+  that emits results in completion order instead of preserving input order.
+- `Source.ractor_producer` for FiberStream-owned single producer Ractors with
+  one-outstanding-ack backpressure and cooperative cleanup.
+- `Source.ractor_merge_producers` for ready-order fan-in from multiple
+  FiberStream-owned producer Ractors without requiring a `Fiber.scheduler`.
+- `Flow.scan(initial)` and `Source#scan(initial)` for lazy running
+  accumulators using `Sink.fold`-style reducer semantics.
+
+### Changed
+
+- Updated README and website reference coverage for owned Ractor producers,
+  unordered parallel mapping, and scan.
+- Prefer high-level owned Ractor producer examples in user-facing
+  documentation while keeping low-level port APIs documented for externally
+  owned producers.
+- Updated the project Ruby pin to 4.0.5.
+
 ## 0.3.0 - 2026-06-06
 
 ### Added

data/README.md

@@ -1,12 +1,12 @@
 # FiberStream
+ FiberStream is a Ruby library for linear stream processing with pull-based backpressure. 
 
-FiberStream is an early-stage Ruby library for linear, pull-based stream
-processing with backpressure.
+It builds lazy Source definitions, transforms values with Flow stages, and materializes results with Sink objects.
 
-Build a lazy `Source`, transform it with `Flow` stages, and materialize it with
-a `Sink`.
+[![Gem Version](https://badge.fury.io/rb/fiber_stream.svg)](https://badge.fury.io/rb/fiber_stream)
 
 ## Quick Start
+Please see the project [documentation](https://dakatsuka.github.io/fiber_stream/) for more details.
 
 ```ruby
 require "fiber_stream"
@@ -27,11 +27,13 @@ FiberStream currently supports linear pipelines only.
 
 Implemented capabilities:
 
-- in-memory, IO, backpressure-aware Ractor port, and Ractor port merge sources
+- in-memory, IO, FiberStream-owned Ractor producer, backpressure-aware Ractor
+  port, and Ractor port merge sources
 - lazy source concatenation, zipping, and scheduler-backed merging
 - mapping, filtering, limiting, predicate-based limiting and dropping,
   fixed-prefix dropping, fixed-size grouping, line splitting, buffering, async
-  boundaries, ordered parallel mapping, and ordered Ractor-backed mapping
+  boundaries, ordered and unordered parallel mapping, and ordered
+  Ractor-backed mapping
 - array, first-element, fold, foreach, and IO sinks
 - reusable flow composition and runnable pipelines
 - foreground and scheduler-backed background pipeline execution
@@ -96,64 +98,51 @@ chunks =
   end.wait
 ```
 
-Ractor port sources connect a producer Ractor with an explicit ack handshake.
-The producer creates its acknowledgment port, waits for `RactorPort::Ack`, and
-then sends one typed message back to the FiberStream data port:
+Owned Ractor producer sources run producer blocks in FiberStream-managed
+Ractors. The producer block receives a `RactorProducer` context and emits one
+value per downstream demand:
 
 ```ruby
-data_port = Ractor::Port.new
-setup_port = Ractor::Port.new
-
-producer =
-  Ractor.new(data_port, setup_port) do |outbox, setup|
-    ack_port = Ractor::Port.new
-    setup.send(ack_port)
-
-    values = [1, 2, 3].to_enum
-
-    loop do
-      case ack_port.receive
-      in FiberStream::RactorPort::Ack
-        begin
-          outbox.send(FiberStream::RactorPort::Element.new(values.next))
-        rescue StopIteration
-          outbox.send(FiberStream::RactorPort::Complete.new)
-          break
-        end
-      in FiberStream::RactorPort::Cancel
-        break
-      end
+PRODUCE_VALUES =
+  Ractor.shareable_proc do |producer, values|
+    values.each do |value|
+      break unless producer.emit(value)
     end
   end
 
-ack_port = setup_port.receive
-
-FiberStream::Source.ractor_port(data_port, ack_port: ack_port)
+FiberStream::Source.ractor_producer([1, 2, 3], &PRODUCE_VALUES)
   .run_with(FiberStream::Sink.to_a)
 # => [1, 2, 3]
-
-producer.value
 ```
 
-`RactorPort::Failure` cause metadata is producer-provided and is surfaced on
-`RactorPortSourceError`. Redact internal paths, secrets, tenant data, or other
-sensitive details before sending failures across trust boundaries.
-
-Multiple producer Ractors can be merged directly without a scheduler-backed
-`Source#merge`. Each producer still receives at most one outstanding ack:
+Multiple owned producer Ractors can be merged directly without a
+scheduler-backed `Source#merge`. Each producer still receives at most one
+outstanding ack:
 
 ```ruby
+PRODUCE_TAGGED_VALUES =
+  Ractor.shareable_proc do |producer, tag, values|
+    values.each do |value|
+      break unless producer.emit([tag, value])
+    end
+  end
+
 source =
-  FiberStream::Source.ractor_merge_ports(
-    [
-      { port: data_port_a, ack_port: ack_port_a },
-      { port: data_port_b, ack_port: ack_port_b }
-    ]
-  )
+  FiberStream::Source.ractor_merge_producers do |group|
+    group.producer(:a, [1, 2], &PRODUCE_TAGGED_VALUES)
+    group.producer(:b, [3, 4], &PRODUCE_TAGGED_VALUES)
+  end
 
-values = source.run_with(FiberStream::Sink.to_a)
+source.run_with(FiberStream::Sink.to_a)
+# Example result: [[:a, 1], [:b, 3], [:a, 2], [:b, 4]]
 ```
 
+Use the lower-level `Source.ractor_port` and `Source.ractor_merge_ports` APIs
+when producer Ractors are owned outside FiberStream or need custom lifecycle
+handling. `RactorPort::Failure` cause metadata is producer-provided and is
+surfaced on `RactorPortSourceError`; redact sensitive details before sending
+failures across trust boundaries.
+
 Streaming HTTP response bodies that implement `#each`, such as
 `async-http` response bodies, can be used with `Source.each` without buffering
 the full body first. Use the HTTP client's block form or an explicit `ensure`
@@ -240,6 +229,26 @@ profiles =
 profiles.map { |profile| profile.fetch(:id) } # => [1, 2, 3, 4]
 ```
 
+Use `parallel_unordered_map` when every result can be handled independently
+and lower head-of-line blocking matters more than input order. It still limits
+in-flight mapping work to `concurrency`, but emits values as mapping jobs
+finish:
+
+```ruby
+require "async"
+require "fiber_stream"
+
+responses =
+  Sync do
+    FiberStream::Source.each(["/a", "/slow", "/b"])
+      .parallel_unordered_map(concurrency: 3) { |path| fetch_path(path) }
+      .run_with(FiberStream::Sink.to_a)
+  end
+
+# Results are in completion order, not necessarily input order.
+responses
+```
+
 Use `ractor_map` for ordered CPU-bound mapping in Ractor workers. The mapper
 must be shareable, usually by creating it with `Ractor.shareable_proc`.
 
@@ -384,6 +393,17 @@ batches =
 batches # => [[1, 2], [3, 4], [5]]
 ```
 
+`Flow.scan` emits the updated accumulator for each upstream element:
+
+```ruby
+running_totals =
+  FiberStream::Source.each([1, 2, 3, 4])
+    .scan(0) { |sum, number| sum + number }
+    .run_with(FiberStream::Sink.to_a)
+
+running_totals # => [1, 3, 6, 10]
+```
+
 `Flow.take_while` emits the leading prefix while a predicate is truthy, then
 closes upstream at the first false or nil result:
 
@@ -450,14 +470,15 @@ merged =
 
 `merge` does not make scheduler-unaware blocking source work non-blocking and
 does not provide CPU parallelism. Use producer ractors with
-`Source.ractor_port` or `Source.ractor_merge_ports` when producer work needs
-true isolation.
+`Source.ractor_producer` or `Source.ractor_merge_producers` when producer work
+needs true isolation.
 
 `Flow.buffer(count)` allows bounded prefetch. `Flow.async`, `Flow.buffer`,
-`Flow.parallel_map`, `Source.io`, `Source#merge`, `Sink.io`, and
-`Pipeline#run_async` require an installed `Fiber.scheduler` and a non-blocking
-current fiber when demanded or started. FiberStream does not install a
-scheduler and does not depend on Async at runtime.
+`Flow.parallel_map`, `Flow.parallel_unordered_map`, `Source.io`,
+`Source#merge`, `Sink.io`, and `Pipeline#run_async` require an installed
+`Fiber.scheduler` and a non-blocking current fiber when demanded or started.
+FiberStream does not install a scheduler and does not depend on Async at
+runtime.
 
 ## API Surface
 
@@ -465,6 +486,8 @@ Sources:
 
 - `FiberStream::Source.each(enumerable)`
 - `FiberStream::Source.io(io, chunk_size: 16 * 1024, close: false)`
+- `FiberStream::Source.ractor_producer(*args, transfer: :copy, ack_transfer: :copy) { |producer, *args| ... }`
+- `FiberStream::Source.ractor_merge_producers(transfer: :copy, ack_transfer: :copy) { |group| ... }`
 - `FiberStream::Source.ractor_port(port, ack_port:, ack_transfer: :copy, cancel: true)`
 - `FiberStream::Source.ractor_merge_ports(ports, ack_transfer: :copy, cancel: true)`
 
@@ -476,11 +499,13 @@ Source convenience methods:
 - `Source#merge(source)`
 - `Source#map { |element| ... }`
 - `Source#parallel_map(concurrency:) { |element| ... }`
+- `Source#parallel_unordered_map(concurrency:) { |element| ... }`
 - `Source#ractor_map(workers:, input_transfer: :copy, output_transfer: :copy) { |element| ... }`
 - `Source#select { |element| ... }`
 - `Source#take(count)`
 - `Source#drop(count)`
 - `Source#grouped(count)`
+- `Source#scan(initial) { |accumulator, element| ... }`
 - `Source#take_while { |element| ... }`
 - `Source#drop_while { |element| ... }`
 - `Source#async`
@@ -494,11 +519,13 @@ Flows:
 
 - `FiberStream::Flow.map { |element| ... }`
 - `FiberStream::Flow.parallel_map(concurrency:) { |element| ... }`
+- `FiberStream::Flow.parallel_unordered_map(concurrency:) { |element| ... }`
 - `FiberStream::Flow.ractor_map(workers:, input_transfer: :copy, output_transfer: :copy) { |element| ... }`
 - `FiberStream::Flow.select { |element| ... }`
 - `FiberStream::Flow.take(count)`
 - `FiberStream::Flow.drop(count)`
 - `FiberStream::Flow.grouped(count)`
+- `FiberStream::Flow.scan(initial) { |accumulator, element| ... }`
 - `FiberStream::Flow.take_while { |element| ... }`
 - `FiberStream::Flow.drop_while { |element| ... }`
 - `FiberStream::Flow.async`
@@ -541,6 +568,7 @@ bundle exec ruby examples/file_copy.rb
 bundle exec ruby examples/backpressure_buffer.rb
 bundle exec ruby examples/background_execution.rb
 bundle exec ruby examples/ractor_map_hashing.rb
+bundle exec ruby examples/ractor_producer_sources.rb
 bundle exec ruby examples/ractor_port_source.rb
 bundle exec ruby examples/ractor_merge_ports_and_map.rb
 bundle exec ruby examples/async_http_requests.rb
@@ -553,6 +581,9 @@ events so the difference between direct demand and bounded prefetch is visible.
 `examples/ractor_map_hashing.rb` demonstrates ordered Ractor-backed hashing
 with a shareable mapper proc and `input_transfer: :move`.
 
+`examples/ractor_producer_sources.rb` demonstrates high-level owned producer
+Ractors with `Source.ractor_producer` and `Source.ractor_merge_producers`.
+
 `examples/ractor_port_source.rb` demonstrates a producer Ractor that waits for
 `RactorPort::Ack` before sending each `RactorPort::Element`.
 
@@ -578,7 +609,7 @@ bundle exec ruby benchmarks/heavy_cpu_map.rb
 
 ## Development
 
-This project targets Ruby 4.x. The repository currently pins Ruby 4.0.3 in
+This project targets Ruby 4.x. The repository currently pins Ruby 4.0.5 in
 `mise.toml`.
 
 Install dependencies:

data/examples/README.md

@@ -11,6 +11,7 @@ bundle exec ruby examples/backpressure_buffer.rb
 bundle exec ruby examples/background_execution.rb
 bundle exec ruby examples/ractor_map_hashing.rb
 bundle exec ruby examples/ractor_port_source.rb
+bundle exec ruby examples/ractor_producer_sources.rb
 bundle exec ruby examples/ractor_merge_ports_and_map.rb
 bundle exec ruby examples/async_http_requests.rb
 bundle exec ruby examples/async_http_streaming_body.rb
@@ -48,6 +49,11 @@ pipeline runs.
 `RactorPort::Ack`, and sends one typed `RactorPort::Element` per downstream
 demand.
 
+`ractor_producer_sources.rb` demonstrates the high-level owned-producer APIs:
+`Source.ractor_producer` for one producer and `Source.ractor_merge_producers`
+for ready-order fan-in from multiple producers. FiberStream creates the ports,
+producer Ractors, and cooperative cleanup path.
+
 `ractor_merge_ports_and_map.rb` runs CPU-bound work in multiple producer
 Ractors, merges their port outputs with `Source.ractor_merge_ports`, then runs
 another CPU-bound verification stage with `ractor_map`.

data/examples/ractor_producer_sources.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "fiber_stream"
+
+EMIT_NUMBERS =
+  Ractor.shareable_proc do |producer, range|
+    range.each do |number|
+      break unless producer.emit(number)
+    end
+  end
+
+EMIT_TAGGED_NUMBERS =
+  Ractor.shareable_proc do |producer, tag, range|
+    range.each do |number|
+      break unless producer.emit([tag, number])
+    end
+  end
+
+squares =
+  FiberStream::Source.ractor_producer(1..5, &EMIT_NUMBERS)
+    .map { |number| number * number }
+    .run_with(FiberStream::Sink.to_a)
+
+puts "Squares from one FiberStream-owned producer Ractor:"
+puts squares.join(", ")
+
+merged =
+  FiberStream::Source.ractor_merge_producers do |group|
+    group.producer(:low, 1..3, &EMIT_TAGGED_NUMBERS)
+    group.producer(:high, 10..12, &EMIT_TAGGED_NUMBERS)
+  end.run_with(FiberStream::Sink.to_a)
+
+puts
+puts "Merged values from two owned producer Ractors:"
+merged.each do |tag, number|
+  puts "- #{tag}: #{number}"
+end
+
+puts
+puts "Producer blocks use RactorProducer#emit and stop when it returns false."
+puts "FiberStream creates the data ports, ack ports, producer Ractors, and cleanup path."

data/lib/fiber_stream/flow.rb

@@ -30,6 +30,23 @@ module FiberStream
       new { |upstream| Pull.parallel_map(upstream, concurrency, block) }
     end
 
+    # Creates an unordered scheduler-backed parallel mapping flow.
+    #
+    # The stage starts internal scheduled fibers on first downstream demand and
+    # requires an installed `Fiber.scheduler` in a non-blocking fiber at that
+    # point. At most `concurrency` mapping blocks run at the same time, and at
+    # most `concurrency` upstream elements are pulled but not yet emitted downstream.
+    # Results are emitted in completion order and input order is not preserved.
+    # Closing the boundary closes upstream and requests internal worker
+    # cancellation. FiberStream does not depend on Async at runtime.
+    def self.parallel_unordered_map(concurrency:, &block)
+      raise ArgumentError, "missing block" unless block
+      raise TypeError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
+      raise ArgumentError, "concurrency must be positive" unless concurrency.positive?
+
+      new { |upstream| Pull.parallel_unordered_map(upstream, concurrency, block) }
+    end
+
     # Creates an ordered Ractor-backed mapping flow.
     #
     # The mapper runs inside worker ractors and must be shareable, typically
@@ -42,8 +59,8 @@ module FiberStream
       raise TypeError, "workers must be an Integer" unless workers.is_a?(Integer)
       raise ArgumentError, "workers must be positive" unless workers.positive?
 
-      validate_ractor_transfer_policy!(:input_transfer, input_transfer)
-      validate_ractor_transfer_policy!(:output_transfer, output_transfer)
+      Internal::RactorTransferPolicy.validate!(:input_transfer, input_transfer)
+      Internal::RactorTransferPolicy.validate!(:output_transfer, output_transfer)
       raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
 
       new { |upstream| Pull.ractor_map(upstream, workers, input_transfer, output_transfer, block) }
@@ -100,6 +117,18 @@ module FiberStream
       new { |upstream| Pull.grouped(upstream, count) }
     end
 
+    # Creates a running-accumulator flow.
+    #
+    # The block is called as `block.call(accumulator, element)` for each
+    # upstream element, matching `Sink.fold`. The block result becomes the new
+    # accumulator and is emitted downstream. The initial accumulator is not
+    # emitted before the first upstream element.
+    def self.scan(initial, &block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.scan(upstream, initial, block) }
+    end
+
     # Creates a predicate-based limiting flow.
     #
     # The flow emits leading elements while the block result is truthy. The
@@ -189,14 +218,10 @@ module FiberStream
       new { |upstream| Pull.split(upstream, separator, keep_separator, max_length) }
     end
 
-    def self.validate_ractor_transfer_policy!(name, value)
-      return if [:copy, :move].include?(value)
-
-      raise ArgumentError, "#{name} must be :copy or :move"
+    def self.build(&attach) # :nodoc:
+      new(&attach)
     end
 
-    private_class_method :validate_ractor_transfer_policy!
-
     # Returns a reusable flow that applies this flow and then `flow`.
     #
     # Construction is lazy. No upstream stream is attached and no elements are
@@ -204,11 +229,11 @@ module FiberStream
     def via(flow)
       raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
 
-      self.class.__send__(:new) do |upstream|
-        attached_stream = attach(upstream)
+      self.class.build do |upstream|
+        attached_stream = attach_to(upstream)
 
         begin
-          flow.__send__(:attach, attached_stream)
+          flow.attach_to(attached_stream)
         rescue StandardError
           begin
             attached_stream.close
@@ -228,13 +253,13 @@ module FiberStream
     def to(sink)
       raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
 
-      Sink.__send__(:new) do |stream|
+      Sink.build do |stream|
         attached_stream = nil
         primary_error = nil
 
         begin
-          attached_stream = attach(stream)
-          sink.__send__(:run, attached_stream)
+          attached_stream = attach_to(stream)
+          sink.run_stream(attached_stream)
         rescue StandardError => error
           primary_error = error
           raise
@@ -254,9 +279,7 @@ module FiberStream
 
     private_class_method :new
 
-    private
-
-    def attach(upstream)
+    def attach_to(upstream) # :nodoc:
       @attach.call(upstream)
     end
   end

data/lib/fiber_stream/internal/ractor_transfer_policy.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Internal # :nodoc:
+    module RactorTransferPolicy # :nodoc:
+      module_function
+
+      def validate!(name, value)
+        return if [:copy, :move].include?(value)
+
+        raise ArgumentError, "#{name} must be :copy or :move"
+      end
+    end
+  end
+
+  private_constant :Internal
+end

data/lib/fiber_stream/pipeline.rb

@@ -2,6 +2,10 @@
 
 module FiberStream
   class Pipeline
+    def self.build(source, sink) # :nodoc:
+      new(source, sink)
+    end
+
     def initialize(source, sink)
       @source = source
       @sink = sink
@@ -27,7 +31,7 @@ module FiberStream
     def run_async
       validate_scheduler!
 
-      RunningPipeline.__send__(:new, Fiber.scheduler) { run }
+      RunningPipeline.start(Fiber.scheduler) { run }
     end
 
     private_class_method :new