fiber_stream 0.3.0 → 0.4.0

data/lib/fiber_stream/sink.rb

@@ -10,10 +10,7 @@ module FiberStream
       new do |stream|
         values = []
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           values << value
         end
 
@@ -45,10 +42,7 @@ module FiberStream
       new do |stream|
         accumulator = initial
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           accumulator = block.call(accumulator, value)
         end
 
@@ -68,10 +62,7 @@ module FiberStream
       new do |stream|
         count = 0
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           block.call(value)
           count += 1
         end
@@ -99,15 +90,17 @@ module FiberStream
       end
     end
 
+    def self.build(&run) # :nodoc:
+      new(&run)
+    end
+
     def initialize(&run)
       @run = run
     end
 
     private_class_method :new
 
-    private
-
-    def run(stream)
+    def run_stream(stream) # :nodoc:
       @run.call(stream)
     end
 
@@ -121,10 +114,7 @@ module FiberStream
       end
 
       def run(stream)
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           write(value)
         end
 

data/lib/fiber_stream/source.rb

@@ -47,7 +47,7 @@ module FiberStream
         raise TypeError, "ack_port must provide Ractor-style send"
       end
 
-      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
       raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
 
       new(-> { Pull.ractor_port(port, ack_port, ack_transfer, cancel) })
@@ -65,12 +65,56 @@ module FiberStream
     def self.ractor_merge_ports(ports, ack_transfer: :copy, cancel: true)
       pairs = normalize_ractor_merge_port_pairs(ports)
 
-      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
       raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
 
       new(-> { Pull.ractor_merge_ports(pairs, ack_transfer, cancel) })
     end
 
+    # Creates a source backed by one FiberStream-owned producer ractor.
+    #
+    # The producer ractor is started lazily on first downstream demand. The
+    # shareable block receives a `RactorProducer` context and the provided
+    # arguments. Calls to the context preserve one-outstanding-ack
+    # backpressure, and cleanup always requests cooperative cancellation.
+    def self.ractor_producer(*args, transfer: :copy, ack_transfer: :copy, &block)
+      raise ArgumentError, "missing block" unless block
+
+      Internal::RactorTransferPolicy.validate!(:transfer, transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
+      raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
+
+      group = RactorProducerGroup.new(transfer)
+      group.producer(*args, &block)
+      definitions = group.definitions
+
+      new(-> { Pull.ractor_producer(definitions, ack_transfer) })
+    end
+
+    # Creates a source backed by multiple FiberStream-owned producer ractors.
+    #
+    # The registration block runs at construction to collect producer
+    # definitions, but producer ractors and ports are started lazily on first
+    # downstream demand. Outputs are merged with the same ready-order semantics
+    # as `Source.ractor_merge_ports`.
+    def self.ractor_merge_producers(transfer: :copy, ack_transfer: :copy, &block)
+      raise ArgumentError, "missing block" unless block
+
+      Internal::RactorTransferPolicy.validate!(:transfer, transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
+
+      group = RactorProducerGroup.new(transfer)
+      block.call(group)
+      definitions = group.definitions
+      raise ArgumentError, "ractor_merge_producers requires at least two producers" if definitions.size < 2
+
+      new(-> { Pull.ractor_merge_producers(definitions, ack_transfer) })
+    end
+
+    def self.build(source_factory, flows = []) # :nodoc:
+      new(source_factory, flows)
+    end
+
     def initialize(source_factory, flows = [])
       @source_factory = source_factory
       @flows = flows
@@ -83,7 +127,7 @@ module FiberStream
     def via(flow)
       raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
 
-      self.class.__send__(:new, @source_factory, @flows + [flow])
+      self.class.build(@source_factory, @flows + [flow])
     end
 
     # Returns a new source definition that emits this source, then `source`.
@@ -95,10 +139,7 @@ module FiberStream
     def concat(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.concat(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.concat(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that emits pairs from this source and
@@ -111,10 +152,7 @@ module FiberStream
     def zip(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.zip(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.zip(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that emits values from this source and
@@ -128,10 +166,7 @@ module FiberStream
     def merge(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.merge(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.merge(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that maps each element with `block`.
@@ -153,6 +188,18 @@ module FiberStream
       via(Flow.parallel_map(concurrency: concurrency, &block))
     end
 
+    # Returns a new source definition that maps elements concurrently and emits
+    # mapped values in completion order.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.parallel_unordered_map(concurrency:) { ... })`.
+    # The operation preserves the same scheduler requirement, validation,
+    # bounded upstream run-ahead, and cancellation behavior while making no
+    # input-order guarantee.
+    def parallel_unordered_map(concurrency:, &block)
+      via(Flow.parallel_unordered_map(concurrency: concurrency, &block))
+    end
+
     # Returns a new source definition that maps elements in Ractor workers.
     #
     # This is a convenience wrapper around
@@ -204,6 +251,15 @@ module FiberStream
       via(Flow.grouped(count))
     end
 
+    # Returns a new source definition that emits running accumulators.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.scan(initial) { ... })` and preserves the same
+    # reducer order, lazy construction, and pull-driven backpressure behavior.
+    def scan(initial, &block)
+      via(Flow.scan(initial, &block))
+    end
+
     # Returns a new source definition that emits leading elements while `block`
     # is truthy.
     #
@@ -270,7 +326,7 @@ module FiberStream
     def to(sink)
       raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
 
-      Pipeline.__send__(:new, self, sink)
+      Pipeline.build(self, sink)
     end
 
     # Materializes and runs this source with `sink`.
@@ -286,7 +342,7 @@ module FiberStream
       begin
         stream = materialize
 
-        sink.__send__(:run, stream)
+        sink.run_stream(stream)
       rescue StandardError => error
         primary_error = error
         raise
@@ -301,6 +357,10 @@ module FiberStream
 
     private_class_method :new
 
+    def to_pull_materializer # :nodoc:
+      method(:materialize)
+    end
+
     def self.normalize_ractor_merge_port_pairs(ports)
       raise TypeError, "ports must respond to each" unless ports.respond_to?(:each)
 
@@ -341,17 +401,13 @@ module FiberStream
 
     private
 
-    def materializer
-      -> { materialize }
-    end
-
     def materialize
       stream = nil
 
       begin
         stream = @source_factory.call
         @flows.each do |flow|
-          stream = flow.__send__(:attach, stream)
+          stream = flow.attach_to(stream)
         end
         stream
       rescue StandardError

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/fiber_stream.rb

@@ -3,7 +3,9 @@
 require_relative "fiber_stream/pull"
 require_relative "fiber_stream/version"
 require_relative "fiber_stream/errors"
+require_relative "fiber_stream/internal/ractor_transfer_policy"
 require_relative "fiber_stream/ractor_port"
+require_relative "fiber_stream/ractor_producer"
 require_relative "fiber_stream/flow"
 require_relative "fiber_stream/sink"
 require_relative "fiber_stream/running_pipeline"

data/sig/fiber_stream.rbs

@@ -2,7 +2,7 @@ module FiberStream
   type ractor_transfer_policy = :copy | :move
   type ractor_port_pair = { port: untyped, ack_port: untyped }
   type ractor_map_error_kind = :input_transfer | :output_transfer | :worker | :worker_termination | :isolation
-  type ractor_port_source_error_kind = :invalid_message | :producer_failure | :receive | :ack_transfer | :cancel_transfer
+  type ractor_port_source_error_kind = :invalid_message | :producer_failure | :receive | :ack_transfer | :cancel_transfer | :producer_setup
   type ractor_port_cancel_reason = :closed
 
   class SchedulerRequiredError < RuntimeError
@@ -48,22 +48,37 @@ module FiberStream
     end
   end
 
+  class RactorProducer
+    def emit: [Elem] (Elem value, ?transfer: ractor_transfer_policy?) -> bool
+    def complete: () -> bool
+    def fail: (?untyped error, ?cause_class_name: String?, ?cause_message: String?) -> bool
+    def cancelled?: () -> bool
+  end
+
+  class RactorProducerGroup
+    def producer: (*untyped args, ?transfer: ractor_transfer_policy?) { (RactorProducer producer, *untyped args) -> void } -> self
+  end
+
   class Source[Elem]
     def self.each: [Elem] (Enumerable[Elem] enumerable) -> Source[Elem]
     def self.io: (untyped io, ?chunk_size: Integer, ?close: bool) -> Source[String]
     def self.ractor_port: [Elem] (untyped port, ack_port: untyped, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
     def self.ractor_merge_ports: [Elem] (Enumerable[ractor_port_pair] ports, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
+    def self.ractor_producer: [Elem] (*untyped args, ?transfer: ractor_transfer_policy, ?ack_transfer: ractor_transfer_policy) { (RactorProducer producer, *untyped args) -> void } -> Source[Elem]
+    def self.ractor_merge_producers: [Elem] (?transfer: ractor_transfer_policy, ?ack_transfer: ractor_transfer_policy) { (RactorProducerGroup group) -> void } -> Source[Elem]
     def via: [Out] (Flow[Elem, Out] flow) -> Source[Out]
     def concat: [Other] (Source[Other] source) -> Source[Elem | Other]
     def zip: [Other] (Source[Other] source) -> Source[[Elem, Other]]
     def merge: [Other] (Source[Other] source) -> Source[Elem | Other]
     def map: [Out] () { (Elem) -> Out } -> Source[Out]
     def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
+    def parallel_unordered_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
     def select: () { (Elem) -> boolish } -> Source[Elem]
     def take: (Integer count) -> Source[Elem]
     def drop: (Integer count) -> Source[Elem]
     def grouped: (Integer count) -> Source[Array[Elem]]
+    def scan: [Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Source[Acc]
     def take_while: () { (Elem) -> boolish } -> Source[Elem]
     def drop_while: () { (Elem) -> boolish } -> Source[Elem]
     def async: () -> Source[Elem]
@@ -77,11 +92,13 @@ module FiberStream
   class Flow[In, Out]
     def self.map: [In, Out] () { (In) -> Out } -> Flow[In, Out]
     def self.parallel_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
+    def self.parallel_unordered_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
     def self.ractor_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
     def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.drop: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.grouped: [Elem] (Integer count) -> Flow[Elem, Array[Elem]]
+    def self.scan: [Elem, Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Flow[Elem, Acc]
     def self.take_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.drop_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.async: [Elem] () -> Flow[Elem, Elem]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_stream
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Dai Akatsuka
@@ -119,9 +119,11 @@ files:
 - examples/ractor_map_hashing.rb
 - examples/ractor_merge_ports_and_map.rb
 - examples/ractor_port_source.rb
+- examples/ractor_producer_sources.rb
 - lib/fiber_stream.rb
 - lib/fiber_stream/errors.rb
 - lib/fiber_stream/flow.rb
+- lib/fiber_stream/internal/ractor_transfer_policy.rb
 - lib/fiber_stream/pipeline.rb
 - lib/fiber_stream/pull.rb
 - lib/fiber_stream/pull/async_boundary.rb
@@ -136,15 +138,19 @@ files:
 - lib/fiber_stream/pull/map.rb
 - lib/fiber_stream/pull/merge.rb
 - lib/fiber_stream/pull/parallel_map_boundary.rb
+- lib/fiber_stream/pull/parallel_unordered_map_boundary.rb
 - lib/fiber_stream/pull/ractor_map_boundary.rb
 - lib/fiber_stream/pull/ractor_merge_ports_source.rb
 - lib/fiber_stream/pull/ractor_port_source.rb
+- lib/fiber_stream/pull/ractor_producer_source.rb
+- lib/fiber_stream/pull/scan.rb
 - lib/fiber_stream/pull/select.rb
 - lib/fiber_stream/pull/split.rb
 - lib/fiber_stream/pull/take.rb
 - lib/fiber_stream/pull/take_while.rb
 - lib/fiber_stream/pull/zip.rb
 - lib/fiber_stream/ractor_port.rb
+- lib/fiber_stream/ractor_producer.rb
 - lib/fiber_stream/running_pipeline.rb
 - lib/fiber_stream/sink.rb
 - lib/fiber_stream/source.rb
@@ -156,7 +162,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/dakatsuka/fiber_stream
-  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.3.0
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.4.0
   changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -173,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Asynchronous, non-blocking stream processing with backpressure.
 test_files: []