fiber_stream 0.3.0 → 0.5.0

data/lib/fiber_stream/flow.rb

@@ -13,6 +13,51 @@ module FiberStream
       new { |upstream| Pull.map(upstream, block) }
     end
 
+    # Creates a transform-and-filter flow.
+    #
+    # The block is called once for each upstream element observed by this
+    # stage. Truthy block results are emitted downstream as transformed values;
+    # false and nil results are dropped. Exceptions raised by the block fail the
+    # stream and are re-raised from `Source#run_with`.
+    def self.filter_map(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.filter_map(upstream, block) }
+    end
+
+    # Creates a nil-dropping flow.
+    #
+    # The flow drops `nil` elements and passes every non-`nil` element through
+    # unchanged, including `false`.
+    def self.compact
+      new { |upstream| Pull.compact(upstream) }
+    end
+
+    # Creates a one-to-many mapping flow.
+    #
+    # The block is called once for each upstream element whose expansion is
+    # needed. It must return an object that responds to `#each`; yielded values
+    # are emitted in order before the next upstream element is pulled.
+    # Exceptions raised by the block or by the returned object's `#each` fail
+    # the stream and are re-raised from `Source#run_with`.
+    def self.map_concat(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.map_concat(upstream, block) }
+    end
+
+    # Creates a pass-through observing flow.
+    #
+    # The block is called once for each element before that element is emitted
+    # downstream. The block return value is ignored and the original element is
+    # passed through unchanged. Exceptions raised by the block fail the stream
+    # and are re-raised from `Source#run_with`.
+    def self.tap(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.tap(upstream, block) }
+    end
+
     # Creates an ordered scheduler-backed parallel mapping flow.
     #
     # The stage starts internal scheduled fibers on first downstream demand and
@@ -30,6 +75,23 @@ module FiberStream
       new { |upstream| Pull.parallel_map(upstream, concurrency, block) }
     end
 
+    # Creates an unordered scheduler-backed parallel mapping flow.
+    #
+    # The stage starts internal scheduled fibers on first downstream demand and
+    # requires an installed `Fiber.scheduler` in a non-blocking fiber at that
+    # point. At most `concurrency` mapping blocks run at the same time, and at
+    # most `concurrency` upstream elements are pulled but not yet emitted downstream.
+    # Results are emitted in completion order and input order is not preserved.
+    # Closing the boundary closes upstream and requests internal worker
+    # cancellation. FiberStream does not depend on Async at runtime.
+    def self.parallel_unordered_map(concurrency:, &block)
+      raise ArgumentError, "missing block" unless block
+      raise TypeError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
+      raise ArgumentError, "concurrency must be positive" unless concurrency.positive?
+
+      new { |upstream| Pull.parallel_unordered_map(upstream, concurrency, block) }
+    end
+
     # Creates an ordered Ractor-backed mapping flow.
     #
     # The mapper runs inside worker ractors and must be shareable, typically
@@ -42,8 +104,8 @@ module FiberStream
       raise TypeError, "workers must be an Integer" unless workers.is_a?(Integer)
       raise ArgumentError, "workers must be positive" unless workers.positive?
 
-      validate_ractor_transfer_policy!(:input_transfer, input_transfer)
-      validate_ractor_transfer_policy!(:output_transfer, output_transfer)
+      Internal::RactorTransferPolicy.validate!(:input_transfer, input_transfer)
+      Internal::RactorTransferPolicy.validate!(:output_transfer, output_transfer)
       raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
 
       new { |upstream| Pull.ractor_map(upstream, workers, input_transfer, output_transfer, block) }
@@ -61,6 +123,19 @@ module FiberStream
       new { |upstream| Pull.select(upstream, block) }
     end
 
+    # Creates a complement filtering flow.
+    #
+    # The block is called for upstream elements until it returns `false` or
+    # `nil`, or upstream completes. Truthy predicate results drop the original
+    # element; false and nil results pass the element through unchanged.
+    # Exceptions raised by the block fail the stream and are re-raised from
+    # `Source#run_with`.
+    def self.reject(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.reject(upstream, block) }
+    end
+
     # Creates a limiting flow.
     #
     # The flow emits at most `count` elements. `take(0)` completes without
@@ -100,6 +175,18 @@ module FiberStream
       new { |upstream| Pull.grouped(upstream, count) }
     end
 
+    # Creates a running-accumulator flow.
+    #
+    # The block is called as `block.call(accumulator, element)` for each
+    # upstream element, matching `Sink.fold`. The block result becomes the new
+    # accumulator and is emitted downstream. The initial accumulator is not
+    # emitted before the first upstream element.
+    def self.scan(initial, &block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.scan(upstream, initial, block) }
+    end
+
     # Creates a predicate-based limiting flow.
     #
     # The flow emits leading elements while the block result is truthy. The
@@ -151,6 +238,19 @@ module FiberStream
       new { |upstream| Pull.buffer(upstream, count) }
     end
 
+    # Creates a scheduler-aware throttling flow.
+    #
+    # The `rate:` form creates a fresh `RateLimiter` for each materialization.
+    # The `limiter:` form uses the supplied limiter object, which must respond
+    # to `acquire(permits:)` and return only after permits are acquired. When
+    # FiberStream-owned waiting is required, the current fiber must be
+    # non-blocking with an installed `Fiber.scheduler`.
+    def self.throttle(**options)
+      limiter = build_throttle_limiter(options)
+
+      new { |upstream| Pull.throttle(upstream, limiter.call) }
+    end
+
     # Creates a line-splitting flow.
     #
     # The flow accepts String chunks and emits lines split on "\n". By default
@@ -189,13 +289,41 @@ module FiberStream
       new { |upstream| Pull.split(upstream, separator, keep_separator, max_length) }
     end
 
-    def self.validate_ractor_transfer_policy!(name, value)
-      return if [:copy, :move].include?(value)
+    def self.build(&attach) # :nodoc:
+      new(&attach)
+    end
+
+    def self.build_throttle_limiter(options)
+      unknown_keywords = options.keys - [:rate, :per, :burst, :limiter]
+      raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" unless unknown_keywords.empty?
+
+      rate_given = options.key?(:rate)
+      per_given = options.key?(:per)
+      burst_given = options.key?(:burst)
+      limiter_given = options.key?(:limiter)
+
+      if limiter_given
+        raise ArgumentError, "cannot pass rate and limiter together" if rate_given
+        raise ArgumentError, "cannot pass per with limiter" if per_given
+        raise ArgumentError, "cannot pass burst with limiter" if burst_given
 
-      raise ArgumentError, "#{name} must be :copy or :move"
+        limiter = options.fetch(:limiter)
+        raise TypeError, "limiter must respond to acquire" unless limiter.respond_to?(:acquire)
+
+        return -> { limiter }
+      end
+
+      raise ArgumentError, "missing rate or limiter" unless rate_given
+
+      rate = options.fetch(:rate)
+      per = options.fetch(:per, 1)
+      burst = options.fetch(:burst, nil)
+      RateLimiter.validate_options!(rate:, per:, burst:)
+
+      -> { RateLimiter.new(rate:, per:, burst:) }
     end
 
-    private_class_method :validate_ractor_transfer_policy!
+    private_class_method :build_throttle_limiter
 
     # Returns a reusable flow that applies this flow and then `flow`.
     #
@@ -204,11 +332,11 @@ module FiberStream
     def via(flow)
       raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
 
-      self.class.__send__(:new) do |upstream|
-        attached_stream = attach(upstream)
+      self.class.build do |upstream|
+        attached_stream = attach_to(upstream)
 
         begin
-          flow.__send__(:attach, attached_stream)
+          flow.attach_to(attached_stream)
         rescue StandardError
           begin
             attached_stream.close
@@ -228,13 +356,13 @@ module FiberStream
     def to(sink)
       raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
 
-      Sink.__send__(:new) do |stream|
+      Sink.build do |stream|
         attached_stream = nil
         primary_error = nil
 
         begin
-          attached_stream = attach(stream)
-          sink.__send__(:run, attached_stream)
+          attached_stream = attach_to(stream)
+          sink.run_stream(attached_stream)
         rescue StandardError => error
           primary_error = error
           raise
@@ -254,9 +382,7 @@ module FiberStream
 
     private_class_method :new
 
-    private
-
-    def attach(upstream)
+    def attach_to(upstream) # :nodoc:
       @attach.call(upstream)
     end
   end

data/lib/fiber_stream/internal/ractor_transfer_policy.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Internal # :nodoc:
+    module RactorTransferPolicy # :nodoc:
+      module_function
+
+      def validate!(name, value)
+        return if [:copy, :move].include?(value)
+
+        raise ArgumentError, "#{name} must be :copy or :move"
+      end
+    end
+  end
+
+  private_constant :Internal
+end

data/lib/fiber_stream/pipeline.rb

@@ -2,6 +2,10 @@
 
 module FiberStream
   class Pipeline
+    def self.build(source, sink) # :nodoc:
+      new(source, sink)
+    end
+
     def initialize(source, sink)
       @source = source
       @sink = sink
@@ -27,7 +31,7 @@ module FiberStream
     def run_async
       validate_scheduler!
 
-      RunningPipeline.__send__(:new, Fiber.scheduler) { run }
+      RunningPipeline.start(Fiber.scheduler) { run }
     end
 
     private_class_method :new

data/lib/fiber_stream/pull/compact.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Nil-dropping stage.
+    #
+    # A single downstream demand may pull multiple upstream elements until a
+    # non-nil value is observed or upstream completes. Dropped nil values are
+    # discarded immediately and are not buffered.
+    class Compact
+      def initialize(upstream)
+        @upstream = upstream
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          return value unless value.nil?
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/filter_map.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Transform-and-filter stage.
+    #
+    # A single downstream demand may pull multiple upstream elements until the
+    # transform returns a truthy value or upstream completes. Falsey transform
+    # results are discarded immediately and are not buffered.
+    class FilterMap
+      def initialize(upstream, transform)
+        @upstream = upstream
+        @transform = transform
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          result = @transform.call(value)
+          return result if result
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/map_concat.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # One-to-many mapping stage.
+    #
+    # It expands one upstream element into the values yielded by one returned
+    # `#each` object. Only one expansion is active at a time, and the stage
+    # never pulls the next upstream element until the active expansion is
+    # exhausted.
+    class MapConcat
+      def initialize(upstream, transform)
+        @upstream = upstream
+        @transform = transform
+        @current_enumerator = nil
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          if @current_enumerator
+            begin
+              return @current_enumerator.next
+            rescue StopIteration
+              @current_enumerator = nil
+            end
+          end
+
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          result = @transform.call(value)
+          unless result.respond_to?(:each)
+            raise TypeError, "map_concat block result must respond to each"
+          end
+
+          @current_enumerator = result.to_enum(:each)
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @current_enumerator = nil
+        @upstream.close
+      end
+    end
+  end
+end