fiber_stream 0.3.0 → 0.5.0

data/lib/fiber_stream/ractor_producer.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module FiberStream
+  # Producer-side context for `Source.ractor_producer`.
+  #
+  # Producer blocks call `emit`, `complete`, or `fail` to send one protocol
+  # message after receiving one downstream acknowledgment. A `false` return
+  # means cooperative cancellation was observed before the requested message
+  # could be sent.
+  class RactorProducer
+    def initialize(data_port, ack_port, transfer)
+      @data_port = data_port
+      @ack_port = ack_port
+      @transfer = transfer
+      @terminal = false
+      @cancelled = false
+      @send_failed = false
+    end
+
+    def emit(value, transfer: nil)
+      return false if terminal? || cancelled?
+
+      message_transfer = validate_transfer_override!(transfer)
+      return false unless wait_for_ack
+
+      send_emitted_message(RactorPort::Element.new(value), message_transfer)
+    end
+
+    def complete
+      return false if terminal? || cancelled?
+      return false unless wait_for_ack
+
+      send_terminal_message(RactorPort::Complete.new)
+    end
+
+    def fail(error = nil, cause_class_name: nil, cause_message: nil)
+      return false if terminal? || cancelled?
+
+      failure = failure_message(error, cause_class_name, cause_message)
+      return false unless wait_for_ack
+
+      send_terminal_message(failure)
+    end
+
+    def cancelled?
+      @cancelled
+    end
+
+    def terminal? # :nodoc:
+      @terminal
+    end
+
+    def send_failed? # :nodoc:
+      @send_failed
+    end
+
+    private
+
+    def send_emitted_message(message, transfer)
+      send_data_message(message, transfer)
+      true
+    rescue Exception => error # rubocop:disable Lint/RescueException
+      report_same_ack_failure(error)
+      false
+    end
+
+    def send_terminal_message(message)
+      send_data_message(message, @transfer)
+      @terminal = true
+      true
+    rescue Exception => send_error # rubocop:disable Lint/RescueException
+      report_same_ack_failure(send_error)
+      false
+    end
+
+    def validate_transfer_override!(transfer)
+      return @transfer if transfer.nil?
+      return transfer if [:copy, :move].include?(transfer)
+
+      raise ArgumentError, "transfer must be :copy or :move"
+    end
+
+    def wait_for_ack
+      case @ack_port.receive
+      in RactorPort::Ack
+        true
+      in RactorPort::Cancel
+        @cancelled = true
+        false
+      else
+        raise TypeError, "invalid ractor producer control message"
+      end
+    end
+
+    def send_data_message(message, transfer)
+      if transfer == :move
+        @data_port.send(message, move: true)
+      else
+        @data_port.send(message)
+      end
+    end
+
+    def failure_message(error, cause_class_name, cause_message)
+      if error
+        return RactorPort::Failure.new(safe_class_name(error), safe_message(error))
+      end
+
+      unless cause_class_name.is_a?(String) && cause_message.is_a?(String)
+        raise ArgumentError, "fail requires an error or String failure metadata"
+      end
+
+      RactorPort::Failure.new(cause_class_name, cause_message)
+    end
+
+    def safe_class_name(error)
+      name = error.class.name
+      name.is_a?(String) && !name.empty? ? name : "Exception"
+    rescue Exception # rubocop:disable Lint/RescueException
+      "Exception"
+    end
+
+    def safe_message(error)
+      message = error.message
+      message.is_a?(String) ? message : ""
+    rescue Exception # rubocop:disable Lint/RescueException
+      ""
+    end
+
+    def report_same_ack_failure(error)
+      send_data_message(RactorPort::Failure.new(safe_class_name(error), safe_message(error)), :copy)
+      @terminal = true
+    rescue Exception # rubocop:disable Lint/RescueException
+      @terminal = true
+      @send_failed = true
+    end
+  end
+
+  # Builder passed to `Source.ractor_merge_producers`.
+  #
+  # Each `producer` call records one lazily started owned producer definition.
+  # Registration validates producer block isolation and transfer policy but
+  # does not create Ractor ports or start producer code.
+  class RactorProducerGroup
+    Definition = Data.define(:args, :transfer, :block)
+    private_constant :Definition
+
+    def initialize(default_transfer)
+      @default_transfer = default_transfer
+      @definitions = []
+    end
+
+    def producer(*args, transfer: nil, &block)
+      raise ArgumentError, "missing block" unless block
+      unless transfer.nil? || [:copy, :move].include?(transfer)
+        raise ArgumentError, "transfer must be :copy or :move"
+      end
+      raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
+
+      @definitions << Definition.new(args:, transfer: transfer || @default_transfer, block:)
+      self
+    end
+
+    def definitions
+      @definitions.dup.freeze
+    end
+  end
+end

data/lib/fiber_stream/rate_limiter.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module FiberStream
+  # Scheduler-aware token-bucket rate limiter.
+  #
+  # `rate` permits refill every `per` seconds. `burst` is the maximum token
+  # capacity and defaults to `rate`. Immediate permit grants do not require a
+  # scheduler. When FiberStream must sleep, the current fiber must be
+  # non-blocking with an installed `Fiber.scheduler`.
+  class RateLimiter
+    Request = Data.define(:rate, :per, :burst, :permits, :now)
+
+    class << self
+      def validate_options!(rate:, per: 1, burst: nil) # :nodoc:
+        rate = validate_rate(rate)
+        validate_duration(:per, per)
+        validate_burst(burst.nil? ? rate : burst)
+        nil
+      end
+
+      private
+
+      def validate_rate(rate)
+        raise TypeError, "rate must be an Integer" unless rate.is_a?(Integer)
+        raise ArgumentError, "rate must be positive" unless rate.positive?
+
+        rate
+      end
+
+      def validate_burst(burst)
+        raise TypeError, "burst must be an Integer" unless burst.is_a?(Integer)
+        raise ArgumentError, "burst must be positive" unless burst.positive?
+
+        burst
+      end
+
+      def validate_duration(name, duration)
+        raise TypeError, "#{name} must be Numeric" unless duration.is_a?(Numeric)
+        raise ArgumentError, "#{name} must be finite and real" unless finite_real?(duration)
+        raise ArgumentError, "#{name} must be positive" unless duration.positive?
+
+        duration
+      end
+
+      def finite_real?(value)
+        return false if value.is_a?(Complex)
+        return value.finite? if value.respond_to?(:finite?)
+
+        true
+      end
+    end
+
+    def initialize(rate:, per: 1, burst: nil, &block)
+      self.class.validate_options!(rate:, per:, burst:)
+
+      @rate = rate
+      @per = per
+      @burst = burst.nil? ? @rate : burst
+      @policy = block
+      @mutex = Mutex.new
+      @tokens = @burst.to_f
+      @updated_at = monotonic_now
+    end
+
+    # Acquires `permits`, waiting when necessary.
+    #
+    # Waits are scheduler-backed and non-blocking. Requests larger than `burst`
+    # are rejected because the local token bucket could never satisfy them.
+    def acquire(permits: 1)
+      permits = validate_permits(permits)
+
+      if @policy
+        acquire_with_policy(permits)
+      else
+        acquire_with_token_bucket(permits)
+      end
+
+      nil
+    end
+
+    private
+
+    def acquire_with_policy(permits)
+      loop do
+        wait = normalize_policy_wait(@policy.call(request_for(permits)))
+        return if wait <= 0
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def acquire_with_token_bucket(permits)
+      loop do
+        wait = nil
+
+        @mutex.synchronize do
+          refill_tokens
+
+          if @tokens >= permits
+            @tokens -= permits
+            return
+          end
+
+          wait = (permits - @tokens) / permits_per_second
+        end
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def request_for(permits)
+      Request.new(rate: @rate, per: @per, burst: @burst, permits:, now: monotonic_now)
+    end
+
+    def refill_tokens
+      now = monotonic_now
+      elapsed = now - @updated_at
+      @updated_at = now
+      @tokens = [@burst, @tokens + (elapsed * permits_per_second)].min
+    end
+
+    def permits_per_second
+      @rate.to_f / @per.to_f
+    end
+
+    def scheduler_sleep(duration)
+      validate_scheduler!
+      sleep(duration)
+    end
+
+    def validate_scheduler!
+      return if Fiber.scheduler && !Fiber.current.blocking?
+
+      message =
+        if Fiber.scheduler
+          "RateLimiter#acquire requires a non-blocking fiber"
+        else
+          "RateLimiter#acquire requires Fiber.scheduler"
+        end
+      raise SchedulerRequiredError, message
+    end
+
+    def validate_permits(permits)
+      raise TypeError, "permits must be an Integer" unless permits.is_a?(Integer)
+      raise ArgumentError, "permits must be positive" unless permits.positive?
+      raise ArgumentError, "permits must be less than or equal to burst" if permits > @burst
+
+      permits
+    end
+
+    def normalize_policy_wait(wait)
+      return 0 if wait.nil?
+      raise TypeError, "custom wait duration must be Numeric or nil" unless wait.is_a?(Numeric)
+      raise ArgumentError, "custom wait duration must be finite and real" unless self.class.send(:finite_real?, wait)
+
+      wait.positive? ? wait : 0
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/fiber_stream/running_pipeline.rb

@@ -7,6 +7,10 @@ module FiberStream
     CancelledMessage = Data.define(:error)
     private_constant :ValueMessage, :ErrorMessage, :CancelledMessage
 
+    def self.start(scheduler, &run) # :nodoc:
+      new(scheduler, &run)
+    end
+
     def initialize(scheduler, &run)
       @scheduler = scheduler
       @completion = nil

data/lib/fiber_stream/sink.rb

@@ -10,10 +10,7 @@ module FiberStream
       new do |stream|
         values = []
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           values << value
         end
 
@@ -32,6 +29,22 @@ module FiberStream
       end
     end
 
+    # Creates a sink that counts all stream elements.
+    #
+    # The sink consumes upstream until normal completion and returns the number
+    # of elements observed. It does not store consumed elements.
+    def self.count
+      new do |stream|
+        count = 0
+
+        Pull.each_value(stream) do
+          count += 1
+        end
+
+        count
+      end
+    end
+
     # Creates a sink that folds all stream elements into an accumulator.
     #
     # The sink consumes upstream until normal completion. It returns the final
@@ -45,10 +58,7 @@ module FiberStream
       new do |stream|
         accumulator = initial
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           accumulator = block.call(accumulator, value)
         end
 
@@ -68,10 +78,7 @@ module FiberStream
       new do |stream|
         count = 0
 
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           block.call(value)
           count += 1
         end
@@ -99,15 +106,17 @@ module FiberStream
       end
     end
 
+    def self.build(&run) # :nodoc:
+      new(&run)
+    end
+
     def initialize(&run)
       @run = run
     end
 
     private_class_method :new
 
-    private
-
-    def run(stream)
+    def run_stream(stream) # :nodoc:
       @run.call(stream)
     end
 
@@ -121,10 +130,7 @@ module FiberStream
       end
 
       def run(stream)
-        loop do
-          value = stream.next
-          break if Pull.done?(value)
-
+        Pull.each_value(stream) do |value|
           write(value)
         end
 

data/lib/fiber_stream/source.rb

@@ -47,7 +47,7 @@ module FiberStream
         raise TypeError, "ack_port must provide Ractor-style send"
       end
 
-      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
       raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
 
       new(-> { Pull.ractor_port(port, ack_port, ack_transfer, cancel) })
@@ -65,12 +65,56 @@ module FiberStream
     def self.ractor_merge_ports(ports, ack_transfer: :copy, cancel: true)
       pairs = normalize_ractor_merge_port_pairs(ports)
 
-      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
       raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
 
       new(-> { Pull.ractor_merge_ports(pairs, ack_transfer, cancel) })
     end
 
+    # Creates a source backed by one FiberStream-owned producer ractor.
+    #
+    # The producer ractor is started lazily on first downstream demand. The
+    # shareable block receives a `RactorProducer` context and the provided
+    # arguments. Calls to the context preserve one-outstanding-ack
+    # backpressure, and cleanup always requests cooperative cancellation.
+    def self.ractor_producer(*args, transfer: :copy, ack_transfer: :copy, &block)
+      raise ArgumentError, "missing block" unless block
+
+      Internal::RactorTransferPolicy.validate!(:transfer, transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
+      raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
+
+      group = RactorProducerGroup.new(transfer)
+      group.producer(*args, &block)
+      definitions = group.definitions
+
+      new(-> { Pull.ractor_producer(definitions, ack_transfer) })
+    end
+
+    # Creates a source backed by multiple FiberStream-owned producer ractors.
+    #
+    # The registration block runs at construction to collect producer
+    # definitions, but producer ractors and ports are started lazily on first
+    # downstream demand. Outputs are merged with the same ready-order semantics
+    # as `Source.ractor_merge_ports`.
+    def self.ractor_merge_producers(transfer: :copy, ack_transfer: :copy, &block)
+      raise ArgumentError, "missing block" unless block
+
+      Internal::RactorTransferPolicy.validate!(:transfer, transfer)
+      Internal::RactorTransferPolicy.validate!(:ack_transfer, ack_transfer)
+
+      group = RactorProducerGroup.new(transfer)
+      block.call(group)
+      definitions = group.definitions
+      raise ArgumentError, "ractor_merge_producers requires at least two producers" if definitions.size < 2
+
+      new(-> { Pull.ractor_merge_producers(definitions, ack_transfer) })
+    end
+
+    def self.build(source_factory, flows = []) # :nodoc:
+      new(source_factory, flows)
+    end
+
     def initialize(source_factory, flows = [])
       @source_factory = source_factory
       @flows = flows
@@ -83,7 +127,7 @@ module FiberStream
     def via(flow)
       raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
 
-      self.class.__send__(:new, @source_factory, @flows + [flow])
+      self.class.build(@source_factory, @flows + [flow])
     end
 
     # Returns a new source definition that emits this source, then `source`.
@@ -95,10 +139,7 @@ module FiberStream
     def concat(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.concat(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.concat(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that emits pairs from this source and
@@ -111,10 +152,7 @@ module FiberStream
     def zip(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.zip(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.zip(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that emits values from this source and
@@ -128,10 +166,7 @@ module FiberStream
     def merge(source)
       raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
 
-      self.class.__send__(
-        :new,
-        -> { Pull.merge(materializer, source.__send__(:materializer)) }
-      )
+      self.class.build(-> { Pull.merge(to_pull_materializer, source.to_pull_materializer) })
     end
 
     # Returns a new source definition that maps each element with `block`.
@@ -143,6 +178,35 @@ module FiberStream
       via(Flow.map(&block))
     end
 
+    # Returns a new source definition that emits truthy transformed values.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.filter_map { ... })` and has the same falsey-drop,
+    # lazy construction, error, and backpressure behavior as the underlying
+    # flow.
+    def filter_map(&block)
+      via(Flow.filter_map(&block))
+    end
+
+    # Returns a new source definition that drops nil elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.compact)` and
+    # preserves the same nil-only filtering, lazy construction, and
+    # backpressure behavior as the underlying flow.
+    def compact
+      via(Flow.compact)
+    end
+
+    # Returns a new source definition that emits each mapped expansion.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.map_concat { ... })` and has the same one-level
+    # flattening, lazy construction, error, and backpressure behavior as the
+    # underlying flow.
+    def map_concat(&block)
+      via(Flow.map_concat(&block))
+    end
+
     # Returns a new source definition that maps elements concurrently.
     #
     # This is a convenience wrapper around
@@ -153,6 +217,18 @@ module FiberStream
       via(Flow.parallel_map(concurrency: concurrency, &block))
     end
 
+    # Returns a new source definition that maps elements concurrently and emits
+    # mapped values in completion order.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.parallel_unordered_map(concurrency:) { ... })`.
+    # The operation preserves the same scheduler requirement, validation,
+    # bounded upstream run-ahead, and cancellation behavior while making no
+    # input-order guarantee.
+    def parallel_unordered_map(concurrency:, &block)
+      via(Flow.parallel_unordered_map(concurrency: concurrency, &block))
+    end
+
     # Returns a new source definition that maps elements in Ractor workers.
     #
     # This is a convenience wrapper around
@@ -179,6 +255,15 @@ module FiberStream
       via(Flow.select(&block))
     end
 
+    # Returns a new source definition that drops elements matching `block`.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.reject { ... })` and has the same truthiness and
+    # lazy construction behavior as the underlying flow.
+    def reject(&block)
+      via(Flow.reject(&block))
+    end
+
     # Returns a new source definition that emits at most `count` elements.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.take(count))`
@@ -204,6 +289,15 @@ module FiberStream
       via(Flow.grouped(count))
     end
 
+    # Returns a new source definition that emits running accumulators.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.scan(initial) { ... })` and preserves the same
+    # reducer order, lazy construction, and pull-driven backpressure behavior.
+    def scan(initial, &block)
+      via(Flow.scan(initial, &block))
+    end
+
     # Returns a new source definition that emits leading elements while `block`
     # is truthy.
     #
@@ -241,6 +335,15 @@ module FiberStream
       via(Flow.buffer(count))
     end
 
+    # Returns a new source definition that rate-limits emitted elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.throttle(...))`.
+    # The `rate:` form creates a fresh default limiter for each materialization;
+    # pass `limiter:` to share quota state across sources or runs.
+    def throttle(**options)
+      via(Flow.throttle(**options))
+    end
+
     # Returns a new source definition that splits String chunks into lines.
     #
     # This is a convenience wrapper around
@@ -270,7 +373,7 @@ module FiberStream
     def to(sink)
       raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
 
-      Pipeline.__send__(:new, self, sink)
+      Pipeline.build(self, sink)
     end
 
     # Materializes and runs this source with `sink`.
@@ -286,7 +389,7 @@ module FiberStream
       begin
         stream = materialize
 
-        sink.__send__(:run, stream)
+        sink.run_stream(stream)
       rescue StandardError => error
         primary_error = error
         raise
@@ -301,6 +404,10 @@ module FiberStream
 
     private_class_method :new
 
+    def to_pull_materializer # :nodoc:
+      method(:materialize)
+    end
+
     def self.normalize_ractor_merge_port_pairs(ports)
       raise TypeError, "ports must respond to each" unless ports.respond_to?(:each)
 
@@ -341,17 +448,13 @@ module FiberStream
 
     private
 
-    def materializer
-      -> { materialize }
-    end
-
     def materialize
       stream = nil
 
       begin
         stream = @source_factory.call
         @flows.each do |flow|
-          stream = flow.__send__(:attach, stream)
+          stream = flow.attach_to(stream)
         end
         stream
       rescue StandardError

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/fiber_stream.rb

@@ -3,7 +3,10 @@
 require_relative "fiber_stream/pull"
 require_relative "fiber_stream/version"
 require_relative "fiber_stream/errors"
+require_relative "fiber_stream/rate_limiter"
+require_relative "fiber_stream/internal/ractor_transfer_policy"
 require_relative "fiber_stream/ractor_port"
+require_relative "fiber_stream/ractor_producer"
 require_relative "fiber_stream/flow"
 require_relative "fiber_stream/sink"
 require_relative "fiber_stream/running_pipeline"