fiber_stream 0.4.0 → 0.6.0

data/lib/fiber_stream/pull.rb

@@ -85,6 +85,22 @@ module FiberStream
       Map.new(upstream, transform)
     end
 
+    def self.filter_map(upstream, transform)
+      FilterMap.new(upstream, transform)
+    end
+
+    def self.compact(upstream)
+      Compact.new(upstream)
+    end
+
+    def self.map_concat(upstream, transform)
+      MapConcat.new(upstream, transform)
+    end
+
+    def self.tap(upstream, observer)
+      Tap.new(upstream, observer)
+    end
+
     def self.parallel_map(upstream, concurrency, transform)
       ParallelMapBoundary.new(upstream, concurrency, transform)
     end
@@ -97,10 +113,18 @@ module FiberStream
       RactorMapBoundary.new(upstream, workers, input_transfer, output_transfer, transform)
     end
 
+    def self.ractor_unordered_map(upstream, workers, input_transfer, output_transfer, transform)
+      RactorUnorderedMapBoundary.new(upstream, workers, input_transfer, output_transfer, transform)
+    end
+
     def self.select(upstream, predicate)
       Select.new(upstream, predicate)
     end
 
+    def self.reject(upstream, predicate)
+      Reject.new(upstream, predicate)
+    end
+
     def self.take(upstream, count)
       Take.new(upstream, count)
     end
@@ -133,6 +157,10 @@ module FiberStream
       BufferBoundary.new(upstream, count)
     end
 
+    def self.throttle(upstream, limiter)
+      Throttle.new(upstream, limiter)
+    end
+
     def self.lines(upstream, chomp, max_length)
       Lines.new(upstream, chomp, max_length)
     end
@@ -154,7 +182,12 @@ require_relative "pull/concat"
 require_relative "pull/zip"
 require_relative "pull/merge"
 require_relative "pull/map"
+require_relative "pull/filter_map"
+require_relative "pull/compact"
+require_relative "pull/map_concat"
+require_relative "pull/tap"
 require_relative "pull/select"
+require_relative "pull/reject"
 require_relative "pull/take"
 require_relative "pull/drop"
 require_relative "pull/grouped"
@@ -165,15 +198,18 @@ require_relative "pull/lines"
 require_relative "pull/split"
 require_relative "pull/async_boundary"
 require_relative "pull/buffer_boundary"
+require_relative "pull/throttle"
 require_relative "pull/parallel_map_boundary"
 require_relative "pull/parallel_unordered_map_boundary"
 require_relative "pull/ractor_map_boundary"
+require_relative "pull/ractor_unordered_map_boundary"
 
 module FiberStream
   module Pull
     private_constant :Each, :IOSource, :RactorPortSource, :RactorMergePortsSource, :RactorProducerSource, :Concat,
-                     :Zip, :Merge, :Map, :Select, :Take, :Drop, :Grouped, :Scan, :TakeWhile, :DropWhile, :Lines, :Split,
-                     :AsyncBoundary, :BufferBoundary, :ParallelMapBoundary, :ParallelUnorderedMapBoundary,
-                     :RactorMapBoundary
+                     :Zip, :Merge, :Map, :FilterMap, :Compact, :MapConcat, :Tap, :Select, :Reject, :Take, :Drop,
+                     :Grouped, :Scan, :TakeWhile, :DropWhile, :Lines, :Split, :AsyncBoundary, :BufferBoundary,
+                     :Throttle, :ParallelMapBoundary, :ParallelUnorderedMapBoundary, :RactorMapBoundary,
+                     :RactorUnorderedMapBoundary
   end
 end

data/lib/fiber_stream/rate_limiter.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module FiberStream
+  # Scheduler-aware token-bucket rate limiter.
+  #
+  # `rate` permits refill every `per` seconds. `burst` is the maximum token
+  # capacity and defaults to `rate`. Immediate permit grants do not require a
+  # scheduler. When FiberStream must sleep, the current fiber must be
+  # non-blocking with an installed `Fiber.scheduler`.
+  class RateLimiter
+    Request = Data.define(:rate, :per, :burst, :permits, :now)
+
+    class << self
+      def validate_options!(rate:, per: 1, burst: nil) # :nodoc:
+        rate = validate_rate(rate)
+        validate_duration(:per, per)
+        validate_burst(burst.nil? ? rate : burst)
+        nil
+      end
+
+      private
+
+      def validate_rate(rate)
+        raise TypeError, "rate must be an Integer" unless rate.is_a?(Integer)
+        raise ArgumentError, "rate must be positive" unless rate.positive?
+
+        rate
+      end
+
+      def validate_burst(burst)
+        raise TypeError, "burst must be an Integer" unless burst.is_a?(Integer)
+        raise ArgumentError, "burst must be positive" unless burst.positive?
+
+        burst
+      end
+
+      def validate_duration(name, duration)
+        raise TypeError, "#{name} must be Numeric" unless duration.is_a?(Numeric)
+        raise ArgumentError, "#{name} must be finite and real" unless finite_real?(duration)
+        raise ArgumentError, "#{name} must be positive" unless duration.positive?
+
+        duration
+      end
+
+      def finite_real?(value)
+        return false if value.is_a?(Complex)
+        return value.finite? if value.respond_to?(:finite?)
+
+        true
+      end
+    end
+
+    def initialize(rate:, per: 1, burst: nil, &block)
+      self.class.validate_options!(rate:, per:, burst:)
+
+      @rate = rate
+      @per = per
+      @burst = burst.nil? ? @rate : burst
+      @policy = block
+      @mutex = Mutex.new
+      @tokens = @burst.to_f
+      @updated_at = monotonic_now
+    end
+
+    # Acquires `permits`, waiting when necessary.
+    #
+    # Waits are scheduler-backed and non-blocking. Requests larger than `burst`
+    # are rejected because the local token bucket could never satisfy them.
+    def acquire(permits: 1)
+      permits = validate_permits(permits)
+
+      if @policy
+        acquire_with_policy(permits)
+      else
+        acquire_with_token_bucket(permits)
+      end
+
+      nil
+    end
+
+    private
+
+    def acquire_with_policy(permits)
+      loop do
+        wait = normalize_policy_wait(@policy.call(request_for(permits)))
+        return if wait <= 0
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def acquire_with_token_bucket(permits)
+      loop do
+        wait = nil
+
+        @mutex.synchronize do
+          refill_tokens
+
+          if @tokens >= permits
+            @tokens -= permits
+            return
+          end
+
+          wait = (permits - @tokens) / permits_per_second
+        end
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def request_for(permits)
+      Request.new(rate: @rate, per: @per, burst: @burst, permits:, now: monotonic_now)
+    end
+
+    def refill_tokens
+      now = monotonic_now
+      elapsed = now - @updated_at
+      @updated_at = now
+      @tokens = [@burst, @tokens + (elapsed * permits_per_second)].min
+    end
+
+    def permits_per_second
+      @rate.to_f / @per.to_f
+    end
+
+    def scheduler_sleep(duration)
+      validate_scheduler!
+      sleep(duration)
+    end
+
+    def validate_scheduler!
+      return if Fiber.scheduler && !Fiber.current.blocking?
+
+      message =
+        if Fiber.scheduler
+          "RateLimiter#acquire requires a non-blocking fiber"
+        else
+          "RateLimiter#acquire requires Fiber.scheduler"
+        end
+      raise SchedulerRequiredError, message
+    end
+
+    def validate_permits(permits)
+      raise TypeError, "permits must be an Integer" unless permits.is_a?(Integer)
+      raise ArgumentError, "permits must be positive" unless permits.positive?
+      raise ArgumentError, "permits must be less than or equal to burst" if permits > @burst
+
+      permits
+    end
+
+    def normalize_policy_wait(wait)
+      return 0 if wait.nil?
+      raise TypeError, "custom wait duration must be Numeric or nil" unless wait.is_a?(Numeric)
+      raise ArgumentError, "custom wait duration must be finite and real" unless self.class.send(:finite_real?, wait)
+
+      wait.positive? ? wait : 0
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/fiber_stream/sink.rb

@@ -29,6 +29,74 @@ module FiberStream
       end
     end
 
+    # Creates a sink that returns the first element matching a predicate.
+    #
+    # The sink pulls upstream until the block returns a truthy value or
+    # upstream completes. It returns the original matching element, or `nil`
+    # when no element matches. Matching `nil` elements are returned as `nil`,
+    # following Ruby's `Enumerable#find` ambiguity.
+    def self.find(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new do |stream|
+        loop do
+          value = stream.next
+          break nil if Pull.done?(value)
+          break value if block.call(value)
+        end
+      end
+    end
+
+    # Creates a sink that returns whether any element matches a predicate.
+    #
+    # The sink pulls upstream until the block returns a truthy value or upstream
+    # completes. It returns `true` after a truthy predicate result and `false`
+    # when no element matches.
+    def self.any?(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new do |stream|
+        loop do
+          value = stream.next
+          break false if Pull.done?(value)
+          break true if block.call(value)
+        end
+      end
+    end
+
+    # Creates a sink that returns whether all elements match a predicate.
+    #
+    # The sink pulls upstream until the block returns false or nil, or upstream
+    # completes. It returns `false` after a falsey predicate result and `true`
+    # when every predicate result is truthy, including for empty upstream.
+    def self.all?(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new do |stream|
+        loop do
+          value = stream.next
+          break true if Pull.done?(value)
+          break false unless block.call(value)
+        end
+      end
+    end
+
+    # Creates a sink that counts all stream elements.
+    #
+    # The sink consumes upstream until normal completion and returns the number
+    # of elements observed. It does not store consumed elements.
+    def self.count
+      new do |stream|
+        count = 0
+
+        Pull.each_value(stream) do
+          count += 1
+        end
+
+        count
+      end
+    end
+
     # Creates a sink that folds all stream elements into an accumulator.
     #
     # The sink consumes upstream until normal completion. It returns the final

data/lib/fiber_stream/source.rb

@@ -178,6 +178,35 @@ module FiberStream
       via(Flow.map(&block))
     end
 
+    # Returns a new source definition that emits truthy transformed values.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.filter_map { ... })` and has the same falsey-drop,
+    # lazy construction, error, and backpressure behavior as the underlying
+    # flow.
+    def filter_map(&block)
+      via(Flow.filter_map(&block))
+    end
+
+    # Returns a new source definition that drops nil elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.compact)` and
+    # preserves the same nil-only filtering, lazy construction, and
+    # backpressure behavior as the underlying flow.
+    def compact
+      via(Flow.compact)
+    end
+
+    # Returns a new source definition that emits each mapped expansion.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.map_concat { ... })` and has the same one-level
+    # flattening, lazy construction, error, and backpressure behavior as the
+    # underlying flow.
+    def map_concat(&block)
+      via(Flow.map_concat(&block))
+    end
+
     # Returns a new source definition that maps elements concurrently.
     #
     # This is a convenience wrapper around
@@ -217,6 +246,25 @@ module FiberStream
       )
     end
 
+    # Returns a new source definition that maps elements in Ractor workers and
+    # emits mapped values as workers complete.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.ractor_unordered_map(workers:) { ... })` and
+    # preserves the same shareable mapper requirement, unordered delivery,
+    # transfer policy, bounded upstream run-ahead, and cooperative worker
+    # shutdown behavior.
+    def ractor_unordered_map(workers:, input_transfer: :copy, output_transfer: :copy, &block)
+      via(
+        Flow.ractor_unordered_map(
+          workers: workers,
+          input_transfer: input_transfer,
+          output_transfer: output_transfer,
+          &block
+        )
+      )
+    end
+
     # Returns a new source definition that keeps elements matching `block`.
     #
     # This is a convenience wrapper around
@@ -226,6 +274,15 @@ module FiberStream
       via(Flow.select(&block))
     end
 
+    # Returns a new source definition that drops elements matching `block`.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.reject { ... })` and has the same truthiness and
+    # lazy construction behavior as the underlying flow.
+    def reject(&block)
+      via(Flow.reject(&block))
+    end
+
     # Returns a new source definition that emits at most `count` elements.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.take(count))`
@@ -297,6 +354,15 @@ module FiberStream
       via(Flow.buffer(count))
     end
 
+    # Returns a new source definition that rate-limits emitted elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.throttle(...))`.
+    # The `rate:` form creates a fresh default limiter for each materialization;
+    # pass `limiter:` to share quota state across sources or runs.
+    def throttle(**options)
+      via(Flow.throttle(**options))
+    end
+
     # Returns a new source definition that splits String chunks into lines.
     #
     # This is a convenience wrapper around

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/fiber_stream.rb

@@ -3,6 +3,7 @@
 require_relative "fiber_stream/pull"
 require_relative "fiber_stream/version"
 require_relative "fiber_stream/errors"
+require_relative "fiber_stream/rate_limiter"
 require_relative "fiber_stream/internal/ractor_transfer_policy"
 require_relative "fiber_stream/ractor_port"
 require_relative "fiber_stream/ractor_producer"

data/sig/fiber_stream.rbs

@@ -14,6 +14,19 @@ module FiberStream
   class PipelineCancelledError < RuntimeError
   end
 
+  class RateLimiter
+    class Request < Data
+      attr_reader rate: Integer
+      attr_reader per: Numeric
+      attr_reader burst: Integer
+      attr_reader permits: Integer
+      attr_reader now: Float
+    end
+
+    def initialize: (rate: Integer, ?per: Numeric, ?burst: Integer?) ?{ (Request request) -> Numeric? } -> void
+    def acquire: (?permits: Integer) -> nil
+  end
+
   class RactorPortSourceError < RuntimeError
     attr_reader kind: ractor_port_source_error_kind
     attr_reader cause_class_name: String
@@ -74,7 +87,12 @@ module FiberStream
     def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def parallel_unordered_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
+    def ractor_unordered_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
+    def filter_map: [Out] () { (Elem) -> (Out | false | nil) } -> Source[Out]
+    def compact: () -> Source[Elem]
+    def map_concat: [Out] () { (Elem) -> Enumerable[Out] } -> Source[Out]
     def select: () { (Elem) -> boolish } -> Source[Elem]
+    def reject: () { (Elem) -> boolish } -> Source[Elem]
     def take: (Integer count) -> Source[Elem]
     def drop: (Integer count) -> Source[Elem]
     def grouped: (Integer count) -> Source[Array[Elem]]
@@ -83,6 +101,7 @@ module FiberStream
     def drop_while: () { (Elem) -> boolish } -> Source[Elem]
     def async: () -> Source[Elem]
     def buffer: (Integer count) -> Source[Elem]
+    def throttle: (?rate: Integer, ?per: Numeric, ?burst: Integer?, ?limiter: untyped) -> Source[Elem]
     def lines: (?chomp: bool, ?max_length: Integer?) -> Source[String]
     def split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Source[String]
     def to: [Mat] (Sink[Elem, Mat] sink) -> Pipeline[Mat]
@@ -91,10 +110,16 @@ module FiberStream
 
   class Flow[In, Out]
     def self.map: [In, Out] () { (In) -> Out } -> Flow[In, Out]
+    def self.tap: [Elem] () { (Elem) -> void } -> Flow[Elem, Elem]
     def self.parallel_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
     def self.parallel_unordered_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
     def self.ractor_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
+    def self.ractor_unordered_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
+    def self.filter_map: [In, Out] () { (In) -> (Out | false | nil) } -> Flow[In, Out]
+    def self.compact: [Elem] () -> Flow[Elem, Elem]
+    def self.map_concat: [In, Out] () { (In) -> Enumerable[Out] } -> Flow[In, Out]
     def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
+    def self.reject: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.drop: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.grouped: [Elem] (Integer count) -> Flow[Elem, Array[Elem]]
@@ -103,6 +128,7 @@ module FiberStream
     def self.drop_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.async: [Elem] () -> Flow[Elem, Elem]
     def self.buffer: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.throttle: [Elem] (?rate: Integer, ?per: Numeric, ?burst: Integer?, ?limiter: untyped) -> Flow[Elem, Elem]
     def self.lines: (?chomp: bool, ?max_length: Integer?) -> Flow[String, String]
     def self.split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Flow[String, String]
     def via: [Next] (Flow[Out, Next] flow) -> Flow[In, Next]
@@ -112,6 +138,10 @@ module FiberStream
   class Sink[In, Mat]
     def self.to_a: [Elem] () -> Sink[Elem, Array[Elem]]
     def self.first: [Elem] () -> Sink[Elem, Elem?]
+    def self.find: [Elem] () { (Elem) -> boolish } -> Sink[Elem, Elem?]
+    def self.any?: [Elem] () { (Elem) -> boolish } -> Sink[Elem, bool]
+    def self.all?: [Elem] () { (Elem) -> boolish } -> Sink[Elem, bool]
+    def self.count: [Elem] () -> Sink[Elem, Integer]
     def self.fold: [Elem, Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Sink[Elem, Acc]
     def self.foreach: [Elem] () { (Elem) -> void } -> Sink[Elem, Integer]
     def self.io: (untyped io, ?close: bool, ?flush: bool) -> Sink[String, Integer]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_stream
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Dai Akatsuka
@@ -120,6 +120,7 @@ files:
 - examples/ractor_merge_ports_and_map.rb
 - examples/ractor_port_source.rb
 - examples/ractor_producer_sources.rb
+- examples/ractor_unordered_map_hashing.rb
 - lib/fiber_stream.rb
 - lib/fiber_stream/errors.rb
 - lib/fiber_stream/flow.rb
@@ -128,14 +129,17 @@ files:
 - lib/fiber_stream/pull.rb
 - lib/fiber_stream/pull/async_boundary.rb
 - lib/fiber_stream/pull/buffer_boundary.rb
+- lib/fiber_stream/pull/compact.rb
 - lib/fiber_stream/pull/concat.rb
 - lib/fiber_stream/pull/drop.rb
 - lib/fiber_stream/pull/drop_while.rb
 - lib/fiber_stream/pull/each.rb
+- lib/fiber_stream/pull/filter_map.rb
 - lib/fiber_stream/pull/grouped.rb
 - lib/fiber_stream/pull/io_source.rb
 - lib/fiber_stream/pull/lines.rb
 - lib/fiber_stream/pull/map.rb
+- lib/fiber_stream/pull/map_concat.rb
 - lib/fiber_stream/pull/merge.rb
 - lib/fiber_stream/pull/parallel_map_boundary.rb
 - lib/fiber_stream/pull/parallel_unordered_map_boundary.rb
@@ -143,14 +147,19 @@ files:
 - lib/fiber_stream/pull/ractor_merge_ports_source.rb
 - lib/fiber_stream/pull/ractor_port_source.rb
 - lib/fiber_stream/pull/ractor_producer_source.rb
+- lib/fiber_stream/pull/ractor_unordered_map_boundary.rb
+- lib/fiber_stream/pull/reject.rb
 - lib/fiber_stream/pull/scan.rb
 - lib/fiber_stream/pull/select.rb
 - lib/fiber_stream/pull/split.rb
 - lib/fiber_stream/pull/take.rb
 - lib/fiber_stream/pull/take_while.rb
+- lib/fiber_stream/pull/tap.rb
+- lib/fiber_stream/pull/throttle.rb
 - lib/fiber_stream/pull/zip.rb
 - lib/fiber_stream/ractor_port.rb
 - lib/fiber_stream/ractor_producer.rb
+- lib/fiber_stream/rate_limiter.rb
 - lib/fiber_stream/running_pipeline.rb
 - lib/fiber_stream/sink.rb
 - lib/fiber_stream/source.rb
@@ -162,7 +171,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/dakatsuka/fiber_stream
-  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.4.0
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.6.0
   changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []