fiber_stream 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc93666d0610e659313a12dc756fca935579e62711972a9ea65d9ad818f6020
-  data.tar.gz: 97f315765ba573a5c047752fd083006db89404eff06e92bda5afbfa0f1933ed1
+  metadata.gz: 504a7400182e09bb66a5a07981bbb2cfab350397d09c23ea036e856f29afe1d2
+  data.tar.gz: 67c7ddead203d42bf869dcba7d840e7d09514d888b90b84b77d97a8cedc19e1e
 SHA512:
-  metadata.gz: 5e635531f9e34510ef0eab76254c33e70f36124be779f48d2f9692c351a2f26ed36c7c14fa0d1d596c5645b511bb4fa1abe2bb773fc434c1f4da39a6e19dbefc
-  data.tar.gz: 67eb50ae0a1d727c65fd172be572f7bf08ba82b3ee0cf4d9094c7e9ad579aac327155cc772bcd4e94b778cef53f63decf64a08e12e7ea5ef6949a6d813336b04
+  metadata.gz: f9c8d80faca53d9c6e059326bc9e6ffe7497aa8a6cbcad3b42eefe0ef3a3d8d29f08a3afbec8fc2b02b2064a745bb00ea644e816f913aa624a2ad11f1b087b99
+  data.tar.gz: 9e6f87b74cbfe2ccaaa9c32990f9786af3ee3ee3d7f2c3dd605a88efa8f3e436e4a1051d65c1eb1664859af212a2fecc82c66dc585cf9333dd799917c2a9035e

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.5.0 - 2026-06-21
+
+### Added
+
+- `Flow.tap { |element| ... }` for lazy pass-through observation without
+  changing emitted elements.
+- `Flow.filter_map { |element| ... }` and `Source#filter_map` for combined
+  transformation and falsey-value dropping.
+- `Flow.reject { |element| ... }` and `Source#reject` for complement
+  predicate filtering.
+- `Flow.compact` and `Source#compact` for nil-only filtering while preserving
+  `false`.
+- `Flow.map_concat { |element| enumerable }` and `Source#map_concat` for
+  one-to-many element expansion.
+- `Flow.throttle(...)`, `Source#throttle(...)`, and `RateLimiter` for
+  pull-driven rate limiting with optional shared quota state.
+- `Sink.count` for counting stream elements without accumulating them.
+
+### Changed
+
+- Expanded README and website reference coverage for the flow operators and
+  rate limiter added in 0.5.0.
+- Promoted completed flow product specs and design docs from draft to accepted
+  status.
+
 ## 0.4.0 - 2026-06-09
 
 ### Added

data/README.md

@@ -1,7 +1,10 @@
 # FiberStream
- FiberStream is a Ruby library for linear stream processing with pull-based backpressure. 
 
-It builds lazy Source definitions, transforms values with Flow stages, and materializes results with Sink objects.
+FiberStream is a Ruby library for linear stream processing with pull-based
+backpressure.
+
+It builds lazy `Source` definitions, transforms values with `Flow` stages, and
+materializes results with `Sink` objects.
 
 [![Gem Version](https://badge.fury.io/rb/fiber_stream.svg)](https://badge.fury.io/rb/fiber_stream)
 
@@ -30,11 +33,12 @@ Implemented capabilities:
 - in-memory, IO, FiberStream-owned Ractor producer, backpressure-aware Ractor
   port, and Ractor port merge sources
 - lazy source concatenation, zipping, and scheduler-backed merging
-- mapping, filtering, limiting, predicate-based limiting and dropping,
-  fixed-prefix dropping, fixed-size grouping, line splitting, buffering, async
-  boundaries, ordered and unordered parallel mapping, and ordered
-  Ractor-backed mapping
-- array, first-element, fold, foreach, and IO sinks
+- mapping, filtering, transform-and-filter, nil compaction, side-effect
+  observation, one-to-many expansion, limiting, predicate-based limiting and
+  dropping, fixed-prefix dropping, fixed-size grouping, line splitting,
+  buffering, async boundaries, throttling, ordered and unordered parallel
+  mapping, and ordered Ractor-backed mapping
+- array, first-element, count, fold, foreach, and IO sinks
 - reusable flow composition and runnable pipelines
 - foreground and scheduler-backed background pipeline execution
 - public RBS signatures
@@ -205,6 +209,62 @@ FiberStream::Source.each([" a ", "", " b "])
 # => ["a", "b"]
 ```
 
+Use `reject` when the predicate names values to drop. Truthy predicate results
+drop the original element; `false` and `nil` results pass it through unchanged:
+
+```ruby
+result =
+  FiberStream::Source.each([1, 2, 3, 4])
+    .reject(&:even?)
+    .run_with(FiberStream::Sink.to_a)
+
+result # => [1, 3]
+```
+
+Use `filter_map` when filtering and transformation are one decision. Truthy
+block results are emitted as transformed values; `false` and `nil` are
+dropped:
+
+```ruby
+ids =
+  FiberStream::Source.each([{ id: 1 }, {}, { id: 3 }])
+    .filter_map { |record| record[:id] }
+    .run_with(FiberStream::Sink.to_a)
+
+ids # => [1, 3]
+```
+
+Use `compact` to drop only `nil` while keeping `false`, and `map_concat` to
+expand one upstream element into zero or more downstream elements:
+
+```ruby
+tokens =
+  FiberStream::Source.each(["alpha beta", nil, "gamma"])
+    .compact
+    .map_concat { |line| line.split }
+    .run_with(FiberStream::Sink.to_a)
+
+tokens # => ["alpha", "beta", "gamma"]
+```
+
+Use `Flow.tap` for observation inside a reusable flow without changing the
+element:
+
+```ruby
+seen = []
+
+observed =
+  FiberStream::Flow.tap { |value| seen << value }
+    .via(FiberStream::Flow.map { |value| value * 10 })
+
+FiberStream::Source.each([1, 2])
+  .via(observed)
+  .run_with(FiberStream::Sink.to_a)
+# => [10, 20]
+
+seen # => [1, 2]
+```
+
 Use `parallel_map` for ordered scheduler-backed mapping when each element
 waits on non-blocking IO. It preserves input order while allowing up to
 `concurrency` mapping operations to be in flight:
@@ -293,6 +353,14 @@ FiberStream::Source.each([1, 2, 3])
 # => 6
 ```
 
+Use `Sink.count` when only the number of elements matters:
+
+```ruby
+FiberStream::Source.each([1, 2, 3])
+  .run_with(FiberStream::Sink.count)
+# => 3
+```
+
 Use `Sink.foreach` when the terminal operation is a side effect and the stream
 values should not be accumulated:
 
@@ -473,12 +541,15 @@ does not provide CPU parallelism. Use producer ractors with
 `Source.ractor_producer` or `Source.ractor_merge_producers` when producer work
 needs true isolation.
 
-`Flow.buffer(count)` allows bounded prefetch. `Flow.async`, `Flow.buffer`,
+`Flow.buffer(count)` allows bounded prefetch. `Flow.throttle(rate:, per:)`
+paces elements before downstream side effects. `Flow.async`, `Flow.buffer`,
 `Flow.parallel_map`, `Flow.parallel_unordered_map`, `Source.io`,
 `Source#merge`, `Sink.io`, and `Pipeline#run_async` require an installed
 `Fiber.scheduler` and a non-blocking current fiber when demanded or started.
-FiberStream does not install a scheduler and does not depend on Async at
-runtime.
+`Flow.throttle` requires that scheduler context only when it needs to wait.
+Pass `throttle(limiter:)` with a `FiberStream::RateLimiter` when multiple
+pipelines or repeated runs should share quota state. FiberStream does not
+install a scheduler and does not depend on Async at runtime.
 
 ## API Surface
 
@@ -498,10 +569,14 @@ Source convenience methods:
 - `Source#zip(source)`
 - `Source#merge(source)`
 - `Source#map { |element| ... }`
+- `Source#filter_map { |element| ... }`
+- `Source#compact`
+- `Source#map_concat { |element| enumerable }`
 - `Source#parallel_map(concurrency:) { |element| ... }`
 - `Source#parallel_unordered_map(concurrency:) { |element| ... }`
 - `Source#ractor_map(workers:, input_transfer: :copy, output_transfer: :copy) { |element| ... }`
 - `Source#select { |element| ... }`
+- `Source#reject { |element| ... }`
 - `Source#take(count)`
 - `Source#drop(count)`
 - `Source#grouped(count)`
@@ -510,6 +585,8 @@ Source convenience methods:
 - `Source#drop_while { |element| ... }`
 - `Source#async`
 - `Source#buffer(count)`
+- `Source#throttle(rate:, per: 1, burst: nil)`
+- `Source#throttle(limiter:)`
 - `Source#lines(chomp: true, max_length: nil)`
 - `Source#split(separator, keep_separator: false, max_length: nil)`
 - `Source#to(sink)`
@@ -518,10 +595,15 @@ Source convenience methods:
 Flows:
 
 - `FiberStream::Flow.map { |element| ... }`
+- `FiberStream::Flow.filter_map { |element| ... }`
+- `FiberStream::Flow.compact`
+- `FiberStream::Flow.map_concat { |element| enumerable }`
+- `FiberStream::Flow.tap { |element| ... }`
 - `FiberStream::Flow.parallel_map(concurrency:) { |element| ... }`
 - `FiberStream::Flow.parallel_unordered_map(concurrency:) { |element| ... }`
 - `FiberStream::Flow.ractor_map(workers:, input_transfer: :copy, output_transfer: :copy) { |element| ... }`
 - `FiberStream::Flow.select { |element| ... }`
+- `FiberStream::Flow.reject { |element| ... }`
 - `FiberStream::Flow.take(count)`
 - `FiberStream::Flow.drop(count)`
 - `FiberStream::Flow.grouped(count)`
@@ -530,10 +612,14 @@ Flows:
 - `FiberStream::Flow.drop_while { |element| ... }`
 - `FiberStream::Flow.async`
 - `FiberStream::Flow.buffer(count)`
+- `FiberStream::Flow.throttle(rate:, per: 1, burst: nil)`
+- `FiberStream::Flow.throttle(limiter:)`
 - `FiberStream::Flow.lines(chomp: true, max_length: nil)`
 - `FiberStream::Flow.split(separator, keep_separator: false, max_length: nil)`
 - `Flow#via(flow)`
 - `Flow#to(sink)`
+- `FiberStream::RateLimiter.new(rate:, per: 1, burst: nil)`
+- `FiberStream::RateLimiter#acquire(permits: 1)`
 
 `lines` and `split` default to `max_length: nil`, which allows one
 unterminated line or frame to buffer without bound. Set a positive
@@ -543,6 +629,7 @@ Sinks:
 
 - `FiberStream::Sink.to_a`
 - `FiberStream::Sink.first`
+- `FiberStream::Sink.count`
 - `FiberStream::Sink.fold(initial) { |accumulator, element| ... }`
 - `FiberStream::Sink.foreach { |element| ... }`
 - `FiberStream::Sink.io(io, close: false, flush: false)`

data/lib/fiber_stream/flow.rb

@@ -13,6 +13,51 @@ module FiberStream
       new { |upstream| Pull.map(upstream, block) }
     end
 
+    # Creates a transform-and-filter flow.
+    #
+    # The block is called once for each upstream element observed by this
+    # stage. Truthy block results are emitted downstream as transformed values;
+    # false and nil results are dropped. Exceptions raised by the block fail the
+    # stream and are re-raised from `Source#run_with`.
+    def self.filter_map(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.filter_map(upstream, block) }
+    end
+
+    # Creates a nil-dropping flow.
+    #
+    # The flow drops `nil` elements and passes every non-`nil` element through
+    # unchanged, including `false`.
+    def self.compact
+      new { |upstream| Pull.compact(upstream) }
+    end
+
+    # Creates a one-to-many mapping flow.
+    #
+    # The block is called once for each upstream element whose expansion is
+    # needed. It must return an object that responds to `#each`; yielded values
+    # are emitted in order before the next upstream element is pulled.
+    # Exceptions raised by the block or by the returned object's `#each` fail
+    # the stream and are re-raised from `Source#run_with`.
+    def self.map_concat(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.map_concat(upstream, block) }
+    end
+
+    # Creates a pass-through observing flow.
+    #
+    # The block is called once for each element before that element is emitted
+    # downstream. The block return value is ignored and the original element is
+    # passed through unchanged. Exceptions raised by the block fail the stream
+    # and are re-raised from `Source#run_with`.
+    def self.tap(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.tap(upstream, block) }
+    end
+
     # Creates an ordered scheduler-backed parallel mapping flow.
     #
     # The stage starts internal scheduled fibers on first downstream demand and
@@ -78,6 +123,19 @@ module FiberStream
       new { |upstream| Pull.select(upstream, block) }
     end
 
+    # Creates a complement filtering flow.
+    #
+    # The block is called for upstream elements until it returns `false` or
+    # `nil`, or upstream completes. Truthy predicate results drop the original
+    # element; false and nil results pass the element through unchanged.
+    # Exceptions raised by the block fail the stream and are re-raised from
+    # `Source#run_with`.
+    def self.reject(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.reject(upstream, block) }
+    end
+
     # Creates a limiting flow.
     #
     # The flow emits at most `count` elements. `take(0)` completes without
@@ -180,6 +238,19 @@ module FiberStream
       new { |upstream| Pull.buffer(upstream, count) }
     end
 
+    # Creates a scheduler-aware throttling flow.
+    #
+    # The `rate:` form creates a fresh `RateLimiter` for each materialization.
+    # The `limiter:` form uses the supplied limiter object, which must respond
+    # to `acquire(permits:)` and return only after permits are acquired. When
+    # FiberStream-owned waiting is required, the current fiber must be
+    # non-blocking with an installed `Fiber.scheduler`.
+    def self.throttle(**options)
+      limiter = build_throttle_limiter(options)
+
+      new { |upstream| Pull.throttle(upstream, limiter.call) }
+    end
+
     # Creates a line-splitting flow.
     #
     # The flow accepts String chunks and emits lines split on "\n". By default
@@ -222,6 +293,38 @@ module FiberStream
       new(&attach)
     end
 
+    def self.build_throttle_limiter(options)
+      unknown_keywords = options.keys - [:rate, :per, :burst, :limiter]
+      raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" unless unknown_keywords.empty?
+
+      rate_given = options.key?(:rate)
+      per_given = options.key?(:per)
+      burst_given = options.key?(:burst)
+      limiter_given = options.key?(:limiter)
+
+      if limiter_given
+        raise ArgumentError, "cannot pass rate and limiter together" if rate_given
+        raise ArgumentError, "cannot pass per with limiter" if per_given
+        raise ArgumentError, "cannot pass burst with limiter" if burst_given
+
+        limiter = options.fetch(:limiter)
+        raise TypeError, "limiter must respond to acquire" unless limiter.respond_to?(:acquire)
+
+        return -> { limiter }
+      end
+
+      raise ArgumentError, "missing rate or limiter" unless rate_given
+
+      rate = options.fetch(:rate)
+      per = options.fetch(:per, 1)
+      burst = options.fetch(:burst, nil)
+      RateLimiter.validate_options!(rate:, per:, burst:)
+
+      -> { RateLimiter.new(rate:, per:, burst:) }
+    end
+
+    private_class_method :build_throttle_limiter
+
     # Returns a reusable flow that applies this flow and then `flow`.
     #
     # Construction is lazy. No upstream stream is attached and no elements are

data/lib/fiber_stream/pull/compact.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Nil-dropping stage.
+    #
+    # A single downstream demand may pull multiple upstream elements until a
+    # non-nil value is observed or upstream completes. Dropped nil values are
+    # discarded immediately and are not buffered.
+    class Compact
+      def initialize(upstream)
+        @upstream = upstream
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          return value unless value.nil?
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/filter_map.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Transform-and-filter stage.
+    #
+    # A single downstream demand may pull multiple upstream elements until the
+    # transform returns a truthy value or upstream completes. Falsey transform
+    # results are discarded immediately and are not buffered.
+    class FilterMap
+      def initialize(upstream, transform)
+        @upstream = upstream
+        @transform = transform
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          result = @transform.call(value)
+          return result if result
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/map_concat.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # One-to-many mapping stage.
+    #
+    # It expands one upstream element into the values yielded by one returned
+    # `#each` object. Only one expansion is active at a time, and the stage
+    # never pulls the next upstream element until the active expansion is
+    # exhausted.
+    class MapConcat
+      def initialize(upstream, transform)
+        @upstream = upstream
+        @transform = transform
+        @current_enumerator = nil
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          if @current_enumerator
+            begin
+              return @current_enumerator.next
+            rescue StopIteration
+              @current_enumerator = nil
+            end
+          end
+
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          result = @transform.call(value)
+          unless result.respond_to?(:each)
+            raise TypeError, "map_concat block result must respond to each"
+          end
+
+          @current_enumerator = result.to_enum(:each)
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @current_enumerator = nil
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/reject.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Complement filtering stage.
+    #
+    # A single downstream demand may pull multiple upstream elements until the
+    # predicate retains a value or upstream completes. Rejected elements are
+    # discarded immediately and are not buffered.
+    class Reject
+      def initialize(upstream, predicate)
+        @upstream = upstream
+        @predicate = predicate
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          return value unless @predicate.call(value)
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/tap.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Stateless observing stage.
+    #
+    # It pulls one upstream element for each downstream demand, calls the
+    # observer for real elements, and emits the original element unchanged.
+    class Tap
+      def initialize(upstream, observer)
+        @upstream = upstream
+        @observer = observer
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        value = @upstream.next
+        if Pull.done?(value)
+          @done = true
+          return DONE
+        end
+
+        @observer.call(value)
+        value
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/throttle.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Pull-driven rate-limiting stage.
+    #
+    # The stage pulls at most one upstream value, acquires one permit, and then
+    # emits that value unless the stage was closed while waiting.
+    class Throttle
+      def initialize(upstream, limiter)
+        @upstream = upstream
+        @limiter = limiter
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        value = @upstream.next
+        if Pull.done?(value)
+          @done = true
+          return DONE
+        end
+
+        @limiter.acquire(permits: 1)
+        if @closed
+          @done = true
+          return DONE
+        end
+
+        value
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull.rb

@@ -85,6 +85,22 @@ module FiberStream
       Map.new(upstream, transform)
     end
 
+    def self.filter_map(upstream, transform)
+      FilterMap.new(upstream, transform)
+    end
+
+    def self.compact(upstream)
+      Compact.new(upstream)
+    end
+
+    def self.map_concat(upstream, transform)
+      MapConcat.new(upstream, transform)
+    end
+
+    def self.tap(upstream, observer)
+      Tap.new(upstream, observer)
+    end
+
     def self.parallel_map(upstream, concurrency, transform)
       ParallelMapBoundary.new(upstream, concurrency, transform)
     end
@@ -101,6 +117,10 @@ module FiberStream
       Select.new(upstream, predicate)
     end
 
+    def self.reject(upstream, predicate)
+      Reject.new(upstream, predicate)
+    end
+
     def self.take(upstream, count)
       Take.new(upstream, count)
     end
@@ -133,6 +153,10 @@ module FiberStream
       BufferBoundary.new(upstream, count)
     end
 
+    def self.throttle(upstream, limiter)
+      Throttle.new(upstream, limiter)
+    end
+
     def self.lines(upstream, chomp, max_length)
       Lines.new(upstream, chomp, max_length)
     end
@@ -154,7 +178,12 @@ require_relative "pull/concat"
 require_relative "pull/zip"
 require_relative "pull/merge"
 require_relative "pull/map"
+require_relative "pull/filter_map"
+require_relative "pull/compact"
+require_relative "pull/map_concat"
+require_relative "pull/tap"
 require_relative "pull/select"
+require_relative "pull/reject"
 require_relative "pull/take"
 require_relative "pull/drop"
 require_relative "pull/grouped"
@@ -165,6 +194,7 @@ require_relative "pull/lines"
 require_relative "pull/split"
 require_relative "pull/async_boundary"
 require_relative "pull/buffer_boundary"
+require_relative "pull/throttle"
 require_relative "pull/parallel_map_boundary"
 require_relative "pull/parallel_unordered_map_boundary"
 require_relative "pull/ractor_map_boundary"
@@ -172,8 +202,8 @@ require_relative "pull/ractor_map_boundary"
 module FiberStream
   module Pull
     private_constant :Each, :IOSource, :RactorPortSource, :RactorMergePortsSource, :RactorProducerSource, :Concat,
-                     :Zip, :Merge, :Map, :Select, :Take, :Drop, :Grouped, :Scan, :TakeWhile, :DropWhile, :Lines, :Split,
-                     :AsyncBoundary, :BufferBoundary, :ParallelMapBoundary, :ParallelUnorderedMapBoundary,
-                     :RactorMapBoundary
+                     :Zip, :Merge, :Map, :FilterMap, :Compact, :MapConcat, :Tap, :Select, :Reject, :Take, :Drop,
+                     :Grouped, :Scan, :TakeWhile, :DropWhile, :Lines, :Split, :AsyncBoundary, :BufferBoundary,
+                     :Throttle, :ParallelMapBoundary, :ParallelUnorderedMapBoundary, :RactorMapBoundary
   end
 end

data/lib/fiber_stream/rate_limiter.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module FiberStream
+  # Scheduler-aware token-bucket rate limiter.
+  #
+  # `rate` permits refill every `per` seconds. `burst` is the maximum token
+  # capacity and defaults to `rate`. Immediate permit grants do not require a
+  # scheduler. When FiberStream must sleep, the current fiber must be
+  # non-blocking with an installed `Fiber.scheduler`.
+  class RateLimiter
+    Request = Data.define(:rate, :per, :burst, :permits, :now)
+
+    class << self
+      def validate_options!(rate:, per: 1, burst: nil) # :nodoc:
+        rate = validate_rate(rate)
+        validate_duration(:per, per)
+        validate_burst(burst.nil? ? rate : burst)
+        nil
+      end
+
+      private
+
+      def validate_rate(rate)
+        raise TypeError, "rate must be an Integer" unless rate.is_a?(Integer)
+        raise ArgumentError, "rate must be positive" unless rate.positive?
+
+        rate
+      end
+
+      def validate_burst(burst)
+        raise TypeError, "burst must be an Integer" unless burst.is_a?(Integer)
+        raise ArgumentError, "burst must be positive" unless burst.positive?
+
+        burst
+      end
+
+      def validate_duration(name, duration)
+        raise TypeError, "#{name} must be Numeric" unless duration.is_a?(Numeric)
+        raise ArgumentError, "#{name} must be finite and real" unless finite_real?(duration)
+        raise ArgumentError, "#{name} must be positive" unless duration.positive?
+
+        duration
+      end
+
+      def finite_real?(value)
+        return false if value.is_a?(Complex)
+        return value.finite? if value.respond_to?(:finite?)
+
+        true
+      end
+    end
+
+    def initialize(rate:, per: 1, burst: nil, &block)
+      self.class.validate_options!(rate:, per:, burst:)
+
+      @rate = rate
+      @per = per
+      @burst = burst.nil? ? @rate : burst
+      @policy = block
+      @mutex = Mutex.new
+      @tokens = @burst.to_f
+      @updated_at = monotonic_now
+    end
+
+    # Acquires `permits`, waiting when necessary.
+    #
+    # Waits are scheduler-backed and non-blocking. Requests larger than `burst`
+    # are rejected because the local token bucket could never satisfy them.
+    def acquire(permits: 1)
+      permits = validate_permits(permits)
+
+      if @policy
+        acquire_with_policy(permits)
+      else
+        acquire_with_token_bucket(permits)
+      end
+
+      nil
+    end
+
+    private
+
+    def acquire_with_policy(permits)
+      loop do
+        wait = normalize_policy_wait(@policy.call(request_for(permits)))
+        return if wait <= 0
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def acquire_with_token_bucket(permits)
+      loop do
+        wait = nil
+
+        @mutex.synchronize do
+          refill_tokens
+
+          if @tokens >= permits
+            @tokens -= permits
+            return
+          end
+
+          wait = (permits - @tokens) / permits_per_second
+        end
+
+        scheduler_sleep(wait)
+      end
+    end
+
+    def request_for(permits)
+      Request.new(rate: @rate, per: @per, burst: @burst, permits:, now: monotonic_now)
+    end
+
+    def refill_tokens
+      now = monotonic_now
+      elapsed = now - @updated_at
+      @updated_at = now
+      @tokens = [@burst, @tokens + (elapsed * permits_per_second)].min
+    end
+
+    def permits_per_second
+      @rate.to_f / @per.to_f
+    end
+
+    def scheduler_sleep(duration)
+      validate_scheduler!
+      sleep(duration)
+    end
+
+    def validate_scheduler!
+      return if Fiber.scheduler && !Fiber.current.blocking?
+
+      message =
+        if Fiber.scheduler
+          "RateLimiter#acquire requires a non-blocking fiber"
+        else
+          "RateLimiter#acquire requires Fiber.scheduler"
+        end
+      raise SchedulerRequiredError, message
+    end
+
+    def validate_permits(permits)
+      raise TypeError, "permits must be an Integer" unless permits.is_a?(Integer)
+      raise ArgumentError, "permits must be positive" unless permits.positive?
+      raise ArgumentError, "permits must be less than or equal to burst" if permits > @burst
+
+      permits
+    end
+
+    def normalize_policy_wait(wait)
+      return 0 if wait.nil?
+      raise TypeError, "custom wait duration must be Numeric or nil" unless wait.is_a?(Numeric)
+      raise ArgumentError, "custom wait duration must be finite and real" unless self.class.send(:finite_real?, wait)
+
+      wait.positive? ? wait : 0
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/fiber_stream/sink.rb

@@ -29,6 +29,22 @@ module FiberStream
       end
     end
 
+    # Creates a sink that counts all stream elements.
+    #
+    # The sink consumes upstream until normal completion and returns the number
+    # of elements observed. It does not store consumed elements.
+    def self.count
+      new do |stream|
+        count = 0
+
+        Pull.each_value(stream) do
+          count += 1
+        end
+
+        count
+      end
+    end
+
     # Creates a sink that folds all stream elements into an accumulator.
     #
     # The sink consumes upstream until normal completion. It returns the final

data/lib/fiber_stream/source.rb

@@ -178,6 +178,35 @@ module FiberStream
       via(Flow.map(&block))
     end
 
+    # Returns a new source definition that emits truthy transformed values.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.filter_map { ... })` and has the same falsey-drop,
+    # lazy construction, error, and backpressure behavior as the underlying
+    # flow.
+    def filter_map(&block)
+      via(Flow.filter_map(&block))
+    end
+
+    # Returns a new source definition that drops nil elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.compact)` and
+    # preserves the same nil-only filtering, lazy construction, and
+    # backpressure behavior as the underlying flow.
+    def compact
+      via(Flow.compact)
+    end
+
+    # Returns a new source definition that emits each mapped expansion.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.map_concat { ... })` and has the same one-level
+    # flattening, lazy construction, error, and backpressure behavior as the
+    # underlying flow.
+    def map_concat(&block)
+      via(Flow.map_concat(&block))
+    end
+
     # Returns a new source definition that maps elements concurrently.
     #
     # This is a convenience wrapper around
@@ -226,6 +255,15 @@ module FiberStream
       via(Flow.select(&block))
     end
 
+    # Returns a new source definition that drops elements matching `block`.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.reject { ... })` and has the same truthiness and
+    # lazy construction behavior as the underlying flow.
+    def reject(&block)
+      via(Flow.reject(&block))
+    end
+
     # Returns a new source definition that emits at most `count` elements.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.take(count))`
@@ -297,6 +335,15 @@ module FiberStream
       via(Flow.buffer(count))
     end
 
+    # Returns a new source definition that rate-limits emitted elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.throttle(...))`.
+    # The `rate:` form creates a fresh default limiter for each materialization;
+    # pass `limiter:` to share quota state across sources or runs.
+    def throttle(**options)
+      via(Flow.throttle(**options))
+    end
+
     # Returns a new source definition that splits String chunks into lines.
     #
     # This is a convenience wrapper around

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/fiber_stream.rb

@@ -3,6 +3,7 @@
 require_relative "fiber_stream/pull"
 require_relative "fiber_stream/version"
 require_relative "fiber_stream/errors"
+require_relative "fiber_stream/rate_limiter"
 require_relative "fiber_stream/internal/ractor_transfer_policy"
 require_relative "fiber_stream/ractor_port"
 require_relative "fiber_stream/ractor_producer"

data/sig/fiber_stream.rbs

@@ -14,6 +14,19 @@ module FiberStream
   class PipelineCancelledError < RuntimeError
   end
 
+  class RateLimiter
+    class Request < Data
+      attr_reader rate: Integer
+      attr_reader per: Numeric
+      attr_reader burst: Integer
+      attr_reader permits: Integer
+      attr_reader now: Float
+    end
+
+    def initialize: (rate: Integer, ?per: Numeric, ?burst: Integer?) ?{ (Request request) -> Numeric? } -> void
+    def acquire: (?permits: Integer) -> nil
+  end
+
   class RactorPortSourceError < RuntimeError
     attr_reader kind: ractor_port_source_error_kind
     attr_reader cause_class_name: String
@@ -74,7 +87,11 @@ module FiberStream
     def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def parallel_unordered_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
+    def filter_map: [Out] () { (Elem) -> (Out | false | nil) } -> Source[Out]
+    def compact: () -> Source[Elem]
+    def map_concat: [Out] () { (Elem) -> Enumerable[Out] } -> Source[Out]
     def select: () { (Elem) -> boolish } -> Source[Elem]
+    def reject: () { (Elem) -> boolish } -> Source[Elem]
     def take: (Integer count) -> Source[Elem]
     def drop: (Integer count) -> Source[Elem]
     def grouped: (Integer count) -> Source[Array[Elem]]
@@ -83,6 +100,7 @@ module FiberStream
     def drop_while: () { (Elem) -> boolish } -> Source[Elem]
     def async: () -> Source[Elem]
     def buffer: (Integer count) -> Source[Elem]
+    def throttle: (?rate: Integer, ?per: Numeric, ?burst: Integer?, ?limiter: untyped) -> Source[Elem]
     def lines: (?chomp: bool, ?max_length: Integer?) -> Source[String]
     def split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Source[String]
     def to: [Mat] (Sink[Elem, Mat] sink) -> Pipeline[Mat]
@@ -91,10 +109,15 @@ module FiberStream
 
   class Flow[In, Out]
     def self.map: [In, Out] () { (In) -> Out } -> Flow[In, Out]
+    def self.tap: [Elem] () { (Elem) -> void } -> Flow[Elem, Elem]
     def self.parallel_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
     def self.parallel_unordered_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
     def self.ractor_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
+    def self.filter_map: [In, Out] () { (In) -> (Out | false | nil) } -> Flow[In, Out]
+    def self.compact: [Elem] () -> Flow[Elem, Elem]
+    def self.map_concat: [In, Out] () { (In) -> Enumerable[Out] } -> Flow[In, Out]
     def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
+    def self.reject: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.drop: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.grouped: [Elem] (Integer count) -> Flow[Elem, Array[Elem]]
@@ -103,6 +126,7 @@ module FiberStream
     def self.drop_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.async: [Elem] () -> Flow[Elem, Elem]
     def self.buffer: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.throttle: [Elem] (?rate: Integer, ?per: Numeric, ?burst: Integer?, ?limiter: untyped) -> Flow[Elem, Elem]
     def self.lines: (?chomp: bool, ?max_length: Integer?) -> Flow[String, String]
     def self.split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Flow[String, String]
     def via: [Next] (Flow[Out, Next] flow) -> Flow[In, Next]
@@ -112,6 +136,7 @@ module FiberStream
   class Sink[In, Mat]
     def self.to_a: [Elem] () -> Sink[Elem, Array[Elem]]
     def self.first: [Elem] () -> Sink[Elem, Elem?]
+    def self.count: [Elem] () -> Sink[Elem, Integer]
     def self.fold: [Elem, Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Sink[Elem, Acc]
     def self.foreach: [Elem] () { (Elem) -> void } -> Sink[Elem, Integer]
     def self.io: (untyped io, ?close: bool, ?flush: bool) -> Sink[String, Integer]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_stream
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Dai Akatsuka
@@ -128,14 +128,17 @@ files:
 - lib/fiber_stream/pull.rb
 - lib/fiber_stream/pull/async_boundary.rb
 - lib/fiber_stream/pull/buffer_boundary.rb
+- lib/fiber_stream/pull/compact.rb
 - lib/fiber_stream/pull/concat.rb
 - lib/fiber_stream/pull/drop.rb
 - lib/fiber_stream/pull/drop_while.rb
 - lib/fiber_stream/pull/each.rb
+- lib/fiber_stream/pull/filter_map.rb
 - lib/fiber_stream/pull/grouped.rb
 - lib/fiber_stream/pull/io_source.rb
 - lib/fiber_stream/pull/lines.rb
 - lib/fiber_stream/pull/map.rb
+- lib/fiber_stream/pull/map_concat.rb
 - lib/fiber_stream/pull/merge.rb
 - lib/fiber_stream/pull/parallel_map_boundary.rb
 - lib/fiber_stream/pull/parallel_unordered_map_boundary.rb
@@ -143,14 +146,18 @@ files:
 - lib/fiber_stream/pull/ractor_merge_ports_source.rb
 - lib/fiber_stream/pull/ractor_port_source.rb
 - lib/fiber_stream/pull/ractor_producer_source.rb
+- lib/fiber_stream/pull/reject.rb
 - lib/fiber_stream/pull/scan.rb
 - lib/fiber_stream/pull/select.rb
 - lib/fiber_stream/pull/split.rb
 - lib/fiber_stream/pull/take.rb
 - lib/fiber_stream/pull/take_while.rb
+- lib/fiber_stream/pull/tap.rb
+- lib/fiber_stream/pull/throttle.rb
 - lib/fiber_stream/pull/zip.rb
 - lib/fiber_stream/ractor_port.rb
 - lib/fiber_stream/ractor_producer.rb
+- lib/fiber_stream/rate_limiter.rb
 - lib/fiber_stream/running_pipeline.rb
 - lib/fiber_stream/sink.rb
 - lib/fiber_stream/source.rb
@@ -162,7 +169,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/dakatsuka/fiber_stream
-  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.4.0
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.5.0
   changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []