fiber_stream 0.2.0 → 0.4.0

data/lib/fiber_stream/flow.rb

@@ -30,6 +30,23 @@ module FiberStream
       new { |upstream| Pull.parallel_map(upstream, concurrency, block) }
     end
 
+    # Creates an unordered scheduler-backed parallel mapping flow.
+    #
+    # The stage starts internal scheduled fibers on first downstream demand and
+    # requires an installed `Fiber.scheduler` in a non-blocking fiber at that
+    # point. At most `concurrency` mapping blocks run at the same time, and at
+    # most `concurrency` upstream elements are pulled but not yet emitted downstream.
+    # Results are emitted in completion order and input order is not preserved.
+    # Closing the boundary closes upstream and requests internal worker
+    # cancellation. FiberStream does not depend on Async at runtime.
+    def self.parallel_unordered_map(concurrency:, &block)
+      raise ArgumentError, "missing block" unless block
+      raise TypeError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
+      raise ArgumentError, "concurrency must be positive" unless concurrency.positive?
+
+      new { |upstream| Pull.parallel_unordered_map(upstream, concurrency, block) }
+    end
+
     # Creates an ordered Ractor-backed mapping flow.
     #
     # The mapper runs inside worker ractors and must be shareable, typically
@@ -42,8 +59,8 @@ module FiberStream
       raise TypeError, "workers must be an Integer" unless workers.is_a?(Integer)
       raise ArgumentError, "workers must be positive" unless workers.positive?
 
-      validate_ractor_transfer_policy!(:input_transfer, input_transfer)
-      validate_ractor_transfer_policy!(:output_transfer, output_transfer)
+      Internal::RactorTransferPolicy.validate!(:input_transfer, input_transfer)
+      Internal::RactorTransferPolicy.validate!(:output_transfer, output_transfer)
       raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
 
       new { |upstream| Pull.ractor_map(upstream, workers, input_transfer, output_transfer, block) }
@@ -87,6 +104,31 @@ module FiberStream
       new { |upstream| Pull.drop(upstream, count) }
     end
 
+    # Creates a fixed-size grouping flow.
+    #
+    # The flow emits arrays containing up to `count` adjacent upstream elements.
+    # Full groups contain exactly `count` elements; normal upstream completion
+    # emits one final partial group when one exists. `count` must be a positive
+    # Integer.
+    def self.grouped(count)
+      raise TypeError, "count must be an Integer" unless count.is_a?(Integer)
+      raise ArgumentError, "count must be positive" unless count.positive?
+
+      new { |upstream| Pull.grouped(upstream, count) }
+    end
+
+    # Creates a running-accumulator flow.
+    #
+    # The block is called as `block.call(accumulator, element)` for each
+    # upstream element, matching `Sink.fold`. The block result becomes the new
+    # accumulator and is emitted downstream. The initial accumulator is not
+    # emitted before the first upstream element.
+    def self.scan(initial, &block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.scan(upstream, initial, block) }
+    end
+
     # Creates a predicate-based limiting flow.
     #
     # The flow emits leading elements while the block result is truthy. The
@@ -142,7 +184,9 @@ module FiberStream
     #
     # The flow accepts String chunks and emits lines split on "\n". By default
     # it chomps the trailing newline and one preceding "\r". `max_length` is an
-    # optional per-line bytesize limit.
+    # optional per-line bytesize limit. With `max_length: nil`, one
+    # unterminated line can buffer without bound. Set a positive `max_length`
+    # for untrusted, network-facing, or otherwise unbounded streams.
     def self.lines(chomp: true, max_length: nil)
       raise TypeError, "chomp must be true or false" unless [true, false].include?(chomp)
       unless max_length.nil? || max_length.is_a?(Integer)
@@ -153,13 +197,30 @@ module FiberStream
       new { |upstream| Pull.lines(upstream, chomp, max_length) }
     end
 
-    def self.validate_ractor_transfer_policy!(name, value)
-      return if [:copy, :move].include?(value)
+    # Creates a delimiter-splitting flow.
+    #
+    # The flow accepts String chunks and emits frames split on the non-empty
+    # String `separator`. Separator matching is byte-oriented. By default
+    # emitted frames exclude the separator; `keep_separator: true` preserves it
+    # on separator-terminated frames. `max_length` is an optional per-frame body
+    # bytesize limit. With `max_length: nil`, one unterminated frame can buffer
+    # without bound. Set a positive `max_length` for untrusted, network-facing,
+    # or otherwise unbounded streams.
+    def self.split(separator, keep_separator: false, max_length: nil)
+      raise TypeError, "separator must be String" unless separator.is_a?(String)
+      raise ArgumentError, "separator must not be empty" if separator.empty?
+      raise TypeError, "keep_separator must be true or false" unless [true, false].include?(keep_separator)
+      unless max_length.nil? || max_length.is_a?(Integer)
+        raise TypeError, "max_length must be nil or an Integer"
+      end
+      raise ArgumentError, "max_length must be positive" if max_length&.<= 0
 
-      raise ArgumentError, "#{name} must be :copy or :move"
+      new { |upstream| Pull.split(upstream, separator, keep_separator, max_length) }
     end
 
-    private_class_method :validate_ractor_transfer_policy!
+    def self.build(&attach) # :nodoc:
+      new(&attach)
+    end
 
     # Returns a reusable flow that applies this flow and then `flow`.
     #
@@ -168,11 +229,11 @@ module FiberStream
     def via(flow)
       raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
 
-      self.class.__send__(:new) do |upstream|
-        attached_stream = attach(upstream)
+      self.class.build do |upstream|
+        attached_stream = attach_to(upstream)
 
         begin
-          flow.__send__(:attach, attached_stream)
+          flow.attach_to(attached_stream)
         rescue StandardError
           begin
             attached_stream.close
@@ -192,13 +253,13 @@ module FiberStream
     def to(sink)
       raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
 
-      Sink.__send__(:new) do |stream|
+      Sink.build do |stream|
         attached_stream = nil
         primary_error = nil
 
         begin
-          attached_stream = attach(stream)
-          sink.__send__(:run, attached_stream)
+          attached_stream = attach_to(stream)
+          sink.run_stream(attached_stream)
         rescue StandardError => error
           primary_error = error
           raise
@@ -218,9 +279,7 @@ module FiberStream
 
     private_class_method :new
 
-    private
-
-    def attach(upstream)
+    def attach_to(upstream) # :nodoc:
       @attach.call(upstream)
     end
   end

data/lib/fiber_stream/internal/ractor_transfer_policy.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Internal # :nodoc:
+    module RactorTransferPolicy # :nodoc:
+      module_function
+
+      def validate!(name, value)
+        return if [:copy, :move].include?(value)
+
+        raise ArgumentError, "#{name} must be :copy or :move"
+      end
+    end
+  end
+
+  private_constant :Internal
+end

data/lib/fiber_stream/pipeline.rb

@@ -2,6 +2,10 @@
 
 module FiberStream
   class Pipeline
+    def self.build(source, sink) # :nodoc:
+      new(source, sink)
+    end
+
     def initialize(source, sink)
       @source = source
       @sink = sink
@@ -27,7 +31,7 @@ module FiberStream
     def run_async
       validate_scheduler!
 
-      RunningPipeline.__send__(:new, Fiber.scheduler) { run }
+      RunningPipeline.start(Fiber.scheduler) { run }
     end
 
     private_class_method :new

data/lib/fiber_stream/pull/async_boundary.rb

@@ -9,12 +9,18 @@ module FiberStream
     # time back to the downstream caller, so it adds an async boundary without
     # adding prefetch.
     class AsyncBoundary
+      ValueMessage = Data.define(:value)
+      DoneMessage = Data.define
+      ErrorMessage = Data.define(:error)
+      private_constant :ValueMessage, :DoneMessage, :ErrorMessage
+
       def initialize(upstream)
         @upstream = upstream
         @producer = nil
         @started = false
         @closed = false
         @done = false
+        @upstream_closed = false
       end
 
       def next
@@ -23,14 +29,14 @@ module FiberStream
         start
         message = @producer.resume
 
-        case message.fetch(0)
-        when :value
-          message.fetch(1)
-        when :done
+        case message
+        in ValueMessage[value:]
+          value
+        in DoneMessage
           complete
-        when :error
+        in ErrorMessage[error:]
           @done = true
-          raise message.fetch(1)
+          raise error
         end
       end
 
@@ -39,7 +45,7 @@ module FiberStream
 
         @closed = true
         @done = true
-        @upstream.close
+        close_upstream
       ensure
         cancel_producer
       end
@@ -60,16 +66,16 @@ module FiberStream
 
           value = @upstream.next
           if Pull.done?(value)
-            Fiber.yield([:done])
+            Fiber.yield(DoneMessage.new)
             break
           end
 
-          Fiber.yield([:value, value])
+          Fiber.yield(ValueMessage.new(value:))
         end
       rescue StandardError => exception
-        Fiber.yield([:error, exception]) unless @closed
+        Fiber.yield(ErrorMessage.new(error: exception)) unless @closed
       ensure
-        @upstream.close
+        close_upstream
       end
 
       def complete
@@ -77,7 +83,18 @@ module FiberStream
         DONE
       end
 
+      def close_upstream
+        return if @upstream_closed
+
+        @upstream_closed = true
+        @upstream.close
+      end
+
       def cancel_producer
+        return unless @producer&.alive?
+
+        @producer.kill
+      rescue StandardError
         nil
       end
     end

data/lib/fiber_stream/pull/buffer_boundary.rb

@@ -9,10 +9,17 @@ module FiberStream
     # queue capacity plus in-flight producer/consumer work. Close is responsible
     # for closing upstream and waking any producer blocked on a full queue.
     class BufferBoundary
+      CancellationError = Class.new(StandardError)
+      ValueMessage = Data.define(:value)
+      DoneMessage = Data.define
+      ErrorMessage = Data.define(:error)
+      private_constant :CancellationError, :ValueMessage, :DoneMessage, :ErrorMessage
+
       def initialize(upstream, count)
         @upstream = upstream
         @queue = Thread::SizedQueue.new(count)
         @producer = nil
+        @scheduler = nil
         @started = false
         @closed = false
         @done = false
@@ -27,14 +34,14 @@ module FiberStream
         message = @queue.pop
         return complete if message.nil?
 
-        case message.fetch(0)
-        when :value
-          message.fetch(1)
-        when :done
+        case message
+        in ValueMessage[value:]
+          value
+        in DoneMessage
           complete
-        when :error
+        in ErrorMessage[error:]
           @done = true
-          raise message.fetch(1)
+          raise error
         end
       end
 
@@ -58,7 +65,9 @@ module FiberStream
         raise SchedulerRequiredError, "Flow.buffer requires Fiber.scheduler" unless Fiber.scheduler
 
         @started = true
+        @scheduler = Fiber.scheduler
         @producer = Fiber.schedule { run_producer }
+        cancel_producer if @closed
       end
 
       def run_producer
@@ -67,8 +76,10 @@ module FiberStream
 
           message = pull_message
           break unless deliver(message)
-          break unless message.fetch(0) == :value
+          break unless message.is_a?(ValueMessage)
         end
+      rescue CancellationError
+        nil
       ensure
         @upstream_close_error ||= close_upstream unless @upstream_closed
       end
@@ -77,15 +88,17 @@ module FiberStream
         value = @upstream.next
         return terminal_done_message if Pull.done?(value)
 
-        [:value, value]
+        ValueMessage.new(value:)
+      rescue CancellationError
+        raise
       rescue StandardError => error
         close_upstream(record_error: false)
-        [:error, error]
+        ErrorMessage.new(error:)
       end
 
       def terminal_done_message
         close_error = close_upstream
-        close_error ? [:error, close_error] : [:done]
+        close_error ? ErrorMessage.new(error: close_error) : DoneMessage.new
       end
 
       def deliver(message)
@@ -116,6 +129,11 @@ module FiberStream
       end
 
       def cancel_producer
+        return unless @producer&.alive?
+        return unless @scheduler.respond_to?(:fiber_interrupt)
+
+        @scheduler.fiber_interrupt(@producer, CancellationError.new)
+      rescue NotImplementedError, StandardError
         nil
       end
     end

data/lib/fiber_stream/pull/concat.rb

@@ -8,7 +8,7 @@ module FiberStream
       def initialize(left_materializer, right_materializer)
         @left_materializer = left_materializer
         @right_materializer = right_materializer
-        @left = @left_materializer.call
+        @left = nil
         @right = nil
         @phase = :left
         @closed = false
@@ -38,6 +38,7 @@ module FiberStream
       private
 
       def next_left
+        materialize_left
         value = @left.next
         return value unless Pull.done?(value)
 
@@ -56,6 +57,13 @@ module FiberStream
         DONE
       end
 
+      def materialize_left
+        return if @left
+
+        stream = @left_materializer.call
+        @left = stream
+      end
+
       def close_left
         stream = @left
         return unless stream

data/lib/fiber_stream/pull/grouped.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Fixed-size grouping stage.
+    #
+    # It collects adjacent upstream elements into distinct arrays of up to
+    # `count` elements. A final partial group is emitted when upstream completes
+    # normally.
+    class Grouped
+      def initialize(upstream, count)
+        @upstream = upstream
+        @count = count
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        group = []
+
+        while group.length < @count
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE if group.empty?
+
+            return group
+          end
+
+          group << value
+        end
+
+        group
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/merge.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Scheduler-backed ready-order source merge.
+    #
+    # Each input source is materialized by a scheduled producer fiber on first
+    # downstream demand. Producers publish values, completion, and failures into
+    # a bounded mailbox; downstream emits values in mailbox arrival order while
+    # preserving each input's own order.
+    class Merge
+      SIDE_ORDER = [:left, :right].freeze
+      CancellationError = Class.new(StandardError)
+      ValueMessage = Data.define(:side, :value)
+      DoneMessage = Data.define(:side)
+      ErrorMessage = Data.define(:side, :error)
+      private_constant :ValueMessage, :DoneMessage, :ErrorMessage
+
+      def initialize(left_materializer, right_materializer)
+        @materializers = { left: left_materializer, right: right_materializer }
+        @streams = { left: nil, right: nil }
+        @stream_closed = { left: false, right: false }
+        @side_done = { left: false, right: false }
+        @producers = {}
+        @mailbox = nil
+        @started = false
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        start
+        next_message
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @done = true
+        close_error = close_materialized_streams
+        close_mailbox
+        raise close_error if close_error
+      ensure
+        cancel_producers
+      end
+
+      private
+
+      def start
+        return if @started
+
+        validate_scheduler!
+
+        @mailbox = MergeMailbox.new(1)
+        @started = true
+        SIDE_ORDER.each do |side|
+          @producers[side] = Fiber.schedule { run_producer(side) }
+        end
+      end
+
+      def next_message
+        loop do
+          message = @mailbox.pop
+          return complete if message.nil?
+
+          case message
+          in ValueMessage[value:]
+            return value
+          in DoneMessage[side:]
+            mark_side_done(side)
+            return complete if all_done?
+          in ErrorMessage[error:]
+            return fail_with(error)
+          end
+        end
+      rescue MergeMailbox::Closed
+        complete
+      end
+
+      def run_producer(side)
+        stream = materialize_side(side)
+
+        loop do
+          break if @closed
+
+          message = pull_message(side, stream)
+          break unless deliver(message)
+          break unless message.is_a?(ValueMessage)
+        end
+      rescue MergeMailbox::Closed, CancellationError
+        nil
+      rescue StandardError => error
+        close_side(side, record_error: false)
+        deliver(ErrorMessage.new(side:, error:)) unless @closed
+      end
+
+      def materialize_side(side)
+        stream = @materializers.fetch(side).call
+        @streams[side] = stream
+        close_side(side) if @closed
+        stream
+      end
+
+      def pull_message(side, stream)
+        value = stream.next
+        return terminal_done_message(side) if Pull.done?(value)
+
+        ValueMessage.new(side:, value:)
+      rescue StandardError => error
+        close_side(side, record_error: false)
+        ErrorMessage.new(side:, error:)
+      end
+
+      def terminal_done_message(side)
+        close_error = close_side(side)
+        close_error ? ErrorMessage.new(side:, error: close_error) : DoneMessage.new(side:)
+      end
+
+      def deliver(message)
+        @mailbox.push(message)
+        true
+      rescue MergeMailbox::Closed
+        false
+      end
+
+      def mark_side_done(side)
+        @side_done[side] = true
+      end
+
+      def all_done?
+        SIDE_ORDER.all? { |side| @side_done.fetch(side) }
+      end
+
+      def complete
+        @done = true
+        close_mailbox
+        DONE
+      end
+
+      def fail_with(error)
+        @done = true
+        close_mailbox
+        close_materialized_streams
+        cancel_producers
+        raise error
+      end
+
+      def close_materialized_streams
+        first_error = nil
+
+        SIDE_ORDER.each do |side|
+          close_error = close_side(side)
+          first_error ||= close_error
+        end
+
+        first_error
+      end
+
+      def close_side(side, record_error: true)
+        return nil if @stream_closed.fetch(side)
+
+        stream = @streams[side]
+        return nil unless stream
+
+        @stream_closed[side] = true
+        @streams[side] = nil
+        stream.close
+        nil
+      rescue StandardError => error
+        error if record_error
+      end
+
+      def close_mailbox
+        @mailbox&.close
+      end
+
+      def cancel_producers
+        scheduler = Fiber.scheduler
+        return unless scheduler.respond_to?(:fiber_interrupt)
+
+        @producers.each_value do |fiber|
+          next unless fiber&.alive?
+
+          scheduler.fiber_interrupt(fiber, CancellationError.new)
+        rescue StandardError
+          nil
+        end
+      end
+
+      def validate_scheduler!
+        return if Fiber.scheduler && !Fiber.current.blocking?
+
+        message =
+          if Fiber.scheduler
+            "Source.merge requires a non-blocking fiber"
+          else
+            "Source.merge requires Fiber.scheduler"
+          end
+        raise SchedulerRequiredError, message
+      end
+
+      class MergeMailbox
+        Closed = Class.new(StandardError)
+
+        def initialize(capacity)
+          @queue = Thread::SizedQueue.new(capacity)
+        end
+
+        def push(message)
+          @queue << message
+        rescue ClosedQueueError
+          raise Closed
+        end
+
+        def pop
+          @queue.pop
+        rescue ClosedQueueError
+          raise Closed
+        end
+
+        def close
+          @queue.close
+        end
+      end
+    end
+  end
+end