fiber_stream 0.1.0 → 0.3.0

data/examples/ractor_merge_ports_and_map.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "digest"
+require "fiber_stream"
+
+PRODUCER_JOBS = [
+  [
+    "producer-a",
+    [
+      { name: "alpha.bin", payload: +"A" * 180_000, seed_rounds: 80, verify_rounds: 60 },
+      { name: "bravo.bin", payload: +"B" * 140_000, seed_rounds: 70, verify_rounds: 55 }
+    ]
+  ],
+  [
+    "producer-b",
+    [
+      { name: "charlie.bin", payload: +"C" * 220_000, seed_rounds: 85, verify_rounds: 65 },
+      { name: "delta.bin", payload: +"D" * 120_000, seed_rounds: 75, verify_rounds: 50 }
+    ]
+  ]
+].freeze
+
+VERIFY_RECORD =
+  Ractor.shareable_proc do |record|
+    digest = record.fetch(:seed_sha256)
+
+    record.fetch(:verify_rounds).times do |index|
+      digest = Digest::SHA256.hexdigest("#{digest}:verify:#{index}")
+    end
+
+    record.merge(final_sha256: digest)
+  end
+
+def spawn_digest_producer(data_port, producer_name, jobs)
+  setup_port = Ractor::Port.new
+  producer =
+    Ractor.new(data_port, setup_port, producer_name, jobs) do |outbox, setup, name, producer_jobs|
+      ack_port = Ractor::Port.new
+      setup.send(ack_port)
+
+      enumerator = producer_jobs.to_enum
+      sent = 0
+
+      loop do
+        case ack_port.receive
+        in FiberStream::RactorPort::Ack
+          begin
+            job = enumerator.next
+            digest = job.fetch(:payload)
+
+            job.fetch(:seed_rounds).times do |index|
+              digest = Digest::SHA256.hexdigest("#{digest}:#{name}:#{index}")
+            end
+
+            sent += 1
+            outbox.send(
+              FiberStream::RactorPort::Element.new(
+                {
+                  producer: name,
+                  name: job.fetch(:name),
+                  bytes: job.fetch(:payload).bytesize,
+                  seed_sha256: digest,
+                  verify_rounds: job.fetch(:verify_rounds)
+                }
+              ),
+              move: true
+            )
+          rescue StopIteration
+            outbox.send(FiberStream::RactorPort::Complete.new)
+            break [:completed, name, sent]
+          end
+        in FiberStream::RactorPort::Cancel[reason]
+          break [:cancelled, name, sent, reason]
+        end
+      end
+    end
+
+  [producer, setup_port.receive]
+end
+
+port_pairs = []
+producers =
+  PRODUCER_JOBS.map do |producer_name, jobs|
+    data_port = Ractor::Port.new
+    producer, ack_port = spawn_digest_producer(data_port, producer_name, jobs)
+    port_pairs << { port: data_port, ack_port: ack_port }
+    producer
+  end
+
+records =
+  FiberStream::Source.ractor_merge_ports(port_pairs)
+    .ractor_map(workers: 2, input_transfer: :move, output_transfer: :move, &VERIFY_RECORD)
+    .run_with(FiberStream::Sink.to_a)
+
+puts "Merged producer Ractors, then verified in ractor_map workers"
+records.each do |record|
+  puts format(
+    "- %-10<producer>s %-11<name>s %7<bytes>d bytes  %<final_sha256>s",
+    producer: record.fetch(:producer),
+    name: record.fetch(:name),
+    bytes: record.fetch(:bytes),
+    final_sha256: record.fetch(:final_sha256)
+  )
+end
+
+puts
+puts "Producer statuses:"
+producers.each do |producer|
+  puts "- #{producer.value.inspect}"
+end
+
+puts
+puts "Source.ractor_merge_ports emits producers in ready order."
+puts "ractor_map preserves that merged input order while running verification in Ractor workers."

data/lib/fiber_stream/errors.rb

@@ -9,7 +9,10 @@ module FiberStream
   #
   # Producer failures, invalid protocol messages, and source-side Ractor port
   # failures use this stable error shape so callers do not need to depend on
-  # Ruby's Ractor transport exceptions.
+  # Ruby's Ractor transport exceptions. For producer failures,
+  # `cause_class_name` and `cause_message` come from the producer's
+  # `RactorPort::Failure` envelope and are included in this error's public
+  # message.
   class RactorPortSourceError < RuntimeError
     attr_reader :kind, :cause_class_name, :cause_message, :original_cause
 

data/lib/fiber_stream/flow.rb

@@ -75,6 +75,56 @@ module FiberStream
       new { |upstream| Pull.take(upstream, count) }
     end
 
+    # Creates a fixed-prefix dropping flow.
+    #
+    # The flow discards the first `count` upstream elements, then passes later
+    # elements through unchanged. `drop(0)` behaves as pass-through. Negative
+    # counts raise `ArgumentError`; non-Integer counts raise `TypeError`.
+    def self.drop(count)
+      raise TypeError, "count must be an Integer" unless count.is_a?(Integer)
+      raise ArgumentError, "count must be non-negative" if count.negative?
+
+      new { |upstream| Pull.drop(upstream, count) }
+    end
+
+    # Creates a fixed-size grouping flow.
+    #
+    # The flow emits arrays containing up to `count` adjacent upstream elements.
+    # Full groups contain exactly `count` elements; normal upstream completion
+    # emits one final partial group when one exists. `count` must be a positive
+    # Integer.
+    def self.grouped(count)
+      raise TypeError, "count must be an Integer" unless count.is_a?(Integer)
+      raise ArgumentError, "count must be positive" unless count.positive?
+
+      new { |upstream| Pull.grouped(upstream, count) }
+    end
+
+    # Creates a predicate-based limiting flow.
+    #
+    # The flow emits leading elements while the block result is truthy. The
+    # first false or nil result completes the stream without emitting that
+    # element and closes upstream during the same downstream pull. Exceptions
+    # raised by the block fail the stream and are re-raised from
+    # `Source#run_with`.
+    def self.take_while(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.take_while(upstream, block) }
+    end
+
+    # Creates a predicate-based prefix-dropping flow.
+    #
+    # The flow drops leading elements while the block result is truthy. The
+    # first false or nil result, and all later elements, pass through unchanged.
+    # After that boundary the block is not called again. Exceptions raised by
+    # the block fail the stream and are re-raised from `Source#run_with`.
+    def self.drop_while(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.drop_while(upstream, block) }
+    end
+
     # Creates a scheduler-backed asynchronous boundary.
     #
     # The boundary starts its producer on the first downstream demand and
@@ -105,7 +155,9 @@ module FiberStream
     #
     # The flow accepts String chunks and emits lines split on "\n". By default
     # it chomps the trailing newline and one preceding "\r". `max_length` is an
-    # optional per-line bytesize limit.
+    # optional per-line bytesize limit. With `max_length: nil`, one
+    # unterminated line can buffer without bound. Set a positive `max_length`
+    # for untrusted, network-facing, or otherwise unbounded streams.
     def self.lines(chomp: true, max_length: nil)
       raise TypeError, "chomp must be true or false" unless [true, false].include?(chomp)
       unless max_length.nil? || max_length.is_a?(Integer)
@@ -116,6 +168,27 @@ module FiberStream
       new { |upstream| Pull.lines(upstream, chomp, max_length) }
     end
 
+    # Creates a delimiter-splitting flow.
+    #
+    # The flow accepts String chunks and emits frames split on the non-empty
+    # String `separator`. Separator matching is byte-oriented. By default
+    # emitted frames exclude the separator; `keep_separator: true` preserves it
+    # on separator-terminated frames. `max_length` is an optional per-frame body
+    # bytesize limit. With `max_length: nil`, one unterminated frame can buffer
+    # without bound. Set a positive `max_length` for untrusted, network-facing,
+    # or otherwise unbounded streams.
+    def self.split(separator, keep_separator: false, max_length: nil)
+      raise TypeError, "separator must be String" unless separator.is_a?(String)
+      raise ArgumentError, "separator must not be empty" if separator.empty?
+      raise TypeError, "keep_separator must be true or false" unless [true, false].include?(keep_separator)
+      unless max_length.nil? || max_length.is_a?(Integer)
+        raise TypeError, "max_length must be nil or an Integer"
+      end
+      raise ArgumentError, "max_length must be positive" if max_length&.<= 0
+
+      new { |upstream| Pull.split(upstream, separator, keep_separator, max_length) }
+    end
+
     def self.validate_ractor_transfer_policy!(name, value)
       return if [:copy, :move].include?(value)
 

data/lib/fiber_stream/pull/async_boundary.rb

@@ -9,12 +9,18 @@ module FiberStream
     # time back to the downstream caller, so it adds an async boundary without
     # adding prefetch.
     class AsyncBoundary
+      ValueMessage = Data.define(:value)
+      DoneMessage = Data.define
+      ErrorMessage = Data.define(:error)
+      private_constant :ValueMessage, :DoneMessage, :ErrorMessage
+
       def initialize(upstream)
         @upstream = upstream
         @producer = nil
         @started = false
         @closed = false
         @done = false
+        @upstream_closed = false
       end
 
       def next
@@ -23,14 +29,14 @@ module FiberStream
         start
         message = @producer.resume
 
-        case message.fetch(0)
-        when :value
-          message.fetch(1)
-        when :done
+        case message
+        in ValueMessage[value:]
+          value
+        in DoneMessage
           complete
-        when :error
+        in ErrorMessage[error:]
           @done = true
-          raise message.fetch(1)
+          raise error
         end
       end
 
@@ -39,7 +45,7 @@ module FiberStream
 
         @closed = true
         @done = true
-        @upstream.close
+        close_upstream
       ensure
         cancel_producer
       end
@@ -60,16 +66,16 @@ module FiberStream
 
           value = @upstream.next
           if Pull.done?(value)
-            Fiber.yield([:done])
+            Fiber.yield(DoneMessage.new)
             break
           end
 
-          Fiber.yield([:value, value])
+          Fiber.yield(ValueMessage.new(value:))
         end
       rescue StandardError => exception
-        Fiber.yield([:error, exception]) unless @closed
+        Fiber.yield(ErrorMessage.new(error: exception)) unless @closed
       ensure
-        @upstream.close
+        close_upstream
       end
 
       def complete
@@ -77,7 +83,18 @@ module FiberStream
         DONE
       end
 
+      def close_upstream
+        return if @upstream_closed
+
+        @upstream_closed = true
+        @upstream.close
+      end
+
       def cancel_producer
+        return unless @producer&.alive?
+
+        @producer.kill
+      rescue StandardError
         nil
       end
     end

data/lib/fiber_stream/pull/buffer_boundary.rb

@@ -9,10 +9,17 @@ module FiberStream
     # queue capacity plus in-flight producer/consumer work. Close is responsible
     # for closing upstream and waking any producer blocked on a full queue.
     class BufferBoundary
+      CancellationError = Class.new(StandardError)
+      ValueMessage = Data.define(:value)
+      DoneMessage = Data.define
+      ErrorMessage = Data.define(:error)
+      private_constant :CancellationError, :ValueMessage, :DoneMessage, :ErrorMessage
+
       def initialize(upstream, count)
         @upstream = upstream
         @queue = Thread::SizedQueue.new(count)
         @producer = nil
+        @scheduler = nil
         @started = false
         @closed = false
         @done = false
@@ -27,14 +34,14 @@ module FiberStream
         message = @queue.pop
         return complete if message.nil?
 
-        case message.fetch(0)
-        when :value
-          message.fetch(1)
-        when :done
+        case message
+        in ValueMessage[value:]
+          value
+        in DoneMessage
           complete
-        when :error
+        in ErrorMessage[error:]
           @done = true
-          raise message.fetch(1)
+          raise error
         end
       end
 
@@ -58,7 +65,9 @@ module FiberStream
         raise SchedulerRequiredError, "Flow.buffer requires Fiber.scheduler" unless Fiber.scheduler
 
         @started = true
+        @scheduler = Fiber.scheduler
         @producer = Fiber.schedule { run_producer }
+        cancel_producer if @closed
       end
 
       def run_producer
@@ -67,8 +76,10 @@ module FiberStream
 
           message = pull_message
           break unless deliver(message)
-          break unless message.fetch(0) == :value
+          break unless message.is_a?(ValueMessage)
         end
+      rescue CancellationError
+        nil
       ensure
         @upstream_close_error ||= close_upstream unless @upstream_closed
       end
@@ -77,15 +88,17 @@ module FiberStream
         value = @upstream.next
         return terminal_done_message if Pull.done?(value)
 
-        [:value, value]
+        ValueMessage.new(value:)
+      rescue CancellationError
+        raise
       rescue StandardError => error
         close_upstream(record_error: false)
-        [:error, error]
+        ErrorMessage.new(error:)
       end
 
       def terminal_done_message
         close_error = close_upstream
-        close_error ? [:error, close_error] : [:done]
+        close_error ? ErrorMessage.new(error: close_error) : DoneMessage.new
       end
 
       def deliver(message)
@@ -116,6 +129,11 @@ module FiberStream
       end
 
       def cancel_producer
+        return unless @producer&.alive?
+        return unless @scheduler.respond_to?(:fiber_interrupt)
+
+        @scheduler.fiber_interrupt(@producer, CancellationError.new)
+      rescue NotImplementedError, StandardError
         nil
       end
     end

data/lib/fiber_stream/pull/concat.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Pull stream that emits all values from one materialized source, then all
+    # values from a second source materialized only after the first completes.
+    class Concat
+      def initialize(left_materializer, right_materializer)
+        @left_materializer = left_materializer
+        @right_materializer = right_materializer
+        @left = nil
+        @right = nil
+        @phase = :left
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        case @phase
+        when :left
+          next_left
+        when :right
+          next_right
+        else
+          DONE
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        close_materialized_streams
+      end
+
+      private
+
+      def next_left
+        materialize_left
+        value = @left.next
+        return value unless Pull.done?(value)
+
+        close_left
+        @phase = :right
+        @right = @right_materializer.call
+        next_right
+      end
+
+      def next_right
+        value = @right.next
+        return value unless Pull.done?(value)
+
+        close_right
+        @done = true
+        DONE
+      end
+
+      def materialize_left
+        return if @left
+
+        stream = @left_materializer.call
+        @left = stream
+      end
+
+      def close_left
+        stream = @left
+        return unless stream
+
+        stream.close
+        @left = nil
+      end
+
+      def close_right
+        stream = @right
+        return unless stream
+
+        stream.close
+        @right = nil
+      end
+
+      def close_materialized_streams
+        first_error = nil
+
+        [@right, @left].each do |stream|
+          next unless stream
+
+          begin
+            stream.close
+          rescue StandardError => error
+            first_error ||= error
+          end
+        end
+
+        @right = nil
+        @left = nil
+
+        raise first_error if first_error
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/drop.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Fixed-prefix dropping stage.
+    #
+    # It discards the first `count` upstream elements on downstream demand, then
+    # passes later elements through without buffering.
+    class Drop
+      def initialize(upstream, count)
+        @upstream = upstream
+        @remaining = count
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        drop_prefix
+        return DONE if @done
+
+        pull_retained_value
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+
+      private
+
+      def drop_prefix
+        while @remaining.positive?
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return
+          end
+
+          @remaining -= 1
+        end
+      end
+
+      def pull_retained_value
+        value = @upstream.next
+        if Pull.done?(value)
+          @done = true
+          return DONE
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/drop_while.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Predicate-based prefix dropping stage.
+    #
+    # It drops leading elements while the predicate is truthy. The first falsey
+    # element and all later elements pass through unchanged.
+    class DropWhile
+      def initialize(upstream, predicate)
+        @upstream = upstream
+        @predicate = predicate
+        @dropping = true
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        return pull_pass_through unless @dropping
+
+        pull_until_retained
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+
+      private
+
+      def pull_until_retained
+        loop do
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE
+          end
+
+          next if @predicate.call(value)
+
+          @dropping = false
+          return value
+        end
+      end
+
+      def pull_pass_through
+        value = @upstream.next
+        if Pull.done?(value)
+          @done = true
+          return DONE
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull/grouped.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Fixed-size grouping stage.
+    #
+    # It collects adjacent upstream elements into distinct arrays of up to
+    # `count` elements. A final partial group is emitted when upstream completes
+    # normally.
+    class Grouped
+      def initialize(upstream, count)
+        @upstream = upstream
+        @count = count
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        group = []
+
+        while group.length < @count
+          value = @upstream.next
+          if Pull.done?(value)
+            @done = true
+            return DONE if group.empty?
+
+            return group
+          end
+
+          group << value
+        end
+
+        group
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @upstream.close
+      end
+    end
+  end
+end