fiber_stream 0.2.0 → 0.3.0

data/lib/fiber_stream/pull/ractor_port_source.rb

@@ -9,7 +9,10 @@ module FiberStream
     # a coordinator thread so scheduler-managed fibers do not call Ractor wait
     # APIs directly.
     class RactorPortSource
-      WAIT_INTERVAL = 0.001
+      ProtocolMessage = Data.define(:message)
+      ErrorMessage = Data.define(:error)
+      ClosedMessage = Data.define
+      private_constant :ProtocolMessage, :ErrorMessage, :ClosedMessage
 
       def initialize(port, ack_port, ack_transfer, cancel)
         @port = port
@@ -81,15 +84,13 @@ module FiberStream
       end
 
       def handle_result(result)
-        tag = result.fetch(0)
-
-        case tag
-        when :message
-          handle_protocol_message(result.fetch(1))
-        when :error
+        case result
+        in ProtocolMessage[message:]
+          handle_protocol_message(message)
+        in ErrorMessage[error:]
           mark_done
-          raise_error(result.fetch(1))
-        when :closed
+          raise_error(error)
+        in ClosedMessage
           DONE
         end
       end
@@ -151,19 +152,19 @@ module FiberStream
 
           ack_error = send_ack
           if ack_error
-            deliver_result([:error, ack_error])
+            deliver_result(ErrorMessage.new(error: ack_error))
             break
           end
 
           selected, message = select_message
           break if selected == @shutdown_port || closed?
 
-          deliver_result([:message, message])
+          deliver_result(ProtocolMessage.new(message:))
         end
       rescue StandardError => error
-        deliver_result([:error, build_error(:receive, error)])
+        deliver_result(ErrorMessage.new(error: build_error(:receive, error)))
       ensure
-        deliver_result([:closed]) if closed?
+        deliver_result(ClosedMessage.new) if closed?
       end
 
       def select_message
@@ -218,7 +219,6 @@ module FiberStream
       def wait_for_coordinator
         return unless @coordinator
 
-        sleep WAIT_INTERVAL while @coordinator.alive?
         @coordinator.join
       end
 

data/lib/fiber_stream/pull/split.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # Delimiter-framing stage for `Flow.split`.
+    #
+    # The stage keeps an internal byte buffer because frames and separators can
+    # cross chunk boundaries. Length checks are per frame body, not against the
+    # aggregate buffer, so already complete valid frames can be emitted before a
+    # later over-limit frame fails.
+    class Split
+      def initialize(upstream, separator, keep_separator, max_length)
+        @upstream = upstream
+        @separator = separator.b.freeze
+        @keep_separator = keep_separator
+        @max_length = max_length
+        @buffer = +"".b
+        @closed = false
+        @upstream_done = false
+      end
+
+      def next
+        return DONE if @closed
+
+        loop do
+          frame = next_buffered_frame
+          return frame if frame
+
+          validate_pending_frame_length!
+          return complete_from_buffer if @upstream_done
+
+          append_next_chunk
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @buffer.clear
+        @upstream.close
+      end
+
+      private
+
+      def next_buffered_frame
+        separator_index = @buffer.index(@separator)
+        return nil unless separator_index
+
+        frame = @buffer.slice!(0, separator_index)
+        @buffer.slice!(0, @separator.bytesize)
+        validate_frame_length!(frame)
+        format_frame(frame)
+      end
+
+      def complete_from_buffer
+        return DONE if @buffer.empty?
+
+        frame = @buffer
+        @buffer = +"".b
+        validate_frame_length!(frame)
+        frame
+      end
+
+      def append_next_chunk
+        chunk = @upstream.next
+        if Pull.done?(chunk)
+          @upstream_done = true
+          return
+        end
+
+        unless chunk.is_a?(String)
+          raise TypeError, "Flow.split elements must be String"
+        end
+
+        @buffer << chunk.b
+        validate_pending_frame_length!
+      end
+
+      def validate_pending_frame_length!
+        return unless @max_length
+        return if pending_frame_body_bytesize <= @max_length
+
+        fail_frame_too_long
+      end
+
+      def validate_frame_length!(frame)
+        return unless @max_length
+        return if frame.bytesize <= @max_length
+
+        fail_frame_too_long
+      end
+
+      def pending_frame_body_bytesize
+        separator_index = @buffer.index(@separator)
+        return separator_index if separator_index
+
+        @buffer.bytesize - partial_separator_suffix_bytesize
+      end
+
+      def partial_separator_suffix_bytesize
+        max_suffix_bytesize = [@separator.bytesize - 1, @buffer.bytesize].min
+        return 0 if max_suffix_bytesize.zero?
+
+        max_suffix_bytesize.downto(1) do |bytesize|
+          suffix = @buffer.byteslice(@buffer.bytesize - bytesize, bytesize)
+          return bytesize if @separator.start_with?(suffix)
+        end
+
+        0
+      end
+
+      def fail_frame_too_long
+        @closed = true
+        close_upstream
+        error = FrameTooLongError.new("frame exceeded max_length #{@max_length}")
+        raise error
+      end
+
+      def close_upstream
+        @upstream.close
+        nil
+      rescue StandardError => error
+        error
+      end
+
+      def format_frame(frame)
+        return frame unless @keep_separator
+
+        frame + @separator
+      end
+    end
+  end
+end

data/lib/fiber_stream/pull.rb

@@ -28,6 +28,10 @@ module FiberStream
       RactorPortSource.new(port, ack_port, ack_transfer, cancel)
     end
 
+    def self.ractor_merge_ports(port_pairs, ack_transfer, cancel)
+      RactorMergePortsSource.new(port_pairs, ack_transfer, cancel)
+    end
+
     def self.concat(left_materializer, right_materializer)
       Concat.new(left_materializer, right_materializer)
     end
@@ -36,6 +40,10 @@ module FiberStream
       Zip.new(left_materializer, right_materializer)
     end
 
+    def self.merge(left_materializer, right_materializer)
+      Merge.new(left_materializer, right_materializer)
+    end
+
     def self.map(upstream, transform)
       Map.new(upstream, transform)
     end
@@ -60,6 +68,10 @@ module FiberStream
       Drop.new(upstream, count)
     end
 
+    def self.grouped(upstream, count)
+      Grouped.new(upstream, count)
+    end
+
     def self.take_while(upstream, predicate)
       TakeWhile.new(upstream, predicate)
     end
@@ -80,6 +92,10 @@ module FiberStream
       Lines.new(upstream, chomp, max_length)
     end
 
+    def self.split(upstream, separator, keep_separator, max_length)
+      Split.new(upstream, separator, keep_separator, max_length)
+    end
+
     private_constant :DONE
   end
 end
@@ -87,15 +103,19 @@ end
 require_relative "pull/each"
 require_relative "pull/io_source"
 require_relative "pull/ractor_port_source"
+require_relative "pull/ractor_merge_ports_source"
 require_relative "pull/concat"
 require_relative "pull/zip"
+require_relative "pull/merge"
 require_relative "pull/map"
 require_relative "pull/select"
 require_relative "pull/take"
 require_relative "pull/drop"
+require_relative "pull/grouped"
 require_relative "pull/take_while"
 require_relative "pull/drop_while"
 require_relative "pull/lines"
+require_relative "pull/split"
 require_relative "pull/async_boundary"
 require_relative "pull/buffer_boundary"
 require_relative "pull/parallel_map_boundary"
@@ -103,8 +123,8 @@ require_relative "pull/ractor_map_boundary"
 
 module FiberStream
   module Pull
-    private_constant :Each, :IOSource, :RactorPortSource, :Concat, :Zip, :Map, :Select, :Take, :Drop,
-                     :TakeWhile, :DropWhile, :Lines, :AsyncBoundary, :BufferBoundary,
-                     :ParallelMapBoundary, :RactorMapBoundary
+    private_constant :Each, :IOSource, :RactorPortSource, :RactorMergePortsSource, :Concat, :Zip, :Merge, :Map,
+                     :Select, :Take, :Drop, :Grouped, :TakeWhile, :DropWhile, :Lines, :Split, :AsyncBoundary,
+                     :BufferBoundary, :ParallelMapBoundary, :RactorMapBoundary
   end
 end

data/lib/fiber_stream/ractor_port.rb

@@ -6,7 +6,9 @@ module FiberStream
   # Producers send `Element`, `Complete`, and `Failure` messages to the data
   # port. FiberStream sends `Ack` and `Cancel` messages to the producer-owned
   # acknowledgment port. The envelopes keep stream values distinct from control
-  # messages and support Ruby pattern matching.
+  # messages and support Ruby pattern matching. `Failure` cause metadata is
+  # producer-provided and is surfaced through `RactorPortSourceError`; producers
+  # should sanitize it before crossing trust boundaries.
   module RactorPort
     Element = ::Data.define(:value)
     Complete = ::Data.define

data/lib/fiber_stream/running_pipeline.rb

@@ -2,6 +2,11 @@
 
 module FiberStream
   class RunningPipeline
+    ValueMessage = Data.define(:value)
+    ErrorMessage = Data.define(:error)
+    CancelledMessage = Data.define(:error)
+    private_constant :ValueMessage, :ErrorMessage, :CancelledMessage
+
     def initialize(scheduler, &run)
       @scheduler = scheduler
       @completion = nil
@@ -82,16 +87,19 @@ module FiberStream
     private
 
     def run_background(run)
-      complete([:value, run.call])
+      complete(ValueMessage.new(value: run.call))
+    rescue SystemExit, SignalException => error
+      complete(ErrorMessage.new(error:))
+      raise
     rescue Exception => error # rubocop:disable Lint/RescueException
       complete(classify_error(error))
     end
 
     def classify_error(error)
       if cancellation_error?(error)
-        [:cancelled, error]
+        CancelledMessage.new(error:)
       else
-        [:error, error]
+        ErrorMessage.new(error:)
       end
     end
 
@@ -114,11 +122,13 @@ module FiberStream
     end
 
     def deliver(message)
-      case message.fetch(0)
-      when :value
-        message.fetch(1)
-      when :error, :cancelled
-        raise message.fetch(1)
+      case message
+      in ValueMessage[value:]
+        value
+      in ErrorMessage[error:]
+        raise error
+      in CancelledMessage[error:]
+        raise error
       end
     end
 

data/lib/fiber_stream/source.rb

@@ -2,6 +2,9 @@
 
 module FiberStream
   class Source
+    RactorMergePortPair = Data.define(:port, :ack_port)
+    private_constant :RactorMergePortPair
+
     # Creates a source definition from an Enumerable.
     #
     # The enumerable is not consumed until values are pulled by `run_with`. Each
@@ -17,7 +20,9 @@ module FiberStream
     # The IO object is not read until values are pulled by `run_with`. Each
     # materialization reads from the same IO object's current position; this
     # source does not snapshot, reopen, or guarantee replayability. The IO is
-    # closed only when `close: true` is passed.
+    # closed only when `close: true` is passed. `chunk_size` is the maximum byte
+    # count passed to `readpartial` for one downstream pull; very large values
+    # may cause the IO implementation to attempt large allocations.
     def self.io(io, chunk_size: 16 * 1024, close: false)
       raise TypeError, "io must respond to readpartial" unless io.respond_to?(:readpartial)
       raise TypeError, "chunk_size must be an Integer" unless chunk_size.is_a?(Integer)
@@ -34,7 +39,8 @@ module FiberStream
     # producer-owned port that receives `RactorPort::Ack` and
     # `RactorPort::Cancel` control messages. The producer must wait for an ack
     # before sending each `RactorPort::Element`, `RactorPort::Complete`, or
-    # `RactorPort::Failure` message.
+    # `RactorPort::Failure` message. Failure metadata is producer-provided and
+    # should be sanitized before crossing trust boundaries.
     def self.ractor_port(port, ack_port:, ack_transfer: :copy, cancel: true)
       raise TypeError, "port must respond to receive" unless port.respond_to?(:receive)
       unless ack_port.respond_to?(:send) && ack_port.method(:send).owner != Kernel
@@ -47,6 +53,24 @@ module FiberStream
       new(-> { Pull.ractor_port(port, ack_port, ack_transfer, cancel) })
     end
 
+    # Creates a backpressure-aware source definition from multiple Ractor port
+    # pairs.
+    #
+    # Each pair must be a Hash with `:port` and `:ack_port`. The source sends
+    # at most one outstanding `RactorPort::Ack` to each active producer and
+    # emits producer values in coordinator-observed ready order. Producer work
+    # is isolated in Ractors, so demanding this source does not require a
+    # `Fiber.scheduler`. Failure metadata is producer-provided and should be
+    # sanitized before crossing trust boundaries.
+    def self.ractor_merge_ports(ports, ack_transfer: :copy, cancel: true)
+      pairs = normalize_ractor_merge_port_pairs(ports)
+
+      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
+
+      new(-> { Pull.ractor_merge_ports(pairs, ack_transfer, cancel) })
+    end
+
     def initialize(source_factory, flows = [])
       @source_factory = source_factory
       @flows = flows
@@ -93,6 +117,23 @@ module FiberStream
       )
     end
 
+    # Returns a new source definition that emits values from this source and
+    # `source` in scheduler-observed ready order.
+    #
+    # Construction is lazy. The merged source starts one scheduled producer
+    # fiber per input source only when downstream demand reaches the merge. Each
+    # input's own element order is preserved, but cross-input ordering is not
+    # deterministic and requires an installed `Fiber.scheduler` from a
+    # non-blocking fiber when demanded.
+    def merge(source)
+      raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
+
+      self.class.__send__(
+        :new,
+        -> { Pull.merge(materializer, source.__send__(:materializer)) }
+      )
+    end
+
     # Returns a new source definition that maps each element with `block`.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.map { ... })`
@@ -154,6 +195,15 @@ module FiberStream
       via(Flow.drop(count))
     end
 
+    # Returns a new source definition that groups adjacent elements into arrays.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.grouped(count))` and preserves the same validation,
+    # ordering, final partial group, and pull-driven backpressure behavior.
+    def grouped(count)
+      via(Flow.grouped(count))
+    end
+
     # Returns a new source definition that emits leading elements while `block`
     # is truthy.
     #
@@ -194,11 +244,25 @@ module FiberStream
     # Returns a new source definition that splits String chunks into lines.
     #
     # This is a convenience wrapper around
-    # `via(FiberStream::Flow.lines(chomp:, max_length:))`.
+    # `via(FiberStream::Flow.lines(chomp:, max_length:))`. With
+    # `max_length: nil`, one unterminated line can buffer without bound. Set a
+    # positive `max_length` for untrusted, network-facing, or otherwise
+    # unbounded streams.
     def lines(chomp: true, max_length: nil)
       via(Flow.lines(chomp: chomp, max_length: max_length))
     end
 
+    # Returns a new source definition that splits String chunks into frames.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.split(separator, keep_separator:, max_length:))`.
+    # With `max_length: nil`, one unterminated frame can buffer without bound.
+    # Set a positive `max_length` for untrusted, network-facing, or otherwise
+    # unbounded streams.
+    def split(separator, keep_separator: false, max_length: nil)
+      via(Flow.split(separator, keep_separator: keep_separator, max_length: max_length))
+    end
+
     # Returns a runnable pipeline from this source to `sink`.
     #
     # Construction is lazy. The source and sink are not materialized until
@@ -237,6 +301,44 @@ module FiberStream
 
     private_class_method :new
 
+    def self.normalize_ractor_merge_port_pairs(ports)
+      raise TypeError, "ports must respond to each" unless ports.respond_to?(:each)
+
+      data_port_ids = {}
+      ack_port_ids = {}
+      pairs =
+        ports.each.map do |pair|
+          normalize_ractor_merge_port_pair(pair, data_port_ids, ack_port_ids)
+        end
+
+      raise ArgumentError, "ractor_merge_ports requires at least two port pairs" if pairs.size < 2
+
+      pairs.freeze
+    end
+
+    def self.normalize_ractor_merge_port_pair(pair, data_port_ids, ack_port_ids)
+      raise TypeError, "port pair must be a Hash" unless pair.is_a?(Hash)
+      raise TypeError, "port pair must include :port and :ack_port" unless pair.key?(:port) && pair.key?(:ack_port)
+
+      port = pair.fetch(:port)
+      ack_port = pair.fetch(:ack_port)
+      raise TypeError, "port must respond to receive" unless port.respond_to?(:receive)
+      unless ack_port.respond_to?(:send) && ack_port.method(:send).owner != Kernel
+        raise TypeError, "ack_port must provide Ractor-style send"
+      end
+
+      port_id = port.object_id
+      ack_port_id = ack_port.object_id
+      raise ArgumentError, "data ports must be distinct" if data_port_ids.key?(port_id)
+      raise ArgumentError, "ack ports must be distinct" if ack_port_ids.key?(ack_port_id)
+
+      data_port_ids[port_id] = true
+      ack_port_ids[ack_port_id] = true
+      RactorMergePortPair.new(port:, ack_port:)
+    end
+
+    private_class_method :normalize_ractor_merge_port_pairs, :normalize_ractor_merge_port_pair
+
     private
 
     def materializer

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/sig/fiber_stream.rbs

@@ -1,5 +1,6 @@
 module FiberStream
   type ractor_transfer_policy = :copy | :move
+  type ractor_port_pair = { port: untyped, ack_port: untyped }
   type ractor_map_error_kind = :input_transfer | :output_transfer | :worker | :worker_termination | :isolation
   type ractor_port_source_error_kind = :invalid_message | :producer_failure | :receive | :ack_transfer | :cancel_transfer
   type ractor_port_cancel_reason = :closed
@@ -51,20 +52,24 @@ module FiberStream
     def self.each: [Elem] (Enumerable[Elem] enumerable) -> Source[Elem]
     def self.io: (untyped io, ?chunk_size: Integer, ?close: bool) -> Source[String]
     def self.ractor_port: [Elem] (untyped port, ack_port: untyped, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
+    def self.ractor_merge_ports: [Elem] (Enumerable[ractor_port_pair] ports, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
     def via: [Out] (Flow[Elem, Out] flow) -> Source[Out]
     def concat: [Other] (Source[Other] source) -> Source[Elem | Other]
     def zip: [Other] (Source[Other] source) -> Source[[Elem, Other]]
+    def merge: [Other] (Source[Other] source) -> Source[Elem | Other]
     def map: [Out] () { (Elem) -> Out } -> Source[Out]
     def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
     def select: () { (Elem) -> boolish } -> Source[Elem]
     def take: (Integer count) -> Source[Elem]
     def drop: (Integer count) -> Source[Elem]
+    def grouped: (Integer count) -> Source[Array[Elem]]
     def take_while: () { (Elem) -> boolish } -> Source[Elem]
     def drop_while: () { (Elem) -> boolish } -> Source[Elem]
     def async: () -> Source[Elem]
     def buffer: (Integer count) -> Source[Elem]
     def lines: (?chomp: bool, ?max_length: Integer?) -> Source[String]
+    def split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Source[String]
     def to: [Mat] (Sink[Elem, Mat] sink) -> Pipeline[Mat]
     def run_with: [Mat] (Sink[Elem, Mat] sink) -> Mat
   end
@@ -76,11 +81,13 @@ module FiberStream
     def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.drop: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.grouped: [Elem] (Integer count) -> Flow[Elem, Array[Elem]]
     def self.take_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.drop_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.async: [Elem] () -> Flow[Elem, Elem]
     def self.buffer: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.lines: (?chomp: bool, ?max_length: Integer?) -> Flow[String, String]
+    def self.split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Flow[String, String]
     def via: [Next] (Flow[Out, Next] flow) -> Flow[In, Next]
     def to: [Mat] (Sink[Out, Mat] sink) -> Sink[In, Mat]
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_stream
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dai Akatsuka
@@ -117,6 +117,7 @@ files:
 - examples/file_copy.rb
 - examples/line_processing.rb
 - examples/ractor_map_hashing.rb
+- examples/ractor_merge_ports_and_map.rb
 - examples/ractor_port_source.rb
 - lib/fiber_stream.rb
 - lib/fiber_stream/errors.rb
@@ -129,13 +130,17 @@ files:
 - lib/fiber_stream/pull/drop.rb
 - lib/fiber_stream/pull/drop_while.rb
 - lib/fiber_stream/pull/each.rb
+- lib/fiber_stream/pull/grouped.rb
 - lib/fiber_stream/pull/io_source.rb
 - lib/fiber_stream/pull/lines.rb
 - lib/fiber_stream/pull/map.rb
+- lib/fiber_stream/pull/merge.rb
 - lib/fiber_stream/pull/parallel_map_boundary.rb
 - lib/fiber_stream/pull/ractor_map_boundary.rb
+- lib/fiber_stream/pull/ractor_merge_ports_source.rb
 - lib/fiber_stream/pull/ractor_port_source.rb
 - lib/fiber_stream/pull/select.rb
+- lib/fiber_stream/pull/split.rb
 - lib/fiber_stream/pull/take.rb
 - lib/fiber_stream/pull/take_while.rb
 - lib/fiber_stream/pull/zip.rb
@@ -151,7 +156,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/dakatsuka/fiber_stream
-  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.2.0
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.3.0
   changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []