fiber_stream 0.1.0 → 0.3.0

data/lib/fiber_stream/pull.rb

@@ -28,6 +28,22 @@ module FiberStream
       RactorPortSource.new(port, ack_port, ack_transfer, cancel)
     end
 
+    def self.ractor_merge_ports(port_pairs, ack_transfer, cancel)
+      RactorMergePortsSource.new(port_pairs, ack_transfer, cancel)
+    end
+
+    def self.concat(left_materializer, right_materializer)
+      Concat.new(left_materializer, right_materializer)
+    end
+
+    def self.zip(left_materializer, right_materializer)
+      Zip.new(left_materializer, right_materializer)
+    end
+
+    def self.merge(left_materializer, right_materializer)
+      Merge.new(left_materializer, right_materializer)
+    end
+
     def self.map(upstream, transform)
       Map.new(upstream, transform)
     end
@@ -48,6 +64,22 @@ module FiberStream
       Take.new(upstream, count)
     end
 
+    def self.drop(upstream, count)
+      Drop.new(upstream, count)
+    end
+
+    def self.grouped(upstream, count)
+      Grouped.new(upstream, count)
+    end
+
+    def self.take_while(upstream, predicate)
+      TakeWhile.new(upstream, predicate)
+    end
+
+    def self.drop_while(upstream, predicate)
+      DropWhile.new(upstream, predicate)
+    end
+
     def self.async(upstream)
       AsyncBoundary.new(upstream)
     end
@@ -60,6 +92,10 @@ module FiberStream
       Lines.new(upstream, chomp, max_length)
     end
 
+    def self.split(upstream, separator, keep_separator, max_length)
+      Split.new(upstream, separator, keep_separator, max_length)
+    end
+
     private_constant :DONE
   end
 end
@@ -67,10 +103,19 @@ end
 require_relative "pull/each"
 require_relative "pull/io_source"
 require_relative "pull/ractor_port_source"
+require_relative "pull/ractor_merge_ports_source"
+require_relative "pull/concat"
+require_relative "pull/zip"
+require_relative "pull/merge"
 require_relative "pull/map"
 require_relative "pull/select"
 require_relative "pull/take"
+require_relative "pull/drop"
+require_relative "pull/grouped"
+require_relative "pull/take_while"
+require_relative "pull/drop_while"
 require_relative "pull/lines"
+require_relative "pull/split"
 require_relative "pull/async_boundary"
 require_relative "pull/buffer_boundary"
 require_relative "pull/parallel_map_boundary"
@@ -78,8 +123,8 @@ require_relative "pull/ractor_map_boundary"
 
 module FiberStream
   module Pull
-    private_constant :Each, :IOSource, :RactorPortSource, :Map, :Select, :Take, :Lines,
-                     :AsyncBoundary, :BufferBoundary, :ParallelMapBoundary,
-                     :RactorMapBoundary
+    private_constant :Each, :IOSource, :RactorPortSource, :RactorMergePortsSource, :Concat, :Zip, :Merge, :Map,
+                     :Select, :Take, :Drop, :Grouped, :TakeWhile, :DropWhile, :Lines, :Split, :AsyncBoundary,
+                     :BufferBoundary, :ParallelMapBoundary, :RactorMapBoundary
   end
 end

data/lib/fiber_stream/ractor_port.rb

@@ -6,7 +6,9 @@ module FiberStream
   # Producers send `Element`, `Complete`, and `Failure` messages to the data
   # port. FiberStream sends `Ack` and `Cancel` messages to the producer-owned
   # acknowledgment port. The envelopes keep stream values distinct from control
-  # messages and support Ruby pattern matching.
+  # messages and support Ruby pattern matching. `Failure` cause metadata is
+  # producer-provided and is surfaced through `RactorPortSourceError`; producers
+  # should sanitize it before crossing trust boundaries.
   module RactorPort
     Element = ::Data.define(:value)
     Complete = ::Data.define

data/lib/fiber_stream/running_pipeline.rb

@@ -2,6 +2,11 @@
 
 module FiberStream
   class RunningPipeline
+    ValueMessage = Data.define(:value)
+    ErrorMessage = Data.define(:error)
+    CancelledMessage = Data.define(:error)
+    private_constant :ValueMessage, :ErrorMessage, :CancelledMessage
+
     def initialize(scheduler, &run)
       @scheduler = scheduler
       @completion = nil
@@ -82,16 +87,19 @@ module FiberStream
     private
 
     def run_background(run)
-      complete([:value, run.call])
+      complete(ValueMessage.new(value: run.call))
+    rescue SystemExit, SignalException => error
+      complete(ErrorMessage.new(error:))
+      raise
     rescue Exception => error # rubocop:disable Lint/RescueException
       complete(classify_error(error))
     end
 
     def classify_error(error)
       if cancellation_error?(error)
-        [:cancelled, error]
+        CancelledMessage.new(error:)
       else
-        [:error, error]
+        ErrorMessage.new(error:)
       end
     end
 
@@ -114,11 +122,13 @@ module FiberStream
     end
 
     def deliver(message)
-      case message.fetch(0)
-      when :value
-        message.fetch(1)
-      when :error, :cancelled
-        raise message.fetch(1)
+      case message
+      in ValueMessage[value:]
+        value
+      in ErrorMessage[error:]
+        raise error
+      in CancelledMessage[error:]
+        raise error
       end
     end
 

data/lib/fiber_stream/sink.rb

@@ -56,6 +56,30 @@ module FiberStream
       end
     end
 
+    # Creates a sink that runs a block for each stream element.
+    #
+    # The sink consumes upstream until normal completion, calls the block once
+    # per element in input order, and returns the number of elements whose block
+    # completed successfully. Exceptions raised by the block fail the stream and
+    # are re-raised from `Source#run_with`.
+    def self.foreach(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new do |stream|
+        count = 0
+
+        loop do
+          value = stream.next
+          break if Pull.done?(value)
+
+          block.call(value)
+          count += 1
+        end
+
+        count
+      end
+    end
+
     # Creates a sink that writes String chunks to an IO-like object.
     #
     # The sink consumes upstream until normal completion and returns the number

data/lib/fiber_stream/source.rb

@@ -2,6 +2,9 @@
 
 module FiberStream
   class Source
+    RactorMergePortPair = Data.define(:port, :ack_port)
+    private_constant :RactorMergePortPair
+
     # Creates a source definition from an Enumerable.
     #
     # The enumerable is not consumed until values are pulled by `run_with`. Each
@@ -17,7 +20,9 @@ module FiberStream
     # The IO object is not read until values are pulled by `run_with`. Each
     # materialization reads from the same IO object's current position; this
     # source does not snapshot, reopen, or guarantee replayability. The IO is
-    # closed only when `close: true` is passed.
+    # closed only when `close: true` is passed. `chunk_size` is the maximum byte
+    # count passed to `readpartial` for one downstream pull; very large values
+    # may cause the IO implementation to attempt large allocations.
     def self.io(io, chunk_size: 16 * 1024, close: false)
       raise TypeError, "io must respond to readpartial" unless io.respond_to?(:readpartial)
       raise TypeError, "chunk_size must be an Integer" unless chunk_size.is_a?(Integer)
@@ -34,7 +39,8 @@ module FiberStream
     # producer-owned port that receives `RactorPort::Ack` and
     # `RactorPort::Cancel` control messages. The producer must wait for an ack
     # before sending each `RactorPort::Element`, `RactorPort::Complete`, or
-    # `RactorPort::Failure` message.
+    # `RactorPort::Failure` message. Failure metadata is producer-provided and
+    # should be sanitized before crossing trust boundaries.
     def self.ractor_port(port, ack_port:, ack_transfer: :copy, cancel: true)
       raise TypeError, "port must respond to receive" unless port.respond_to?(:receive)
       unless ack_port.respond_to?(:send) && ack_port.method(:send).owner != Kernel
@@ -47,6 +53,24 @@ module FiberStream
       new(-> { Pull.ractor_port(port, ack_port, ack_transfer, cancel) })
     end
 
+    # Creates a backpressure-aware source definition from multiple Ractor port
+    # pairs.
+    #
+    # Each pair must be a Hash with `:port` and `:ack_port`. The source sends
+    # at most one outstanding `RactorPort::Ack` to each active producer and
+    # emits producer values in coordinator-observed ready order. Producer work
+    # is isolated in Ractors, so demanding this source does not require a
+    # `Fiber.scheduler`. Failure metadata is producer-provided and should be
+    # sanitized before crossing trust boundaries.
+    def self.ractor_merge_ports(ports, ack_transfer: :copy, cancel: true)
+      pairs = normalize_ractor_merge_port_pairs(ports)
+
+      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
+
+      new(-> { Pull.ractor_merge_ports(pairs, ack_transfer, cancel) })
+    end
+
     def initialize(source_factory, flows = [])
       @source_factory = source_factory
       @flows = flows
@@ -62,6 +86,54 @@ module FiberStream
       self.class.__send__(:new, @source_factory, @flows + [flow])
     end
 
+    # Returns a new source definition that emits this source, then `source`.
+    #
+    # Construction is lazy. The appended source is not materialized or pulled
+    # until downstream demand observes completion from this source. Flows
+    # attached before concat stay scoped to their source; flows attached after
+    # concat apply to the combined output.
+    def concat(source)
+      raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
+
+      self.class.__send__(
+        :new,
+        -> { Pull.concat(materializer, source.__send__(:materializer)) }
+      )
+    end
+
+    # Returns a new source definition that emits pairs from this source and
+    # `source`.
+    #
+    # Construction is lazy. The receiver side is materialized only when
+    # downstream demand reaches the zip stage; the other side is materialized
+    # only after the receiver produces an element for a pair. The zipped source
+    # completes when either input completes.
+    def zip(source)
+      raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
+
+      self.class.__send__(
+        :new,
+        -> { Pull.zip(materializer, source.__send__(:materializer)) }
+      )
+    end
+
+    # Returns a new source definition that emits values from this source and
+    # `source` in scheduler-observed ready order.
+    #
+    # Construction is lazy. The merged source starts one scheduled producer
+    # fiber per input source only when downstream demand reaches the merge. Each
+    # input's own element order is preserved, but cross-input ordering is not
+    # deterministic and requires an installed `Fiber.scheduler` from a
+    # non-blocking fiber when demanded.
+    def merge(source)
+      raise TypeError, "expected FiberStream::Source" unless source.is_a?(Source)
+
+      self.class.__send__(
+        :new,
+        -> { Pull.merge(materializer, source.__send__(:materializer)) }
+      )
+    end
+
     # Returns a new source definition that maps each element with `block`.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.map { ... })`
@@ -115,6 +187,43 @@ module FiberStream
       via(Flow.take(count))
     end
 
+    # Returns a new source definition that drops the first `count` elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.drop(count))`
+    # and preserves the same validation and pull-driven backpressure behavior.
+    def drop(count)
+      via(Flow.drop(count))
+    end
+
+    # Returns a new source definition that groups adjacent elements into arrays.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.grouped(count))` and preserves the same validation,
+    # ordering, final partial group, and pull-driven backpressure behavior.
+    def grouped(count)
+      via(Flow.grouped(count))
+    end
+
+    # Returns a new source definition that emits leading elements while `block`
+    # is truthy.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.take_while { ... })` and preserves the same
+    # predicate truthiness, early completion, and upstream close behavior.
+    def take_while(&block)
+      via(Flow.take_while(&block))
+    end
+
+    # Returns a new source definition that drops leading elements while `block`
+    # is truthy.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.drop_while { ... })` and preserves the same
+    # predicate truthiness, prefix-dropping, and pass-through behavior.
+    def drop_while(&block)
+      via(Flow.drop_while(&block))
+    end
+
     # Returns a new source definition with an asynchronous boundary.
     #
     # This is a convenience wrapper around `via(FiberStream::Flow.async)` and
@@ -135,11 +244,25 @@ module FiberStream
     # Returns a new source definition that splits String chunks into lines.
     #
     # This is a convenience wrapper around
-    # `via(FiberStream::Flow.lines(chomp:, max_length:))`.
+    # `via(FiberStream::Flow.lines(chomp:, max_length:))`. With
+    # `max_length: nil`, one unterminated line can buffer without bound. Set a
+    # positive `max_length` for untrusted, network-facing, or otherwise
+    # unbounded streams.
     def lines(chomp: true, max_length: nil)
       via(Flow.lines(chomp: chomp, max_length: max_length))
     end
 
+    # Returns a new source definition that splits String chunks into frames.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.split(separator, keep_separator:, max_length:))`.
+    # With `max_length: nil`, one unterminated frame can buffer without bound.
+    # Set a positive `max_length` for untrusted, network-facing, or otherwise
+    # unbounded streams.
+    def split(separator, keep_separator: false, max_length: nil)
+      via(Flow.split(separator, keep_separator: keep_separator, max_length: max_length))
+    end
+
     # Returns a runnable pipeline from this source to `sink`.
     #
     # Construction is lazy. The source and sink are not materialized until
@@ -161,10 +284,7 @@ module FiberStream
       primary_error = nil
 
       begin
-        stream = @source_factory.call
-        @flows.each do |flow|
-          stream = flow.__send__(:attach, stream)
-        end
+        stream = materialize
 
         sink.__send__(:run, stream)
       rescue StandardError => error
@@ -180,5 +300,68 @@ module FiberStream
     end
 
     private_class_method :new
+
+    def self.normalize_ractor_merge_port_pairs(ports)
+      raise TypeError, "ports must respond to each" unless ports.respond_to?(:each)
+
+      data_port_ids = {}
+      ack_port_ids = {}
+      pairs =
+        ports.each.map do |pair|
+          normalize_ractor_merge_port_pair(pair, data_port_ids, ack_port_ids)
+        end
+
+      raise ArgumentError, "ractor_merge_ports requires at least two port pairs" if pairs.size < 2
+
+      pairs.freeze
+    end
+
+    def self.normalize_ractor_merge_port_pair(pair, data_port_ids, ack_port_ids)
+      raise TypeError, "port pair must be a Hash" unless pair.is_a?(Hash)
+      raise TypeError, "port pair must include :port and :ack_port" unless pair.key?(:port) && pair.key?(:ack_port)
+
+      port = pair.fetch(:port)
+      ack_port = pair.fetch(:ack_port)
+      raise TypeError, "port must respond to receive" unless port.respond_to?(:receive)
+      unless ack_port.respond_to?(:send) && ack_port.method(:send).owner != Kernel
+        raise TypeError, "ack_port must provide Ractor-style send"
+      end
+
+      port_id = port.object_id
+      ack_port_id = ack_port.object_id
+      raise ArgumentError, "data ports must be distinct" if data_port_ids.key?(port_id)
+      raise ArgumentError, "ack ports must be distinct" if ack_port_ids.key?(ack_port_id)
+
+      data_port_ids[port_id] = true
+      ack_port_ids[ack_port_id] = true
+      RactorMergePortPair.new(port:, ack_port:)
+    end
+
+    private_class_method :normalize_ractor_merge_port_pairs, :normalize_ractor_merge_port_pair
+
+    private
+
+    def materializer
+      -> { materialize }
+    end
+
+    def materialize
+      stream = nil
+
+      begin
+        stream = @source_factory.call
+        @flows.each do |flow|
+          stream = flow.__send__(:attach, stream)
+        end
+        stream
+      rescue StandardError
+        begin
+          stream&.close
+        rescue StandardError
+          nil
+        end
+        raise
+      end
+    end
   end
 end

data/lib/fiber_stream/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberStream
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/sig/fiber_stream.rbs

@@ -1,7 +1,9 @@
 module FiberStream
   type ractor_transfer_policy = :copy | :move
+  type ractor_port_pair = { port: untyped, ack_port: untyped }
   type ractor_map_error_kind = :input_transfer | :output_transfer | :worker | :worker_termination | :isolation
   type ractor_port_source_error_kind = :invalid_message | :producer_failure | :receive | :ack_transfer | :cancel_transfer
+  type ractor_port_cancel_reason = :closed
 
   class SchedulerRequiredError < RuntimeError
   end
@@ -42,7 +44,7 @@ module FiberStream
     end
 
     class Cancel < Data
-      attr_reader reason: Symbol
+      attr_reader reason: ractor_port_cancel_reason
     end
   end
 
@@ -50,15 +52,24 @@ module FiberStream
     def self.each: [Elem] (Enumerable[Elem] enumerable) -> Source[Elem]
     def self.io: (untyped io, ?chunk_size: Integer, ?close: bool) -> Source[String]
     def self.ractor_port: [Elem] (untyped port, ack_port: untyped, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
+    def self.ractor_merge_ports: [Elem] (Enumerable[ractor_port_pair] ports, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
     def via: [Out] (Flow[Elem, Out] flow) -> Source[Out]
+    def concat: [Other] (Source[Other] source) -> Source[Elem | Other]
+    def zip: [Other] (Source[Other] source) -> Source[[Elem, Other]]
+    def merge: [Other] (Source[Other] source) -> Source[Elem | Other]
     def map: [Out] () { (Elem) -> Out } -> Source[Out]
     def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
     def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
     def select: () { (Elem) -> boolish } -> Source[Elem]
     def take: (Integer count) -> Source[Elem]
+    def drop: (Integer count) -> Source[Elem]
+    def grouped: (Integer count) -> Source[Array[Elem]]
+    def take_while: () { (Elem) -> boolish } -> Source[Elem]
+    def drop_while: () { (Elem) -> boolish } -> Source[Elem]
     def async: () -> Source[Elem]
     def buffer: (Integer count) -> Source[Elem]
     def lines: (?chomp: bool, ?max_length: Integer?) -> Source[String]
+    def split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Source[String]
     def to: [Mat] (Sink[Elem, Mat] sink) -> Pipeline[Mat]
     def run_with: [Mat] (Sink[Elem, Mat] sink) -> Mat
   end
@@ -69,9 +80,14 @@ module FiberStream
     def self.ractor_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
     def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.drop: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.grouped: [Elem] (Integer count) -> Flow[Elem, Array[Elem]]
+    def self.take_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
+    def self.drop_while: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
     def self.async: [Elem] () -> Flow[Elem, Elem]
     def self.buffer: [Elem] (Integer count) -> Flow[Elem, Elem]
     def self.lines: (?chomp: bool, ?max_length: Integer?) -> Flow[String, String]
+    def self.split: (String separator, ?keep_separator: bool, ?max_length: Integer?) -> Flow[String, String]
     def via: [Next] (Flow[Out, Next] flow) -> Flow[In, Next]
     def to: [Mat] (Sink[Out, Mat] sink) -> Sink[In, Mat]
   end
@@ -80,6 +96,7 @@ module FiberStream
     def self.to_a: [Elem] () -> Sink[Elem, Array[Elem]]
     def self.first: [Elem] () -> Sink[Elem, Elem?]
     def self.fold: [Elem, Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Sink[Elem, Acc]
+    def self.foreach: [Elem] () { (Elem) -> void } -> Sink[Elem, Integer]
     def self.io: (untyped io, ?close: bool, ?flush: bool) -> Sink[String, Integer]
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_stream
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dai Akatsuka
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.95'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.95'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +109,7 @@ files:
 - README.md
 - examples/README.md
 - examples/async_http_requests.rb
+- examples/async_http_streaming_body.rb
 - examples/background_execution.rb
 - examples/backpressure_buffer.rb
 - examples/basic_pipeline.rb
@@ -102,6 +117,7 @@ files:
 - examples/file_copy.rb
 - examples/line_processing.rb
 - examples/ractor_map_hashing.rb
+- examples/ractor_merge_ports_and_map.rb
 - examples/ractor_port_source.rb
 - lib/fiber_stream.rb
 - lib/fiber_stream/errors.rb
@@ -110,15 +126,24 @@ files:
 - lib/fiber_stream/pull.rb
 - lib/fiber_stream/pull/async_boundary.rb
 - lib/fiber_stream/pull/buffer_boundary.rb
+- lib/fiber_stream/pull/concat.rb
+- lib/fiber_stream/pull/drop.rb
+- lib/fiber_stream/pull/drop_while.rb
 - lib/fiber_stream/pull/each.rb
+- lib/fiber_stream/pull/grouped.rb
 - lib/fiber_stream/pull/io_source.rb
 - lib/fiber_stream/pull/lines.rb
 - lib/fiber_stream/pull/map.rb
+- lib/fiber_stream/pull/merge.rb
 - lib/fiber_stream/pull/parallel_map_boundary.rb
 - lib/fiber_stream/pull/ractor_map_boundary.rb
+- lib/fiber_stream/pull/ractor_merge_ports_source.rb
 - lib/fiber_stream/pull/ractor_port_source.rb
 - lib/fiber_stream/pull/select.rb
+- lib/fiber_stream/pull/split.rb
 - lib/fiber_stream/pull/take.rb
+- lib/fiber_stream/pull/take_while.rb
+- lib/fiber_stream/pull/zip.rb
 - lib/fiber_stream/ractor_port.rb
 - lib/fiber_stream/running_pipeline.rb
 - lib/fiber_stream/sink.rb
@@ -131,7 +156,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/dakatsuka/fiber_stream
-  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.1.0
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.3.0
   changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []