fiber_stream 0.1.0

data/lib/fiber_stream/sink.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module FiberStream
+  class Sink
+    # Creates a sink that collects all stream elements into an Array.
+    #
+    # The sink consumes upstream until normal completion and returns the
+    # collected array as the stream materialized value.
+    def self.to_a
+      new do |stream|
+        values = []
+
+        loop do
+          value = stream.next
+          break if Pull.done?(value)
+
+          values << value
+        end
+
+        values
+      end
+    end
+
+    # Creates a sink that returns the first stream element.
+    #
+    # The sink pulls at most one element. It returns `nil` when upstream
+    # completes before producing a value.
+    def self.first
+      new do |stream|
+        value = stream.next
+        Pull.done?(value) ? nil : value
+      end
+    end
+
+    # Creates a sink that folds all stream elements into an accumulator.
+    #
+    # The sink consumes upstream until normal completion. It returns the final
+    # accumulator, or the initial accumulator when upstream is empty. Exceptions
+    # raised by the block fail the stream and are re-raised from
+    # `Source#run_with`. FiberStream assigns the initial accumulator directly;
+    # it does not duplicate or freeze that object.
+    def self.fold(initial, &block)
+      raise ArgumentError, "missing block" unless block
+
+      new do |stream|
+        accumulator = initial
+
+        loop do
+          value = stream.next
+          break if Pull.done?(value)
+
+          accumulator = block.call(accumulator, value)
+        end
+
+        accumulator
+      end
+    end
+
+    # Creates a sink that writes String chunks to an IO-like object.
+    #
+    # The sink consumes upstream until normal completion and returns the number
+    # of chunks successfully written. It requires a scheduler-backed
+    # non-blocking fiber before write, flush, or normal close operations. The IO
+    # object is closed only when `close: true` is passed, and flushed on normal
+    # completion only when `flush: true` is passed.
+    def self.io(io, close: false, flush: false)
+      raise TypeError, "io must respond to write" unless io.respond_to?(:write)
+      raise TypeError, "close must be true or false" unless [true, false].include?(close)
+      raise TypeError, "flush must be true or false" unless [true, false].include?(flush)
+      raise TypeError, "io must respond to close" if close && !io.respond_to?(:close)
+      raise TypeError, "io must respond to flush" if flush && !io.respond_to?(:flush)
+
+      new do |stream|
+        IOSink.new(io, close, flush).run(stream)
+      end
+    end
+
+    def initialize(&run)
+      @run = run
+    end
+
+    private_class_method :new
+
+    private
+
+    def run(stream)
+      @run.call(stream)
+    end
+
+    class IOSink
+      def initialize(io, close_io, flush_io)
+        @io = io
+        @close_io = close_io
+        @flush_io = flush_io
+        @chunks_written = 0
+        @io_closed = false
+      end
+
+      def run(stream)
+        loop do
+          value = stream.next
+          break if Pull.done?(value)
+
+          write(value)
+        end
+
+        finish
+      rescue StandardError
+        close_suppressing_error
+        raise
+      end
+
+      private
+
+      def write(value)
+        unless value.is_a?(String)
+          raise TypeError, "Sink.io elements must be String"
+        end
+
+        validate_scheduler!
+        @io.write(value)
+        @chunks_written += 1
+      end
+
+      def finish
+        flush
+        close_on_normal_completion
+        @chunks_written
+      end
+
+      def flush
+        return unless @flush_io
+
+        validate_scheduler!
+        @io.flush
+      end
+
+      def close_on_normal_completion
+        return unless @close_io
+
+        validate_scheduler!
+        close_error = close_io
+        raise close_error if close_error
+      end
+
+      def validate_scheduler!
+        return if Fiber.scheduler && !Fiber.current.blocking?
+
+        message =
+          if Fiber.scheduler
+            "Sink.io requires a non-blocking fiber"
+          else
+            "Sink.io requires Fiber.scheduler"
+          end
+        raise SchedulerRequiredError, message
+      end
+
+      def close_suppressing_error
+        close_io
+      end
+
+      def close_io
+        return nil unless @close_io
+        return nil if @io_closed
+
+        @io_closed = true
+        @io.close
+        nil
+      rescue StandardError => error
+        error
+      end
+    end
+
+    private_constant :IOSink
+  end
+end

data/lib/fiber_stream/source.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module FiberStream
+  class Source
+    # Creates a source definition from an Enumerable.
+    #
+    # The enumerable is not consumed until values are pulled by `run_with`. Each
+    # materialization creates an Enumerator with `enumerable.to_enum(:each)`;
+    # FiberStream does not snapshot values or guarantee replayability for
+    # one-shot enumerables.
+    def self.each(enumerable)
+      new(-> { Pull.each(enumerable) })
+    end
+
+    # Creates a source definition from an IO-like object.
+    #
+    # The IO object is not read until values are pulled by `run_with`. Each
+    # materialization reads from the same IO object's current position; this
+    # source does not snapshot, reopen, or guarantee replayability. The IO is
+    # closed only when `close: true` is passed.
+    def self.io(io, chunk_size: 16 * 1024, close: false)
+      raise TypeError, "io must respond to readpartial" unless io.respond_to?(:readpartial)
+      raise TypeError, "chunk_size must be an Integer" unless chunk_size.is_a?(Integer)
+      raise ArgumentError, "chunk_size must be positive" unless chunk_size.positive?
+      raise TypeError, "close must be true or false" unless [true, false].include?(close)
+      raise TypeError, "io must respond to close" if close && !io.respond_to?(:close)
+
+      new(-> { Pull.io(io, chunk_size, close) })
+    end
+
+    # Creates a backpressure-aware source definition from Ractor ports.
+    #
+    # `port` is the data/control port received by FiberStream. `ack_port` is a
+    # producer-owned port that receives `RactorPort::Ack` and
+    # `RactorPort::Cancel` control messages. The producer must wait for an ack
+    # before sending each `RactorPort::Element`, `RactorPort::Complete`, or
+    # `RactorPort::Failure` message.
+    def self.ractor_port(port, ack_port:, ack_transfer: :copy, cancel: true)
+      raise TypeError, "port must respond to receive" unless port.respond_to?(:receive)
+      unless ack_port.respond_to?(:send) && ack_port.method(:send).owner != Kernel
+        raise TypeError, "ack_port must provide Ractor-style send"
+      end
+
+      Flow.__send__(:validate_ractor_transfer_policy!, :ack_transfer, ack_transfer)
+      raise TypeError, "cancel must be true or false" unless [true, false].include?(cancel)
+
+      new(-> { Pull.ractor_port(port, ack_port, ack_transfer, cancel) })
+    end
+
+    def initialize(source_factory, flows = [])
+      @source_factory = source_factory
+      @flows = flows
+    end
+
+    # Returns a new source definition that passes this source through `flow`.
+    #
+    # This method is lazy. It does not run the source, enumerate values, or call
+    # flow blocks.
+    def via(flow)
+      raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
+
+      self.class.__send__(:new, @source_factory, @flows + [flow])
+    end
+
+    # Returns a new source definition that maps each element with `block`.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.map { ... })`
+    # and has the same lazy construction, error, and backpressure behavior as
+    # the underlying flow.
+    def map(&block)
+      via(Flow.map(&block))
+    end
+
+    # Returns a new source definition that maps elements concurrently.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.parallel_map(concurrency:) { ... })` and preserves
+    # the same ordered delivery, scheduler requirement, validation, bounded
+    # upstream run-ahead, and cancellation behavior.
+    def parallel_map(concurrency:, &block)
+      via(Flow.parallel_map(concurrency: concurrency, &block))
+    end
+
+    # Returns a new source definition that maps elements in Ractor workers.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.ractor_map(workers:) { ... })` and preserves the
+    # same shareable mapper requirement, ordered delivery, transfer policy,
+    # bounded upstream run-ahead, and cooperative worker shutdown behavior.
+    def ractor_map(workers:, input_transfer: :copy, output_transfer: :copy, &block)
+      via(
+        Flow.ractor_map(
+          workers: workers,
+          input_transfer: input_transfer,
+          output_transfer: output_transfer,
+          &block
+        )
+      )
+    end
+
+    # Returns a new source definition that keeps elements matching `block`.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.select { ... })` and has the same truthiness and
+    # lazy construction behavior as the underlying flow.
+    def select(&block)
+      via(Flow.select(&block))
+    end
+
+    # Returns a new source definition that emits at most `count` elements.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.take(count))`
+    # and preserves the same validation and upstream close behavior.
+    def take(count)
+      via(Flow.take(count))
+    end
+
+    # Returns a new source definition with an asynchronous boundary.
+    #
+    # This is a convenience wrapper around `via(FiberStream::Flow.async)` and
+    # preserves the same scheduler requirement and cancellation behavior.
+    def async
+      via(Flow.async)
+    end
+
+    # Returns a new source definition with a bounded asynchronous buffer.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.buffer(count))` and preserves the same validation,
+    # scheduler requirement, and cancellation behavior.
+    def buffer(count)
+      via(Flow.buffer(count))
+    end
+
+    # Returns a new source definition that splits String chunks into lines.
+    #
+    # This is a convenience wrapper around
+    # `via(FiberStream::Flow.lines(chomp:, max_length:))`.
+    def lines(chomp: true, max_length: nil)
+      via(Flow.lines(chomp: chomp, max_length: max_length))
+    end
+
+    # Returns a runnable pipeline from this source to `sink`.
+    #
+    # Construction is lazy. The source and sink are not materialized until
+    # `Pipeline#run` is called.
+    def to(sink)
+      raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
+
+      Pipeline.__send__(:new, self, sink)
+    end
+
+    # Materializes and runs this source with `sink`.
+    #
+    # The stream runs in the current fiber until completion or failure. The
+    # method returns the sink's materialized value and closes the materialized
+    # pull chain on success, failure, or early sink completion.
+    def run_with(sink)
+      raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
+
+      primary_error = nil
+
+      begin
+        stream = @source_factory.call
+        @flows.each do |flow|
+          stream = flow.__send__(:attach, stream)
+        end
+
+        sink.__send__(:run, stream)
+      rescue StandardError => error
+        primary_error = error
+        raise
+      ensure
+        begin
+          stream&.close
+        rescue StandardError => close_error
+          raise close_error unless primary_error
+        end
+      end
+    end
+
+    private_class_method :new
+  end
+end

data/lib/fiber_stream/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FiberStream
+  VERSION = "0.1.0"
+end

data/lib/fiber_stream.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "fiber_stream/pull"
+require_relative "fiber_stream/version"
+require_relative "fiber_stream/errors"
+require_relative "fiber_stream/ractor_port"
+require_relative "fiber_stream/flow"
+require_relative "fiber_stream/sink"
+require_relative "fiber_stream/running_pipeline"
+require_relative "fiber_stream/pipeline"
+require_relative "fiber_stream/source"
+
+module FiberStream
+  private_constant :Pull
+end

data/sig/fiber_stream.rbs

@@ -0,0 +1,97 @@
+module FiberStream
+  type ractor_transfer_policy = :copy | :move
+  type ractor_map_error_kind = :input_transfer | :output_transfer | :worker | :worker_termination | :isolation
+  type ractor_port_source_error_kind = :invalid_message | :producer_failure | :receive | :ack_transfer | :cancel_transfer
+
+  class SchedulerRequiredError < RuntimeError
+  end
+
+  class FrameTooLongError < RuntimeError
+  end
+
+  class PipelineCancelledError < RuntimeError
+  end
+
+  class RactorPortSourceError < RuntimeError
+    attr_reader kind: ractor_port_source_error_kind
+    attr_reader cause_class_name: String
+    attr_reader cause_message: String
+  end
+
+  class RactorMapError < RuntimeError
+    attr_reader sequence: Integer
+    attr_reader kind: ractor_map_error_kind
+    attr_reader cause_class_name: String
+    attr_reader cause_message: String
+  end
+
+  module RactorPort
+    class Element[Elem] < Data
+      attr_reader value: Elem
+    end
+
+    class Complete < Data
+    end
+
+    class Failure < Data
+      attr_reader cause_class_name: String
+      attr_reader cause_message: String
+    end
+
+    class Ack < Data
+    end
+
+    class Cancel < Data
+      attr_reader reason: Symbol
+    end
+  end
+
+  class Source[Elem]
+    def self.each: [Elem] (Enumerable[Elem] enumerable) -> Source[Elem]
+    def self.io: (untyped io, ?chunk_size: Integer, ?close: bool) -> Source[String]
+    def self.ractor_port: [Elem] (untyped port, ack_port: untyped, ?ack_transfer: ractor_transfer_policy, ?cancel: bool) -> Source[Elem]
+    def via: [Out] (Flow[Elem, Out] flow) -> Source[Out]
+    def map: [Out] () { (Elem) -> Out } -> Source[Out]
+    def parallel_map: [Out] (concurrency: Integer) { (Elem) -> Out } -> Source[Out]
+    def ractor_map: [Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (Elem) -> Out } -> Source[Out]
+    def select: () { (Elem) -> boolish } -> Source[Elem]
+    def take: (Integer count) -> Source[Elem]
+    def async: () -> Source[Elem]
+    def buffer: (Integer count) -> Source[Elem]
+    def lines: (?chomp: bool, ?max_length: Integer?) -> Source[String]
+    def to: [Mat] (Sink[Elem, Mat] sink) -> Pipeline[Mat]
+    def run_with: [Mat] (Sink[Elem, Mat] sink) -> Mat
+  end
+
+  class Flow[In, Out]
+    def self.map: [In, Out] () { (In) -> Out } -> Flow[In, Out]
+    def self.parallel_map: [In, Out] (concurrency: Integer) { (In) -> Out } -> Flow[In, Out]
+    def self.ractor_map: [In, Out] (workers: Integer, ?input_transfer: ractor_transfer_policy, ?output_transfer: ractor_transfer_policy) { (In) -> Out } -> Flow[In, Out]
+    def self.select: [Elem] () { (Elem) -> boolish } -> Flow[Elem, Elem]
+    def self.take: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.async: [Elem] () -> Flow[Elem, Elem]
+    def self.buffer: [Elem] (Integer count) -> Flow[Elem, Elem]
+    def self.lines: (?chomp: bool, ?max_length: Integer?) -> Flow[String, String]
+    def via: [Next] (Flow[Out, Next] flow) -> Flow[In, Next]
+    def to: [Mat] (Sink[Out, Mat] sink) -> Sink[In, Mat]
+  end
+
+  class Sink[In, Mat]
+    def self.to_a: [Elem] () -> Sink[Elem, Array[Elem]]
+    def self.first: [Elem] () -> Sink[Elem, Elem?]
+    def self.fold: [Elem, Acc] (Acc initial) { (Acc, Elem) -> Acc } -> Sink[Elem, Acc]
+    def self.io: (untyped io, ?close: bool, ?flush: bool) -> Sink[String, Integer]
+  end
+
+  class Pipeline[Mat]
+    def run: () -> Mat
+    def run_async: () -> RunningPipeline[Mat]
+  end
+
+  class RunningPipeline[Mat]
+    def wait: () -> Mat
+    def cancel: () -> self
+    def done?: () -> bool
+    def cancel_requested?: () -> bool
+  end
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: fiber_stream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dai Akatsuka
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A Ruby stream processing library built around Fiber and Fiber.scheduler.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- examples/README.md
+- examples/async_http_requests.rb
+- examples/background_execution.rb
+- examples/backpressure_buffer.rb
+- examples/basic_pipeline.rb
+- examples/composable_pipeline.rb
+- examples/file_copy.rb
+- examples/line_processing.rb
+- examples/ractor_map_hashing.rb
+- examples/ractor_port_source.rb
+- lib/fiber_stream.rb
+- lib/fiber_stream/errors.rb
+- lib/fiber_stream/flow.rb
+- lib/fiber_stream/pipeline.rb
+- lib/fiber_stream/pull.rb
+- lib/fiber_stream/pull/async_boundary.rb
+- lib/fiber_stream/pull/buffer_boundary.rb
+- lib/fiber_stream/pull/each.rb
+- lib/fiber_stream/pull/io_source.rb
+- lib/fiber_stream/pull/lines.rb
+- lib/fiber_stream/pull/map.rb
+- lib/fiber_stream/pull/parallel_map_boundary.rb
+- lib/fiber_stream/pull/ractor_map_boundary.rb
+- lib/fiber_stream/pull/ractor_port_source.rb
+- lib/fiber_stream/pull/select.rb
+- lib/fiber_stream/pull/take.rb
+- lib/fiber_stream/ractor_port.rb
+- lib/fiber_stream/running_pipeline.rb
+- lib/fiber_stream/sink.rb
+- lib/fiber_stream/source.rb
+- lib/fiber_stream/version.rb
+- sig/fiber_stream.rbs
+homepage: https://github.com/dakatsuka/fiber_stream
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/dakatsuka/fiber_stream
+  source_code_uri: https://github.com/dakatsuka/fiber_stream/tree/v0.1.0
+  changelog_uri: https://github.com/dakatsuka/fiber_stream/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Asynchronous, non-blocking stream processing with backpressure.
+test_files: []