fiber_stream 0.1.0

data/examples/basic_pipeline.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "fiber_stream"
+
+orders = [
+  { id: 101, customer: "Aki", total: 4_800 },
+  { id: 102, customer: "Mina", total: 12_400 },
+  { id: 103, customer: "Ren", total: 9_900 },
+  { id: 104, customer: "Sora", total: 18_200 }
+]
+
+high_value_summaries =
+  FiberStream::Source.each(orders)
+    .select { |order| order.fetch(:total) >= 10_000 }
+    .map do |order|
+      format(
+        "#%<id>d %-4<customer>s JPY %<total>d",
+        id: order.fetch(:id),
+        customer: order.fetch(:customer),
+        total: order.fetch(:total)
+      )
+    end
+    .run_with(FiberStream::Sink.to_a)
+
+puts "High-value orders"
+high_value_summaries.each { |summary| puts "- #{summary}" }

data/examples/composable_pipeline.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "fiber_stream"
+
+input_lines = [
+  "  id,status,total ",
+  " 1001,active,12000 ",
+  "",
+  " 1002,cancelled,3000 ",
+  " 1003,active,25000 "
+]
+
+normalize_lines =
+  FiberStream::Flow.map(&:strip)
+    .via(FiberStream::Flow.select { |line| !line.empty? })
+
+parse_order =
+  FiberStream::Flow.map do |line|
+    id, status, total = line.split(",", 3)
+
+    {
+      id: id,
+      status: status,
+      total: Integer(total)
+    }
+  end
+
+active_order_sink =
+  parse_order
+    .via(FiberStream::Flow.select { |order| order.fetch(:status) == "active" })
+    .to(FiberStream::Sink.to_a)
+
+pipeline =
+  FiberStream::Source.each(input_lines.drop(1))
+    .via(normalize_lines)
+    .to(active_order_sink)
+
+puts "Active orders"
+pipeline.run.each do |order|
+  puts "- ##{order.fetch(:id)} JPY #{order.fetch(:total)}"
+end

data/examples/file_copy.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "async"
+require "fiber_stream"
+require "tmpdir"
+
+source_text = <<~TEXT
+  FiberStream can read from Ruby core IO objects.
+  Source.io emits String chunks, and Sink.io writes them to another IO.
+  This keeps stream processing explicit while preserving pull-based demand.
+TEXT
+
+Dir.mktmpdir("fiber_stream-example-") do |dir|
+  input_path = File.join(dir, "input.txt")
+  output_path = File.join(dir, "output.txt")
+
+  File.write(input_path, source_text)
+
+  chunks_written =
+    Async do
+      input = File.open(input_path, "rb")
+      output = File.open(output_path, "wb")
+
+      FiberStream::Source.io(input, chunk_size: 24, close: true)
+        .map(&:upcase)
+        .run_with(FiberStream::Sink.io(output, close: true, flush: true))
+    end.wait
+
+  puts "Wrote #{chunks_written} chunks to #{output_path}"
+  puts File.read(output_path)
+end

data/examples/line_processing.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "fiber_stream"
+
+log_chunks = [
+  "INFO boot\nWARN slow query",
+  "\nERROR failed job\n",
+  "INFO recovered"
+]
+
+warnings_and_errors =
+  FiberStream::Source.each(log_chunks)
+    .lines
+    .select { |line| line.start_with?("WARN", "ERROR") }
+    .run_with(FiberStream::Sink.to_a)
+
+puts "Warnings and errors"
+warnings_and_errors.each { |line| puts "- #{line}" }

data/examples/ractor_map_hashing.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "digest"
+require "fiber_stream"
+
+records = [
+  { name: "alpha.bin", payload: +"A" * 200_000 },
+  { name: "bravo.bin", payload: +"B" * 120_000 },
+  { name: "charlie.bin", payload: +"C" * 260_000 },
+  { name: "delta.bin", payload: +"D" * 80_000 }
+]
+
+HASH_RECORD =
+  Ractor.shareable_proc do |record|
+    payload = record.fetch(:payload)
+
+    {
+      name: record.fetch(:name),
+      bytes: payload.bytesize,
+      sha256: Digest::SHA256.hexdigest(payload)
+    }
+  end
+
+digests =
+  FiberStream::Source.each(records)
+    .ractor_map(workers: 2, input_transfer: :move, &HASH_RECORD)
+    .run_with(FiberStream::Sink.to_a)
+
+puts "SHA-256 digests"
+digests.each do |digest|
+  puts format(
+    "- %-11<name>s %7<bytes>d bytes  %<sha256>s",
+    name: digest.fetch(:name),
+    bytes: digest.fetch(:bytes),
+    sha256: digest.fetch(:sha256)
+  )
+end
+
+puts
+puts "Results are emitted in input order."
+puts "input_transfer: :move is safe here because the input records are not reused."

data/examples/ractor_port_source.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "fiber_stream"
+
+data_port = Ractor::Port.new
+setup_port = Ractor::Port.new
+
+producer =
+  Ractor.new(data_port, setup_port) do |outbox, setup|
+    ack_port = Ractor::Port.new
+    setup.send(ack_port)
+
+    values = (1..5).to_enum
+    sent = 0
+
+    loop do
+      case ack_port.receive
+      in FiberStream::RactorPort::Ack
+        begin
+          value = values.next
+          sent += 1
+          outbox.send(FiberStream::RactorPort::Element.new(value))
+        rescue StopIteration
+          outbox.send(FiberStream::RactorPort::Complete.new)
+          break [:completed, sent]
+        end
+      in FiberStream::RactorPort::Cancel[reason]
+        break [:cancelled, sent, reason]
+      end
+    end
+  end
+
+ack_port = setup_port.receive
+
+result =
+  FiberStream::Source.ractor_port(data_port, ack_port: ack_port)
+    .map { |number| number * number }
+    .run_with(FiberStream::Sink.to_a)
+
+puts "Squares from a producer Ractor:"
+puts result.join(", ")
+puts
+puts "Producer status: #{producer.value.inspect}"

data/lib/fiber_stream/errors.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module FiberStream
+  class SchedulerRequiredError < RuntimeError; end
+  class FrameTooLongError < RuntimeError; end
+  class PipelineCancelledError < RuntimeError; end
+
+  # Normalized failure raised by `Source.ractor_port`.
+  #
+  # Producer failures, invalid protocol messages, and source-side Ractor port
+  # failures use this stable error shape so callers do not need to depend on
+  # Ruby's Ractor transport exceptions.
+  class RactorPortSourceError < RuntimeError
+    attr_reader :kind, :cause_class_name, :cause_message, :original_cause
+
+    def initialize(kind:, cause_class_name:, cause_message:, cause: nil)
+      @kind = kind
+      @cause_class_name = cause_class_name
+      @cause_message = cause_message
+      @original_cause = cause
+
+      super("ractor_port #{kind} failure: #{cause_class_name}: #{cause_message}")
+    end
+  end
+
+  # Normalized failure raised for Ractor-backed mapping errors.
+  #
+  # Worker exceptions and Ractor transfer failures may not be directly
+  # transferable back to the main ractor. This error preserves the ordered input
+  # sequence, failure kind, and original exception class/message metadata.
+  class RactorMapError < RuntimeError
+    attr_reader :sequence, :kind, :cause_class_name, :cause_message, :original_cause
+
+    def initialize(sequence:, kind:, cause_class_name:, cause_message:, cause: nil)
+      @sequence = sequence
+      @kind = kind
+      @cause_class_name = cause_class_name
+      @cause_message = cause_message
+      @original_cause = cause
+
+      super("ractor_map #{kind} failure at sequence #{sequence}: #{cause_class_name}: #{cause_message}")
+    end
+  end
+end

data/lib/fiber_stream/flow.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module FiberStream
+  class Flow
+    # Creates a mapping flow.
+    #
+    # The block is called once for each element pulled through this flow.
+    # Exceptions raised by the block fail the stream and are re-raised from
+    # `Source#run_with`.
+    def self.map(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.map(upstream, block) }
+    end
+
+    # Creates an ordered scheduler-backed parallel mapping flow.
+    #
+    # The stage starts internal scheduled fibers on first downstream demand and
+    # requires an installed `Fiber.scheduler` in a non-blocking fiber at that
+    # point. At most `concurrency` mapping blocks run at the same time, and at
+    # most `concurrency` upstream elements are pulled but not yet emitted downstream.
+    # Results are emitted in input order. Closing the boundary closes upstream
+    # and requests internal worker cancellation. FiberStream does not depend on
+    # Async at runtime.
+    def self.parallel_map(concurrency:, &block)
+      raise ArgumentError, "missing block" unless block
+      raise TypeError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
+      raise ArgumentError, "concurrency must be positive" unless concurrency.positive?
+
+      new { |upstream| Pull.parallel_map(upstream, concurrency, block) }
+    end
+
+    # Creates an ordered Ractor-backed mapping flow.
+    #
+    # The mapper runs inside worker ractors and must be shareable, typically
+    # created with `Ractor.shareable_proc`. Results are emitted in input order,
+    # and at most `workers` upstream elements are pulled but not yet emitted.
+    # `input_transfer` and `output_transfer` must be `:copy` or `:move` and are
+    # passed to Ractor message sends for element and result transfer.
+    def self.ractor_map(workers:, input_transfer: :copy, output_transfer: :copy, &block)
+      raise ArgumentError, "missing block" unless block
+      raise TypeError, "workers must be an Integer" unless workers.is_a?(Integer)
+      raise ArgumentError, "workers must be positive" unless workers.positive?
+
+      validate_ractor_transfer_policy!(:input_transfer, input_transfer)
+      validate_ractor_transfer_policy!(:output_transfer, output_transfer)
+      raise TypeError, "block must be shareable" unless Ractor.shareable?(block)
+
+      new { |upstream| Pull.ractor_map(upstream, workers, input_transfer, output_transfer, block) }
+    end
+
+    # Creates a filtering flow.
+    #
+    # The block is called for upstream elements until it returns a truthy value
+    # or upstream completes. Matching elements pass through unchanged.
+    # Exceptions raised by the block fail the stream and are re-raised from
+    # `Source#run_with`.
+    def self.select(&block)
+      raise ArgumentError, "missing block" unless block
+
+      new { |upstream| Pull.select(upstream, block) }
+    end
+
+    # Creates a limiting flow.
+    #
+    # The flow emits at most `count` elements. `take(0)` completes without
+    # pulling upstream and closes upstream on the first downstream demand. After
+    # the limit is reached, upstream is closed during the pull that forwards
+    # the final element. Negative counts raise `ArgumentError`; non-Integer
+    # counts raise `TypeError`.
+    def self.take(count)
+      raise TypeError, "count must be an Integer" unless count.is_a?(Integer)
+      raise ArgumentError, "count must be non-negative" if count.negative?
+
+      new { |upstream| Pull.take(upstream, count) }
+    end
+
+    # Creates a scheduler-backed asynchronous boundary.
+    #
+    # The boundary starts its producer on the first downstream demand and
+    # requires an installed `Fiber.scheduler` at that point. Upstream stages run
+    # in a non-blocking producer fiber, downstream stages remain in the caller's
+    # current fiber, and each downstream pull resumes at most one upstream pull.
+    # Closing the boundary closes upstream and requests producer cancellation.
+    # FiberStream does not depend on Async at runtime.
+    def self.async
+      new { |upstream| Pull.async(upstream) }
+    end
+
+    # Creates a bounded asynchronous buffer.
+    #
+    # The buffer starts its producer on the first downstream demand and requires
+    # an installed `Fiber.scheduler` at that point. It preserves element order,
+    # stores at most `count` messages, and closes upstream while requesting
+    # producer cancellation when closed. `count` must be a positive Integer.
+    # FiberStream does not depend on Async at runtime.
+    def self.buffer(count)
+      raise TypeError, "count must be an Integer" unless count.is_a?(Integer)
+      raise ArgumentError, "count must be positive" unless count.positive?
+
+      new { |upstream| Pull.buffer(upstream, count) }
+    end
+
+    # Creates a line-splitting flow.
+    #
+    # The flow accepts String chunks and emits lines split on "\n". By default
+    # it chomps the trailing newline and one preceding "\r". `max_length` is an
+    # optional per-line bytesize limit.
+    def self.lines(chomp: true, max_length: nil)
+      raise TypeError, "chomp must be true or false" unless [true, false].include?(chomp)
+      unless max_length.nil? || max_length.is_a?(Integer)
+        raise TypeError, "max_length must be nil or an Integer"
+      end
+      raise ArgumentError, "max_length must be positive" if max_length&.<= 0
+
+      new { |upstream| Pull.lines(upstream, chomp, max_length) }
+    end
+
+    def self.validate_ractor_transfer_policy!(name, value)
+      return if [:copy, :move].include?(value)
+
+      raise ArgumentError, "#{name} must be :copy or :move"
+    end
+
+    private_class_method :validate_ractor_transfer_policy!
+
+    # Returns a reusable flow that applies this flow and then `flow`.
+    #
+    # Construction is lazy. No upstream stream is attached and no elements are
+    # pulled until the composed flow is materialized by a source or sink.
+    def via(flow)
+      raise TypeError, "expected FiberStream::Flow" unless flow.is_a?(Flow)
+
+      self.class.__send__(:new) do |upstream|
+        attached_stream = attach(upstream)
+
+        begin
+          flow.__send__(:attach, attached_stream)
+        rescue StandardError
+          begin
+            attached_stream.close
+          rescue StandardError
+            nil
+          end
+          raise
+        end
+      end
+    end
+
+    # Returns a sink that runs this flow before `sink`.
+    #
+    # The composed sink accepts this flow's input elements and returns the
+    # wrapped sink's materialized value. It closes the attached flow chain after
+    # normal completion, failure, or early sink completion.
+    def to(sink)
+      raise TypeError, "expected FiberStream::Sink" unless sink.is_a?(Sink)
+
+      Sink.__send__(:new) do |stream|
+        attached_stream = nil
+        primary_error = nil
+
+        begin
+          attached_stream = attach(stream)
+          sink.__send__(:run, attached_stream)
+        rescue StandardError => error
+          primary_error = error
+          raise
+        ensure
+          begin
+            attached_stream&.close
+          rescue StandardError => close_error
+            raise close_error unless primary_error
+          end
+        end
+      end
+    end
+
+    def initialize(&attach)
+      @attach = attach
+    end
+
+    private_class_method :new
+
+    private
+
+    def attach(upstream)
+      @attach.call(upstream)
+    end
+  end
+end

data/lib/fiber_stream/pipeline.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module FiberStream
+  class Pipeline
+    def initialize(source, sink)
+      @source = source
+      @sink = sink
+    end
+
+    # Runs this pipeline in the current fiber.
+    #
+    # This is equivalent to `source.run_with(sink)` for the source and sink
+    # definitions captured by `Source#to`. Repeated runs create new
+    # materializations, subject to the replayability and resource ownership
+    # semantics of the captured endpoints.
+    def run
+      @source.run_with(@sink)
+    end
+
+    # Runs this pipeline in a scheduler-backed background fiber.
+    #
+    # The method starts one new materialization and returns a `RunningPipeline`
+    # handle that can wait for the materialized value, observe completion, and
+    # request cancellation. Starting background execution requires an installed
+    # `Fiber.scheduler` from a non-blocking current fiber. FiberStream does not
+    # depend on Async at runtime.
+    def run_async
+      validate_scheduler!
+
+      RunningPipeline.__send__(:new, Fiber.scheduler) { run }
+    end
+
+    private_class_method :new
+
+    private
+
+    def validate_scheduler!
+      return if Fiber.scheduler && !Fiber.current.blocking?
+
+      message =
+        if Fiber.scheduler
+          "Pipeline#run_async requires a non-blocking fiber"
+        else
+          "Pipeline#run_async requires Fiber.scheduler"
+        end
+      raise SchedulerRequiredError, message
+    end
+  end
+end

data/lib/fiber_stream/pull/async_boundary.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module FiberStream
+  module Pull
+    # One-element asynchronous boundary for `Flow.async`.
+    #
+    # The producer fiber is created lazily on first downstream demand. It
+    # advances upstream in a non-blocking fiber and yields one message at a
+    # time back to the downstream caller, so it adds an async boundary without
+    # adding prefetch.
+    class AsyncBoundary
+      def initialize(upstream)
+        @upstream = upstream
+        @producer = nil
+        @started = false
+        @closed = false
+        @done = false
+      end
+
+      def next
+        return DONE if @closed || @done
+
+        start
+        message = @producer.resume
+
+        case message.fetch(0)
+        when :value
+          message.fetch(1)
+        when :done
+          complete
+        when :error
+          @done = true
+          raise message.fetch(1)
+        end
+      end
+
+      def close
+        return if @closed
+
+        @closed = true
+        @done = true
+        @upstream.close
+      ensure
+        cancel_producer
+      end
+
+      private
+
+      def start
+        return if @started
+        raise SchedulerRequiredError, "Flow.async requires Fiber.scheduler" unless Fiber.scheduler
+
+        @started = true
+        @producer = Fiber.new(blocking: false) { run_producer }
+      end
+
+      def run_producer
+        loop do
+          break if @closed
+
+          value = @upstream.next
+          if Pull.done?(value)
+            Fiber.yield([:done])
+            break
+          end
+
+          Fiber.yield([:value, value])
+        end
+      rescue StandardError => exception
+        Fiber.yield([:error, exception]) unless @closed
+      ensure
+        @upstream.close
+      end
+
+      def complete
+        @done = true
+        DONE
+      end
+
+      def cancel_producer
+        nil
+      end
+    end
+  end
+end