curb 1.3.7 → 1.4.0

data/tests/tc_curl_multi_poll.rb

@@ -0,0 +1,223 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+# Coverage for the blocking (non-scheduler) Curl::Multi#perform wait loop:
+# descriptors above FD_SETSIZE, long waits, and interrupting those waits.
+class TestCurbCurlMultiPoll < Test::Unit::TestCase
+  FD_SETSIZE = 1024
+  HIGH_FD_TARGET = FD_SETSIZE + 76
+
+  class PollInterrupt < StandardError; end
+
+  def setup
+    super
+    @server = CurbTinyHTTPServer.new
+    @original_default_timeout = Curl::Multi.default_timeout
+  end
+
+  def teardown
+    Curl::Multi.default_timeout = @original_default_timeout
+    @server.close if @server
+    super
+  end
+
+  def monotonic
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+
+  # Under valgrind these bounds can no longer tell the old fixed-sleep loop
+  # from ordinary overhead. Skip only the bounds; the requests and their
+  # assertions still run under memcheck.
+  def assert_faster_than(limit, elapsed, message)
+    return if curb_under_valgrind?
+    assert_operator elapsed, :<, limit, message
+  end
+
+  # Occupy low descriptor numbers so every socket libcurl opens afterwards is
+  # numbered above FD_SETSIZE and cannot be represented in an fd_set.
+  def with_fds_above_fd_setsize
+    omit('fd numbering differs on Windows') if WINDOWS
+    # Before 7.68 there is no curl_multi_poll and perform falls back to
+    # select(), which cannot represent descriptors above FD_SETSIZE.
+    omit('libcurl < 7.68 has no curl_multi_poll') if Gem::Version.new(Curl::CURL_VERSION[/\A\d+(\.\d+)*/]) < Gem::Version.new('7.68.0')
+
+    soft, hard = Process.getrlimit(:NOFILE)
+    needed = HIGH_FD_TARGET + 256
+    if soft < needed
+      omit("RLIMIT_NOFILE hard limit #{hard} is too low") if hard != Process::RLIM_INFINITY && hard < needed
+      Process.setrlimit(:NOFILE, needed, hard)
+    end
+
+    hog = []
+    hog << File.open(File::NULL) while hog.empty? || hog.last.fileno < HIGH_FD_TARGET
+
+    probe = Socket.new(:INET, :STREAM)
+    assert_operator probe.fileno, :>, FD_SETSIZE, "new sockets should be numbered above FD_SETSIZE"
+    probe.close
+
+    yield
+  ensure
+    hog.each(&:close) if hog
+    Process.setrlimit(:NOFILE, soft, hard) if soft && Process.getrlimit(:NOFILE)[0] != soft
+  end
+
+  # A request whose socket is above FD_SETSIZE used to be invisible to the
+  # curl_multi_fdset/select loop, which then fell back to fixed 100ms sleeps,
+  # so each request took at least 100ms. Ten sequential local requests would
+  # need a full second.
+  def test_easy_perform_with_socket_fds_above_fd_setsize
+    with_fds_above_fd_setsize do
+      started = monotonic
+      10.times do
+        assert_equal "fast", Curl.get(@server.url('/fast')).body_str
+      end
+      elapsed = monotonic - started
+
+      assert_faster_than 0.6, elapsed, "10 local requests took #{elapsed.round(3)}s with high-numbered sockets"
+    end
+  end
+
+  # Batches of concurrent requests share each 100ms sleep of the old loop, so
+  # run several batches to make that fixed floor (>= 1s total) visible.
+  def test_multi_concurrent_requests_with_socket_fds_above_fd_setsize
+    with_fds_above_fd_setsize do
+      started = monotonic
+      5.times do |batch|
+        multi = Curl::Multi.new
+        easies = 20.times.map do |i|
+          easy = Curl::Easy.new(@server.url("/fast?#{batch}-#{i}"))
+          multi.add(easy)
+          easy
+        end
+
+        multi.perform
+
+        easies.each do |easy|
+          assert_equal 200, easy.response_code
+          assert_equal "fast", easy.body_str
+        end
+      ensure
+        multi.close if multi
+      end
+      elapsed = monotonic - started
+
+      assert_faster_than 0.8, elapsed, "5 batches of 20 concurrent local requests took #{elapsed.round(3)}s with high-numbered sockets"
+    end
+  end
+
+  # With a large default_timeout, readiness on the transfer's socket must still
+  # wake the wait immediately rather than waiting out the timeout.
+  def test_long_default_timeout_still_wakes_on_socket_activity
+    Curl::Multi.default_timeout = 5_000
+
+    multi = Curl::Multi.new
+    fast = Curl::Easy.new(@server.url('/fast'))
+    delayed = Curl::Easy.new(@server.url('/delay/0.3'))
+    multi.add(fast)
+    multi.add(delayed)
+
+    started = monotonic
+    multi.perform
+    elapsed = monotonic - started
+
+    assert_equal "fast", fast.body_str
+    assert_equal "delayed", delayed.body_str
+    assert_operator elapsed, :<, 2.0, "perform waited #{elapsed.round(3)}s; the 5s default_timeout should not delay completion"
+  ensure
+    multi.close if multi
+  end
+
+  # The block passed to perform is yielded each time the wait times out, so a
+  # small default_timeout keeps it running while a transfer is idle.
+  def test_perform_block_yields_while_waiting
+    Curl::Multi.default_timeout = 20
+
+    multi = Curl::Multi.new
+    easy = Curl::Easy.new(@server.url('/delay/0.4'))
+    multi.add(easy)
+
+    yields = 0
+    multi.perform { yields += 1 }
+
+    assert_equal "delayed", easy.body_str
+    assert_operator yields, :>=, 5, "perform block ran #{yields} times during a 400ms idle transfer"
+  ensure
+    multi.close if multi
+  end
+
+  def run_interruptible_hang
+    Curl::Multi.default_timeout = 30_000
+    errors = Queue.new
+
+    worker = Thread.new do
+      Thread.current.report_on_exception = false
+      begin
+        Curl.get(@server.url('/hang'))
+        errors << nil
+      rescue Exception => e
+        errors << e
+        raise
+      end
+    end
+
+    assert_equal '/hang', curb_pop_within(@server.requests, 5), "server never received the request"
+    # Give the worker a moment to settle into the blocking wait.
+    sleep 0.1
+
+    started = monotonic
+    yield worker
+    joined = worker.join(3) rescue worker
+    elapsed = monotonic - started
+
+    assert_not_nil joined, "worker was still blocked 3s after being interrupted"
+    assert_operator elapsed, :<, 2.0, "interrupt took #{elapsed.round(3)}s to stop the wait"
+    errors.empty? ? nil : errors.pop
+  ensure
+    if worker&.alive?
+      worker.kill
+      worker.join(1)
+    end
+  end
+
+  def test_thread_raise_interrupts_long_wait
+    error = run_interruptible_hang { |worker| worker.raise(PollInterrupt, "stop waiting") }
+    assert_kind_of PollInterrupt, error
+  end
+
+  def test_thread_kill_interrupts_long_wait
+    run_interruptible_hang { |worker| worker.kill }
+  end
+
+  # A trapped signal must run its handler promptly during a long wait, and the
+  # wait must then resume so the transfer still completes. The signal can
+  # interrupt the poll with EINTR directly, so this guards against treating
+  # that as a failed wait rather than exercising the unblocking function
+  # (the Thread#raise/#kill tests cover that).
+  def test_signal_trap_runs_during_long_wait_and_transfer_completes
+    omit('POSIX signals are not available on Windows') if WINDOWS
+    Curl::Multi.default_timeout = 30_000
+
+    handled_at = nil
+    previous = trap('USR2') { handled_at = monotonic }
+    trapped = true
+    sent_at = nil
+
+    signaller = Thread.new do
+      curb_pop_within(@server.requests, 5)
+      sleep 0.1
+      sent_at = monotonic
+      Process.kill('USR2', Process.pid)
+      sleep 0.3
+      @server.release_hung(1)
+    end
+
+    easy = Curl.get(@server.url('/hang'))
+    signaller.join(5)
+
+    assert_equal "released", easy.body_str
+    assert_not_nil handled_at, "USR2 handler never ran"
+    assert_operator handled_at - sent_at, :<, 0.25, "signal handler was delayed #{(handled_at - sent_at).round(3)}s by the wait"
+  ensure
+    trap('USR2', previous || 'DEFAULT') if trapped
+    signaller&.kill
+  end
+end

data/tests/tc_curl_multi_socket_poller.rb

@@ -0,0 +1,374 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+begin
+  require 'async'
+rescue LoadError
+  nil
+end
+
+# Coverage for the socket-action loop's kernel event queue (epoll/kqueue):
+# under a fiber scheduler every libcurl socket is waited on through a single
+# poller descriptor with io_wait, instead of the optional io_select hook.
+class TestCurbCurlMultiSocketPoller < Test::Unit::TestCase
+  class PollerAbort < StandardError; end
+
+  class RecordingScheduler
+    attr_reader :io_wait_calls, :io_select_calls
+
+    def initialize
+      @io_wait_calls = []
+      @io_select_calls = 0
+    end
+
+    def fiber(&block)
+      Fiber.new(blocking: false, &block)
+    end
+
+    def io_wait(io, events, timeout = nil)
+      @io_wait_calls << [io.fileno, events]
+      readers = (events & IO::READABLE) != 0 ? [io] : nil
+      writers = (events & IO::WRITABLE) != 0 ? [io] : nil
+      readable, writable = blocking_io { IO.select(readers, writers, nil, timeout) }
+
+      ready = 0
+      ready |= IO::READABLE if readable && !readable.empty?
+      ready |= IO::WRITABLE if writable && !writable.empty?
+      ready.zero? ? false : ready
+    end
+
+    def io_select(readers, writers, excepts, timeout = nil)
+      @io_select_calls += 1
+      blocking_io { IO.select(readers, writers, excepts, timeout) }
+    end
+
+    def kernel_sleep(duration = nil)
+      blocking_io { sleep(duration || 0) }
+    end
+
+    def block(_blocker, timeout = nil)
+      blocking_io { sleep(timeout || 0) }
+      false
+    end
+
+    def unblock(*)
+    end
+
+    def close
+    end
+
+    def fiber_interrupt(*)
+    end
+
+    private
+
+    def blocking_io(&block)
+      if Fiber.respond_to?(:blocking)
+        Fiber.blocking(&block)
+      else
+        Fiber.new(blocking: true, &block).resume
+      end
+    end
+  end
+
+  # io_select is an optional hook; many schedulers only implement io_wait.
+  class IoWaitOnlyScheduler < RecordingScheduler
+    undef_method :io_select
+  end
+
+  # Counts Async's io_select calls, each of which starts a Thread. Prepended
+  # to a single scheduler's singleton class so other tests are unaffected.
+  module AsyncIoSelectCounter
+    attr_reader :curb_io_select_calls
+
+    def io_select(*args)
+      @curb_io_select_calls = (@curb_io_select_calls || 0) + 1
+      super
+    end
+  end
+
+  STREAM_CHUNKS = 32
+  STREAM_BYTES = 8192
+
+  def setup
+    super
+    omit('The socket-action loop is not built on Windows') if WINDOWS
+    omit('Fiber scheduler API unavailable on this Ruby') unless Fiber.respond_to?(:set_scheduler) && Fiber.respond_to?(:schedule)
+    omit('socket-action perform path is not available in this build') unless Curl::Multi.private_method_defined?(:_socket_perform)
+
+    @server = CurbTinyHTTPServer.new
+    @original_default_timeout = Curl::Multi.default_timeout
+  end
+
+  def teardown
+    Curl::Multi.default_timeout = @original_default_timeout if @original_default_timeout
+    # Easy#perform under a scheduler keeps a per-thread shared multi; drop it so
+    # later tests do not inherit state from this file's Async runs.
+    state = Thread.current.thread_variable_get(:curb_scheduler_state)
+    state[:multi].close if state && state[:multi]
+    Thread.current.thread_variable_set(:curb_scheduler_state, nil)
+    @server.close if @server
+    super
+  end
+
+  def poller
+    Curl::Multi.send(:_socket_poller)
+  end
+
+  def monotonic
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+
+  def with_scheduler(scheduler)
+    previous_scheduler = Fiber.scheduler if Fiber.respond_to?(:scheduler)
+    Fiber.set_scheduler(scheduler)
+    fiber = Fiber.schedule { yield }
+    fiber.resume while fiber.alive?
+  ensure
+    Fiber.set_scheduler(previous_scheduler)
+  end
+
+  def stream_path
+    "/stream/#{STREAM_CHUNKS}/#{STREAM_BYTES}"
+  end
+
+  def expected_stream_body
+    CurbTinyHTTPServer.stream_body(STREAM_CHUNKS, STREAM_BYTES)
+  end
+
+  def perform_streams(count)
+    multi = Curl::Multi.new
+    easies = count.times.map do
+      easy = Curl::Easy.new(@server.url(stream_path))
+      multi.add(easy)
+      easy
+    end
+    multi.perform
+    easies
+  ensure
+    multi.close if multi
+  end
+
+  def eventpoll_fd_count
+    Dir.children('/proc/self/fd').count do |fd|
+      begin
+        File.readlink("/proc/self/fd/#{fd}") == 'anon_inode:[eventpoll]'
+      rescue SystemCallError
+        false
+      end
+    end
+  end
+
+  def test_socket_poller_reports_backend
+    assert_include [nil, 'epoll', 'kqueue'], poller
+    if RUBY_PLATFORM =~ /linux/
+      assert_equal 'epoll', poller
+    elsif RUBY_PLATFORM =~ /darwin|bsd/
+      assert_equal 'kqueue', poller
+    end
+  end
+
+  # With several sockets in flight, every wait goes through io_wait on one
+  # descriptor (the poller), and the io_select hook is never used.
+  def test_concurrent_streams_wait_on_single_poller_descriptor
+    omit('no kernel event queue in this build') unless poller
+    scheduler = RecordingScheduler.new
+    easies = nil
+
+    with_scheduler(scheduler) { easies = perform_streams(8) }
+
+    easies.each do |easy|
+      assert_equal 200, easy.response_code
+      assert_equal expected_stream_body, easy.body_str
+    end
+    assert_equal 0, scheduler.io_select_calls, 'io_select should not be used when a poller is available'
+    assert_operator scheduler.io_wait_calls.length, :>=, 1
+    assert_equal 1, scheduler.io_wait_calls.map(&:first).uniq.length,
+                 "expected every wait on one poller descriptor, saw #{scheduler.io_wait_calls.map(&:first).uniq.inspect}"
+    assert(scheduler.io_wait_calls.all? { |_fd, events| events == IO::READABLE },
+           'the poller descriptor should only be waited on for readability')
+  end
+
+  # Without io_select the old loop waited on one arbitrary socket for up to
+  # default_timeout, so an idle socket first in line delayed every other
+  # transfer. The poller wakes for whichever socket becomes ready.
+  def test_scheduler_without_io_select_is_not_delayed_by_idle_socket
+    omit('no kernel event queue in this build') unless poller
+    Curl::Multi.default_timeout = 2_000
+    scheduler = IoWaitOnlyScheduler.new
+    fast_elapsed = nil
+    hung = Curl::Easy.new(@server.url('/hang'))
+    fast = Curl::Easy.new(@server.url('/fast'))
+
+    started = monotonic
+    fast.on_complete do
+      fast_elapsed = monotonic - started
+      @server.release_hung(1)
+    end
+
+    with_scheduler(scheduler) do
+      multi = Curl::Multi.new
+      begin
+        multi.add(hung)
+        multi.add(fast)
+        multi.perform
+      ensure
+        multi.close
+      end
+    end
+
+    assert_equal "fast", fast.body_str
+    assert_equal "released", hung.body_str
+    assert_not_nil fast_elapsed
+    assert_operator fast_elapsed, :<, 1.0, "fast transfer waited #{fast_elapsed.round(3)}s behind an idle socket"
+  end
+
+  def test_async_concurrent_streams_do_not_use_io_select
+    omit('Async gem not available') unless defined?(Async::Scheduler)
+    omit('no kernel event queue in this build') unless poller
+    bodies = []
+    scheduler = nil
+
+    Async do |task|
+      scheduler = Fiber.scheduler
+      scheduler.singleton_class.prepend(AsyncIoSelectCounter)
+      12.times.map do
+        task.async { bodies << Curl.get(@server.url(stream_path)).body_str }
+      end.each(&:wait)
+    end
+
+    assert_equal 12, bodies.length
+    assert(bodies.all? { |body| body == expected_stream_body }, 'streamed bodies were not reassembled intact')
+    assert_equal 0, scheduler.curb_io_select_calls.to_i, 'Async#io_select starts a thread per call and should not be used'
+  end
+
+  # Completions that queue new requests close and open sockets within one
+  # perform, so descriptor numbers are reused while the poller stays open.
+  def test_socket_churn_within_one_perform
+    omit('Async gem not available') unless defined?(Async::Scheduler)
+    total = 30
+    bodies = []
+    started = monotonic
+
+    Async do
+      multi = Curl::Multi.new
+      remaining = total
+      add_next = lambda do
+        next if remaining.zero?
+        remaining -= 1
+        easy = Curl::Easy.new(@server.url('/fast'))
+        easy.on_complete do |completed|
+          bodies << completed.body_str
+          add_next.call
+        end
+        multi.add(easy)
+      end
+
+      begin
+        2.times { add_next.call }
+        multi.perform
+      ensure
+        multi.close
+      end
+    end
+
+    assert_equal Array.new(total, "fast"), bodies
+    assert_operator monotonic - started, :<, 10.0
+  end
+
+  # A peer that disconnects early or resets the connection must fail only its
+  # own transfer: readiness is reported, libcurl sees the error on read, and
+  # the loop neither spins nor stalls the healthy transfer.
+  def test_truncated_and_reset_transfers_fail_without_stalling
+    scheduler = RecordingScheduler.new
+    truncated = Curl::Easy.new(@server.url('/truncate'))
+    reset = Curl::Easy.new(@server.url('/reset'))
+    healthy = Curl::Easy.new(@server.url(stream_path))
+    started = monotonic
+
+    with_scheduler(scheduler) do
+      multi = Curl::Multi.new
+      begin
+        [truncated, reset, healthy].each { |easy| multi.add(easy) }
+        multi.perform
+      ensure
+        multi.close
+      end
+    end
+
+    assert_operator monotonic - started, :<, 5.0
+    assert_equal expected_stream_body, healthy.body_str
+    assert_not_equal 0, truncated.last_result, 'a truncated body should fail the transfer'
+    assert_not_equal 0, reset.last_result, 'a reset connection should fail the transfer'
+  end
+
+  def test_poller_descriptor_closed_after_perform
+    omit('eventpoll descriptors are only visible on Linux') unless poller == 'epoll' && File.directory?('/proc/self/fd')
+    before = eventpoll_fd_count
+
+    with_scheduler(RecordingScheduler.new) { 3.times { perform_streams(2) } }
+
+    assert_equal before, eventpoll_fd_count
+  end
+
+  def test_poller_descriptor_closed_when_perform_raises
+    omit('eventpoll descriptors are only visible on Linux') unless poller == 'epoll' && File.directory?('/proc/self/fd')
+    before = eventpoll_fd_count
+
+    assert_raise(PollerAbort) do
+      with_scheduler(RecordingScheduler.new) do
+        multi = Curl::Multi.new
+        begin
+          2.times { multi.add(Curl::Easy.new(@server.url('/delay/0.2'))) }
+          yields = 0
+          # Raise mid-transfer, after the loop has waited on the poller.
+          multi.perform do
+            yields += 1
+            raise PollerAbort, 'stop the drive loop' if yields >= 3
+          end
+        ensure
+          multi.close
+        end
+      end
+    end
+
+    assert_equal before, eventpoll_fd_count
+  end
+
+  # If wrapping the poller descriptor in an IO fails, the error propagates and
+  # the descriptor is still closed.
+  def test_poller_descriptor_closed_when_io_wrapper_fails
+    omit('eventpoll descriptors are only visible on Linux') unless poller == 'epoll' && File.directory?('/proc/self/fd')
+    wrapper_error = Class.new(StandardError)
+    original_for_fd = IO.method(:for_fd)
+    before = eventpoll_fd_count
+
+    begin
+      redefine_io_for_fd { |*| raise wrapper_error, 'IO wrapper failed' }
+      assert_raise(wrapper_error) do
+        with_scheduler(RecordingScheduler.new) do
+          multi = Curl::Multi.new
+          begin
+            2.times { multi.add(Curl::Easy.new(@server.url('/fast'))) }
+            multi.perform
+          ensure
+            multi.close
+          end
+        end
+      end
+    ensure
+      redefine_io_for_fd(original_for_fd)
+    end
+
+    assert_equal before, eventpoll_fd_count
+  end
+
+  private
+
+  def redefine_io_for_fd(callable = nil, &block)
+    previous_verbose = $VERBOSE
+    $VERBOSE = nil
+    IO.define_singleton_method(:for_fd, callable || block)
+  ensure
+    $VERBOSE = previous_verbose
+  end
+end

data/tests/tc_curl_postfield.rb

@@ -318,3 +318,42 @@ class TestCurbCurlPostfieldMultipartCoverage < Test::Unit::TestCase
     assert_match(/Content-Type: text\/plain/, body)
   end
 end
+
+class TestCurbCurlPostfieldInvalidToS < Test::Unit::TestCase
+  class BadToS
+    def to_s
+      42
+    end
+  end
+
+  def test_to_s_with_non_string_content_to_s_result_does_not_crash
+    pf = Curl::PostField.content('name', BadToS.new)
+    body = nil
+    assert_nothing_raised { body = pf.to_s }
+    assert_kind_of String, body
+    assert body.start_with?('name=')
+  end
+
+  def test_to_s_with_non_string_name_to_s_result_does_not_crash
+    pf = Curl::PostField.content(BadToS.new, 'value')
+    body = nil
+    assert_nothing_raised { body = pf.to_s }
+    assert_kind_of String, body
+  end
+
+  def test_to_s_with_content_proc_returning_non_string_does_not_crash
+    pf = Curl::PostField.content('name') { BadToS.new }
+    assert_nothing_raised { pf.to_s }
+  end
+
+  def test_to_s_content_proc_raising_error_still_propagates
+    pf = Curl::PostField.content('name') { raise 'boom' }
+    error = assert_raise(RuntimeError) { pf.to_s }
+    assert_equal 'boom', error.message
+  end
+
+  def test_to_s_still_coerces_objects_with_valid_to_s
+    pf = Curl::PostField.content('name', 42)
+    assert_equal 'name=42', pf.to_s
+  end
+end