curb 1.3.6 → 1.4.0

data/tests/tc_fiber_scheduler.rb

@@ -46,11 +46,12 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
     end
 
     def kernel_sleep(duration = nil)
-      sleep(duration || 0)
+      # A bare sleep here would re-enter this hook through the scheduler.
+      blocking_io { sleep(duration || 0) }
     end
 
     def block(_blocker, timeout = nil)
-      sleep(timeout || 0)
+      blocking_io { sleep(timeout || 0) }
       false
     end
 
@@ -69,7 +70,7 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
       if Fiber.respond_to?(:blocking)
         Fiber.blocking(&block)
       else
-        block.call
+        Fiber.new(blocking: true, &block).resume
       end
     end
   end
@@ -117,7 +118,7 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
 
     duration = Time.now - started
 
-    assert duration < THRESHOLD, "Requests did not run concurrently under fiber scheduler (#{duration}s) which exceeds the expected threshold of: #{THRESHOLD} serial time would be about: #{SERIAL_TIME_WOULD_BE_ABOUT}"
+    assert_ran_concurrently(duration)
     assert_equal ITERS, results.size
     assert_equal ITERS.times.map {200}, results
   end
@@ -148,7 +149,7 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
 
     duration = Time.now - started
 
-    assert duration < THRESHOLD, "Requests did not run concurrently under fiber scheduler (#{duration}s) which exceeds the expected threshold of: #{THRESHOLD} serial time would be about: #{SERIAL_TIME_WOULD_BE_ABOUT}"
+    assert_ran_concurrently(duration)
     assert_equal ITERS, results.size
     assert_equal ITERS.times.map {200}, results
   end
@@ -215,7 +216,7 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
       c = Curl::Easy.new(url)
       c.on_complete { result = c.code }
       m.add(c)
-      unless m.respond_to?(:_socket_perform, true)
+      unless socket_perform_supported?
         omit('socket-action perform path is not available in this build')
       end
       m.send(:_socket_perform)
@@ -231,6 +232,249 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
     end
   end
 
+  # Multi#perform must not fall back to the thread-blocking legacy fdset loop
+  # when the calling fiber runs under a scheduler: the waits have to go
+  # through the scheduler's io_wait/io_select hooks so sibling fibers run.
+  def test_multi_perform_waits_via_scheduler_hooks
+    omit('Skipping custom scheduler test on Windows') if WINDOWS
+    omit('Fiber scheduler API unavailable on this Ruby') unless fiber_scheduler_supported?
+
+    url = "http://127.0.0.1:#{@port}/test"
+    scheduler = RecordingScheduler.new
+    result = nil
+
+    with_scheduler(scheduler) do
+      m = Curl::Multi.new
+      unless socket_perform_supported?
+        omit('socket-action perform path is not available in this build')
+      end
+      c = Curl::Easy.new(url)
+      c.on_complete { result = c.code }
+      m.add(c)
+      m.perform
+    ensure
+      m.close if m
+    end
+
+    assert_equal 200, result
+    assert_operator scheduler.io_wait_events.length + scheduler.io_select_calls, :>=, 1,
+                    'Multi#perform under a fiber scheduler should wait through the scheduler hooks'
+  end
+
+  def test_multi_perform_propagates_io_wrapper_errors
+    omit('Skipping custom scheduler test on Windows') if WINDOWS
+    omit('Fiber scheduler API unavailable on this Ruby') unless fiber_scheduler_supported?
+    omit('socket-action perform path is not available in this build') unless socket_perform_supported?
+
+    url = "http://127.0.0.1:#{@port}/test"
+    scheduler = RecordingScheduler.new
+    wrapper_error = Class.new(StandardError)
+    original_for_fd = IO.method(:for_fd)
+    redefine_io_for_fd do |*_args|
+      raise wrapper_error, 'IO wrapper failed'
+    end
+
+    error = assert_raise(wrapper_error) do
+      with_scheduler(scheduler) do
+        m = Curl::Multi.new
+        m.add(Curl::Easy.new(url))
+        m.perform
+      ensure
+        m.close if m
+      end
+    end
+
+    assert_equal 'IO wrapper failed', error.message
+  ensure
+    redefine_io_for_fd(original_for_fd) if original_for_fd
+  end
+
+  # Probe-style regression tests: all previous concurrency assertions routed N
+  # requests through one multi handle, so libcurl provided the parallelism and
+  # they passed even when the wait loop blocked the whole thread. Here a
+  # sibling fiber ticks every 10ms while a single request is in flight; a
+  # blocking wait loop caps the ticks at roughly one per 100ms select chunk,
+  # while a scheduler-friendly loop lets most of the ~MIN_S/0.01 ticks happen.
+  def test_sibling_fibers_progress_during_multi_perform
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    url = "http://127.0.0.1:#{@port}/test"
+    result = nil
+
+    ticks = probe_sibling_ticks do
+      m = Curl::Multi.new
+      begin
+        c = Curl::Easy.new(url)
+        c.on_complete { result = c.code }
+        m.add(c)
+        m.perform
+      ensure
+        m.close
+      end
+    end
+
+    assert_equal 200, result
+    assert_operator ticks, :>=, SIBLING_TICKS_MIN,
+                    "sibling fiber starved during Multi#perform: #{ticks} ticks in a #{MIN_S}s transfer (~#{SIBLING_TICKS_POSSIBLE} possible)"
+  end
+
+  def test_sibling_fibers_progress_during_easy_perform
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    url = "http://127.0.0.1:#{@port}/test"
+    result = nil
+
+    ticks = probe_sibling_ticks do
+      c = Curl::Easy.new(url)
+      c.perform
+      result = c.code
+    end
+
+    assert_equal 200, result
+    assert_operator ticks, :>=, SIBLING_TICKS_MIN,
+                    "sibling fiber starved during Easy#perform: #{ticks} ticks in a #{MIN_S}s transfer (~#{SIBLING_TICKS_POSSIBLE} possible)"
+  end
+
+  def test_sibling_fibers_progress_during_multi_perform_with_multiple_sockets
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    url = "http://127.0.0.1:#{@port}/test"
+    results = []
+
+    ticks = probe_sibling_ticks do
+      m = Curl::Multi.new
+      begin
+        2.times do
+          c = Curl::Easy.new(url)
+          c.on_complete { results << c.code }
+          m.add(c)
+        end
+        m.perform
+      ensure
+        m.close
+      end
+    end
+
+    assert_equal [200, 200], results.sort
+    assert_operator ticks, :>=, SIBLING_TICKS_MIN,
+                    "sibling fiber starved during multi-fd Multi#perform: #{ticks} ticks in a #{MIN_S}s transfer (~#{SIBLING_TICKS_POSSIBLE} possible)"
+  end
+
+  def test_scheduler_timeout_interrupts_single_socket_multi_perform
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    assert_scheduler_timeout_interrupts_multi_perform(1)
+  end
+
+  def test_scheduler_timeout_interrupts_multi_socket_multi_perform
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    assert_scheduler_timeout_interrupts_multi_perform(2)
+  end
+
+  def test_multi_perform_runs_work_added_from_final_idle_yield_under_scheduler
+    return if skip_no_async
+    return if skip_no_socket_perform
+
+    url = "http://127.0.0.1:#{@port}/test"
+    completions = []
+    empty_yields = 0
+    queued_added = false
+    previous_autoclose = Curl::Multi.autoclose
+    Curl::Multi.autoclose = true
+
+    begin
+      async_run do
+        m = Curl::Multi.new
+        begin
+          first = Curl::Easy.new(url)
+          first.on_complete { completions << :first }
+          m.add(first)
+
+          m.perform do |performing_multi|
+            empty_yields += 1 if performing_multi.requests.empty?
+            if !queued_added && empty_yields >= 2
+              queued_added = true
+              queued = Curl::Easy.new(url)
+              queued.on_complete { completions << :queued }
+              performing_multi.add(queued)
+            end
+          end
+        ensure
+          m.close
+        end
+      end
+
+      assert queued_added, 'test should add queued work from the final idle yield'
+      assert_equal [:first, :queued], completions
+    ensure
+      Curl::Multi.autoclose = previous_autoclose
+    end
+  end
+
+  # A scheduler that implements io_wait but not the optional io_select hook
+  # must neither busy-loop nor fall back to a thread-blocking wait. The broken
+  # loop spins without an interrupt checkpoint, so the probe runs in a
+  # subprocess guarded by a parent-side SIGKILL fuse.
+  def test_socket_perform_progresses_when_scheduler_lacks_io_select
+    omit('Skipping custom scheduler test on Windows') if WINDOWS
+    omit('Fiber scheduler API unavailable on this Ruby') unless fiber_scheduler_supported?
+    unless socket_perform_supported?
+      omit('socket-action perform path is not available in this build')
+    end
+
+    script = File.join(File.dirname(__FILE__), 'io_select_less_scheduler_probe.rb')
+    out_r, out_w = IO.pipe
+    pid = Process.spawn(RbConfig.ruby, '-I', $LIBDIR, '-I', $EXTDIR, script, @port.to_s,
+                        :out => out_w, :err => out_w)
+    out_w.close
+
+    fuse_seconds = 30
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + fuse_seconds
+    status = nil
+    loop do
+      _, status = Process.waitpid2(pid, Process::WNOHANG)
+      break if status
+      if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+        begin
+          Process.kill('KILL', pid)
+        rescue Errno::ESRCH
+        end
+        begin
+          Process.waitpid(pid)
+        rescue Errno::ECHILD
+        end
+        flunk "scheduler without io_select busy-looped _socket_perform (no exit within #{fuse_seconds}s; killed)"
+      end
+      sleep 0.05
+    end
+
+    output = out_r.read
+    assert status.success?, "io_select-less scheduler subprocess failed: #{output}"
+    match = /^OK ticks=(\d+)$/.match(output)
+    assert_not_nil match, "unexpected io_select-less scheduler output: #{output}"
+    assert_operator match[1].to_i, :>=, 2,
+                    "scheduler without io_select did not make sibling-fiber progress: #{output}"
+  ensure
+    if pid && !status
+      begin
+        Process.kill('KILL', pid)
+      rescue Errno::ESRCH
+      end
+      begin
+        Process.waitpid(pid)
+      rescue Errno::ECHILD
+      end
+    end
+    out_r.close if out_r && !out_r.closed?
+    out_w.close if out_w && !out_w.closed?
+  end
+
   def test_multi_reuse_after_scheduler_perform
     unless HAS_ASYNC
       warn 'Skipping fiber scheduler test (Async gem not available)'
@@ -490,6 +734,97 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
     end
   end
 
+  # Records unblock calls in place of a real scheduler.
+  class UnblockRecorder
+    attr_reader :unblocked
+
+    def initialize
+      @unblocked = []
+    end
+
+    def unblock(blocker, _fiber)
+      @unblocked << blocker
+    end
+  end
+
+  # Enumerating every waiter on every drive-loop pass made release cost
+  # O(waiters) per pass; only the queued completions should be visited.
+  class NonEnumerableWaiters < Hash
+    %i[each each_pair each_value each_key].each do |name|
+      define_method(name) { |*| raise 'release_scheduler_waiters must not enumerate all waiters' }
+    end
+  end
+
+  def scheduler_test_waiter(scheduler)
+    {completed: true, done: false, error: nil, fiber: Fiber.new {}, scheduler: scheduler}
+  end
+
+  def test_release_scheduler_waiters_visits_only_queued_completions
+    state = Curl.scheduler_state
+    recorder = UnblockRecorder.new
+    waiters = NonEnumerableWaiters.new
+    1_000.times { |i| waiters[i] = scheduler_test_waiter(recorder).merge(completed: false) }
+    finished = scheduler_test_waiter(recorder)
+    waiters[:finished] = finished
+    state[:waiters] = waiters
+    state[:completed] << [:finished, finished]
+
+    Curl.release_scheduler_waiters(state)
+
+    assert_equal [finished], recorder.unblocked
+    assert finished[:done]
+    assert_empty state[:completed]
+  ensure
+    cleanup_scheduler_state
+  end
+
+  # A waiter whose perform already returned (it raised) is unregistered and its
+  # fiber may be blocked on something else, so a late completion must not
+  # unblock it.
+  def test_release_scheduler_waiters_skips_unregistered_waiter
+    state = Curl.scheduler_state
+    recorder = UnblockRecorder.new
+    stale = scheduler_test_waiter(recorder)
+    state[:completed] << [:stale, stale]
+
+    Curl.release_scheduler_waiters(state)
+
+    assert_empty recorder.unblocked
+    assert_equal false, stale[:done]
+    assert_empty state[:completed]
+  ensure
+    cleanup_scheduler_state
+  end
+
+  # The transfer whose callback raised the deferred exception receives that
+  # error rather than a normal completion; it stays queued so it is completed
+  # normally if the exception source changes.
+  def test_release_scheduler_waiters_holds_back_deferred_exception_source
+    state = Curl.scheduler_state
+    recorder = UnblockRecorder.new
+    source = scheduler_test_waiter(recorder)
+    sibling = scheduler_test_waiter(recorder)
+    state[:waiters][:source] = source
+    state[:waiters][:sibling] = sibling
+    state[:completed] << [:source, source] << [:sibling, sibling]
+    state[:multi].instance_variable_set(:@__curb_deferred_exception_source_id, :source)
+
+    Curl.release_scheduler_waiters(state)
+
+    assert_equal [sibling], recorder.unblocked
+    assert_equal false, source[:done]
+    assert_equal [[:source, source]], state[:completed]
+
+    state[:multi].remove_instance_variable(:@__curb_deferred_exception_source_id)
+    Curl.release_scheduler_waiters(state)
+
+    assert_equal [sibling, source], recorder.unblocked
+    assert source[:done]
+    assert_empty state[:completed]
+  ensure
+    cleanup_scheduler_state
+  end
+
   def test_drain_scheduler_pending_does_not_drop_work_rejected_during_deferred_abort
     state = Curl.scheduler_state
     easy = Curl::Easy.new("http://127.0.0.1:#{@port}/test")
@@ -508,7 +843,79 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
     cleanup_scheduler_state
   end
 
+  # Sibling-fiber probe bounds: ~MIN_S/0.01 ticks are possible while the
+  # request is in flight. A thread-blocking wait loop yields only one tick per
+  # 100ms select chunk (2-3 for MIN_S=0.25); require a comfortably higher
+  # count while leaving slack for slow CI hosts.
+  SIBLING_TICKS_POSSIBLE = (MIN_S / 0.01).to_i
+  SIBLING_TICKS_MIN = 8
+
   private
+  def redefine_io_for_fd(callable = nil, &block)
+    previous_verbose = $VERBOSE
+    $VERBOSE = nil
+    if callable
+      IO.define_singleton_method(:for_fd, callable)
+    else
+      IO.define_singleton_method(:for_fd, &block)
+    end
+  ensure
+    $VERBOSE = previous_verbose
+  end
+
+  def assert_scheduler_timeout_interrupts_multi_perform(easy_count)
+    url = "http://127.0.0.1:#{@port}/test"
+    timeout_error = nil
+    started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    async_run do |task|
+      begin
+        task.with_timeout(0.05) do
+          m = Curl::Multi.new
+          begin
+            easy_count.times { m.add(Curl::Easy.new(url)) }
+            m.perform
+          ensure
+            m.close
+          end
+        end
+      rescue Async::TimeoutError => error
+        timeout_error = error
+      end
+    end
+
+    elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+    assert_kind_of Async::TimeoutError, timeout_error,
+                   "scheduler timeout was swallowed for #{easy_count} easy handle(s)"
+    assert_operator elapsed, :<, MIN_S * 0.8,
+                    "scheduler cancellation took #{elapsed}s for #{easy_count} easy handle(s)"
+  end
+
+  # Runs the block in one fiber while a sibling fiber ticks every 10ms;
+  # returns how many ticks the sibling managed before the block finished.
+  def probe_sibling_ticks
+    ticks = 0
+    done = false
+
+    async_run do |top|
+      ticker = top.async do
+        until done
+          sleep 0.01
+          ticks += 1 unless done
+        end
+      end
+
+      top.async do
+        yield
+      ensure
+        done = true
+      end.wait
+      ticker.wait
+    end
+
+    ticks
+  end
+
   def skip_no_async
     if WINDOWS
       warn 'Skipping fiber scheduler tests on Windows'
@@ -525,6 +932,26 @@ class TestCurbFiberScheduler < Test::Unit::TestCase
     false
   end
 
+  def skip_no_socket_perform
+    unless socket_perform_supported?
+      warn 'Skipping fiber scheduler test (socket-action perform path unavailable)'
+      return true
+    end
+    false
+  end
+
+  # Wall-clock bounds are not reliable under valgrind, which slows and
+  # serializes threads unpredictably; there the requests and their results are
+  # still checked under memcheck, only the duration bound is skipped.
+  def assert_ran_concurrently(duration)
+    return if curb_under_valgrind?
+    assert duration < THRESHOLD, "Requests did not run concurrently under fiber scheduler (#{duration}s) which exceeds the expected threshold of: #{THRESHOLD} serial time would be about: #{SERIAL_TIME_WOULD_BE_ABOUT}"
+  end
+
+  def socket_perform_supported?
+    Curl::Multi.private_method_defined?(:_socket_perform)
+  end
+
   def async_run(&block)
     # Prefer newer Async.run to avoid deprecated scheduler.async path.
     if defined?(Async) && Async.respond_to?(:run)

data/tests/tc_ftp_options.rb

@@ -1,6 +1,23 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
 
 class TestCurbFtpOptions < Test::Unit::TestCase
+  SCALAR_AND_STRING_OPTIONS = {
+    :ftpport => "-",
+    :dirlistonly => true,
+    :append => true,
+    :ftp_use_eprt => false,
+    :ftp_use_epsv => false,
+    :ftp_use_pret => true,
+    :ftp_create_missing_dirs => true,
+    :ftp_response_timeout => 30,
+    :ftp_alternative_to_user => "anonymous",
+    :ftp_skip_pasv_ip => true,
+    :ftpsslauth => 0,
+    :ftp_ssl_ccc => 0,
+    :ftp_account => "account",
+    :ftp_filemethod => 0
+  }.freeze
+
   # Ensure FTP-related set(:option, ...) mappings are accepted and do not raise
   # a TypeError (they used to be unsupported in setopt dispatch).
   def test_can_set_ftp_listing_related_flags
@@ -15,6 +32,47 @@ class TestCurbFtpOptions < Test::Unit::TestCase
     end
   end
 
+  def test_can_set_detected_scalar_and_string_ftp_options
+    c = Curl::Easy.new('ftp://example.com/')
+
+    SCALAR_AND_STRING_OPTIONS.each do |name, value|
+      constant = "CURLOPT_#{name.to_s.upcase}"
+      next unless Curl.const_defined?(constant)
+
+      assert_nothing_raised("setting #{name}") { c.set(name, value) }
+    end
+  end
+
+  def test_issue_481_ftp_create_missing_dirs_accessor_and_set
+    omit('CURLOPT_FTP_CREATE_MISSING_DIRS is not available in this build') unless
+      Curl.const_defined?(:CURLOPT_FTP_CREATE_MISSING_DIRS)
+
+    c = Curl::Easy.new('ftp://example.com/')
+
+    assert_respond_to(c, :ftp_create_missing_dirs=)
+    assert_respond_to(c, :ftp_create_missing_dirs)
+    assert_nil(c.ftp_create_missing_dirs)
+
+    c.ftp_create_missing_dirs = true
+    assert_equal(true, c.ftp_create_missing_dirs)
+
+    assert_equal(1, c.set(:ftp_create_missing_dirs, 1))
+    assert_equal(1, c.ftp_create_missing_dirs)
+
+    retry_mode = if Curl.const_defined?(:CURLFTP_CREATE_DIR_RETRY)
+      Curl::CURLFTP_CREATE_DIR_RETRY
+    else
+      2
+    end
+    c.set(:ftp_create_missing_dirs, retry_mode)
+    assert_equal(retry_mode, c.ftp_create_missing_dirs)
+    assert_equal(retry_mode, c.clone.ftp_create_missing_dirs)
+
+    settings = c.reset
+    assert_equal(retry_mode, settings[:ftp_create_missing_dirs])
+    assert_nil(c.ftp_create_missing_dirs)
+  end
+
   # Setting ftp_commands remains supported for control-connection commands.
   def test_can_assign_ftp_commands
     c = Curl::Easy.new('ftp://example.com/')

data/tests/tc_gc_compact.rb

@@ -96,6 +96,59 @@ class TestGcCompact < Test::Unit::TestCase
     end
   end
 
+  # Regression: after compaction relocates a persistent Curl::Easy and its old
+  # heap slot is recycled by another T_DATA object, completion dispatch (which
+  # reads the rbce->self back-reference via CURLINFO_PRIVATE) must still
+  # resolve to the correct Curl::Easy. Without pinning in curl_easy_mark this
+  # raises "TypeError: wrong argument type Curl::Upload (expected Curl::Easy)"
+  # on the first round, or defers it onto the multi and raises it from a later
+  # request.
+  def test_completion_dispatch_survives_compact_and_slot_reuse
+    omit('needs GC.verify_compaction_references') unless GC.respond_to?(:verify_compaction_references)
+
+    # Heap-only reference: holding the easy in a local would pin it to the
+    # stack and mask the bug.
+    holder = { easy: Curl::Easy.new($TEST_URL) }
+    holder[:easy].timeout = 5
+    completed = nil
+    holder[:easy].on_complete { |e| completed = e }
+    holder[:easy].http(:GET)
+    assert_same holder[:easy], completed
+    expected_body = holder[:easy].body_str
+    refute_empty expected_body
+
+    ruby_version = RUBY_VERSION.split('.').first(2).map(&:to_i)
+    heap_expansion = if (ruby_version <=> [3, 2]) >= 0
+                       { expand_heap: true }
+                     else
+                       { double_heap: true }
+                     end
+
+    3.times do
+      # Relocate every movable object, then recycle freed slots with other
+      # T_DATA instances so a stale back-reference cannot fail soft.
+      GC.verify_compaction_references(toward: :empty, **heap_expansion)
+      churn = Array.new(50_000) { Curl::Upload.new }
+
+      completed = nil
+      holder[:easy].http(:GET)
+
+      multi = holder[:easy].multi
+      if multi&.instance_variable_defined?(:@__curb_deferred_exception)
+        raise multi.instance_variable_get(:@__curb_deferred_exception)
+      end
+      assert_same holder[:easy], completed
+      assert_equal expected_body, holder[:easy].body_str
+      churn.clear
+    end
+  ensure
+    begin
+      holder[:easy]&.close
+    rescue StandardError
+      nil
+    end
+  end
+
   private
 
   def run_multi_perform_compact_iteration