curb 1.3.6 → 1.4.0

data/tests/io_select_less_scheduler_probe.rb

@@ -0,0 +1,105 @@
+# Reproduces the FIBER_TODO.md [Critical] finding: a Fiber scheduler that
+# implements io_wait/kernel_sleep/block/unblock but NOT the optional io_select
+# hook. rb_fiber_scheduler_io_select returns Qundef (without waiting) for such
+# schedulers; if the multi-fd wait path of Curl::Multi#_socket_perform treats
+# that as a completed wait, the drive loop spins at 100% CPU with no interrupt
+# checkpoint and this script never exits — it cannot even service SIGTERM.
+#
+# Run by tc_fiber_scheduler.rb in a subprocess guarded by a SIGKILL fuse.
+# Usage: ruby io_select_less_scheduler_probe.rb <port>
+#   <port> must serve HTTP GET /test (see BugTestServerSetupTeardown).
+
+require 'fiber'
+require 'curb'
+
+port = Integer(ARGV[0])
+url = "http://127.0.0.1:#{port}/test"
+
+class NoIoSelectScheduler
+  def fiber(&block)
+    Fiber.new(blocking: false, &block)
+  end
+
+  def io_wait(io, events, timeout = nil)
+    readers = (events & IO::READABLE) != 0 ? [io] : nil
+    writers = (events & IO::WRITABLE) != 0 ? [io] : nil
+    deadline = monotonic_now + timeout if timeout
+
+    loop do
+      readable, writable = IO.select(readers, writers, nil, 0)
+      ready = 0
+      ready |= IO::READABLE if readable && !readable.empty?
+      ready |= IO::WRITABLE if writable && !writable.empty?
+      return ready unless ready.zero?
+      return false if deadline && monotonic_now >= deadline
+
+      Fiber.yield
+    end
+  end
+
+  def kernel_sleep(duration = nil)
+    deadline = monotonic_now + (duration || 0)
+    Fiber.yield while monotonic_now < deadline
+    duration
+  end
+
+  def block(_blocker, timeout = nil)
+    kernel_sleep(timeout)
+    false
+  end
+
+  def unblock(*); end
+
+  def close; end
+
+  def fiber_interrupt(*); end
+
+  private
+
+  def monotonic_now
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end
+
+results = []
+ticks = 0
+done = false
+
+Fiber.set_scheduler(NoIoSelectScheduler.new)
+fiber = Fiber.schedule do
+  multi = Curl::Multi.new
+  begin
+    # Two easies so the drive loop tracks more than one socket and takes the
+    # multi-fd (io_select) wait branch.
+    2.times do
+      easy = Curl::Easy.new(url)
+      easy.on_complete { results << easy.response_code }
+      multi.add(easy)
+    end
+    multi.send(:_socket_perform)
+  ensure
+    multi.close
+    done = true
+  end
+end
+ticker = Fiber.schedule do
+  until done
+    sleep 0.005
+    ticks += 1 unless done
+  end
+end
+
+while fiber.alive? || ticker.alive?
+  fiber.resume if fiber.alive?
+  ticker.resume if ticker.alive?
+  sleep 0.001
+end
+Fiber.set_scheduler(nil)
+
+if results == [200, 200]
+  puts "OK ticks=#{ticks}"
+  exit 0
+else
+  puts "FAILED results=#{results.inspect}"
+  exit 1
+end

data/tests/tc_curl_easy.rb

@@ -1,8 +1,14 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
-class FooNoToS 
+class FooNoToS
   undef to_s
 end
 
+class FooBadToS
+  def to_s
+    42
+  end
+end
+
 class TestCurbCurlEasy < Test::Unit::TestCase
   def test_global_reset
     Curl.get($TEST_URL)
@@ -46,6 +52,7 @@ class TestCurbCurlEasy < Test::Unit::TestCase
 
     http = Curl::Easy.new(url)
     http.dns_cache_timeout = 0
+    http.proxy_url = ""
     http.resolve = [mapping]
     http.get
     assert_match(/GET/, http.body)
@@ -559,7 +566,24 @@ class TestCurbCurlEasy < Test::Unit::TestCase
       assert_equal "one\000two three", c.unescape("one%00two%20three")
     end
   end
-  
+
+  def test_escape_with_non_string_to_s_result_does_not_crash
+    c = Curl::Easy.new
+    result = nil
+    # #to_s returning a non-String (42, a Fixnum) makes rb_obj_as_string
+    # fall back to the default Object#to_s representation rather than "42" -
+    # the point of this test is that it doesn't crash, not the exact value.
+    assert_nothing_raised { result = c.escape(FooBadToS.new) }
+    assert_kind_of String, result
+  end
+
+  def test_unescape_with_non_string_to_s_result_raises_type_error
+    c = Curl::Easy.new
+    # unescape expects an actual String (or something #to_str-coercible);
+    # it must raise cleanly rather than crash on a bad #to_s.
+    assert_raise(TypeError) { c.unescape(FooBadToS.new) }
+  end
+
   def test_headers
     c = Curl::Easy.new($TEST_URL)
     
@@ -744,7 +768,23 @@ class TestCurbCurlEasy < Test::Unit::TestCase
     c.http_auth_types = nil
     assert_nil c.http_auth_types
   end
-  
+
+  def test_http_auth_types_with_non_string_to_s_result_does_not_crash
+    c = Curl::Easy.new
+    assert_nothing_raised { c.http_auth_types = FooBadToS.new }
+    assert_nothing_raised { c.http_auth_types = [FooBadToS.new, :basic] }
+    assert_equal Curl::CURLAUTH_BASIC, c.http_auth_types
+  end
+
+  def test_http_auth_types_array_of_symbols
+    c = Curl::Easy.new
+    c.http_auth_types = [:basic, :digest]
+    assert_equal Curl::CURLAUTH_BASIC | Curl::CURLAUTH_DIGEST, c.http_auth_types
+
+    c.send(:http_auth_types=, :basic, :ntlm)
+    assert_equal Curl::CURLAUTH_BASIC | Curl::CURLAUTH_NTLM, c.http_auth_types
+  end
+
   def test_proxy_auth_types_01
     c = Curl::Easy.new($TEST_URL)
     
@@ -756,7 +796,23 @@ class TestCurbCurlEasy < Test::Unit::TestCase
     c.proxy_auth_types = nil
     assert_nil c.proxy_auth_types
   end
-  
+
+  def test_proxy_auth_types_symbols
+    c = Curl::Easy.new
+
+    c.proxy_auth_types = :basic
+    assert_equal Curl::CURLAUTH_BASIC, c.proxy_auth_types
+
+    c.proxy_auth_types = [:basic, :ntlm]
+    assert_equal Curl::CURLAUTH_BASIC | Curl::CURLAUTH_NTLM, c.proxy_auth_types
+
+    c.send(:proxy_auth_types=, :basic, :digest)
+    assert_equal Curl::CURLAUTH_BASIC | Curl::CURLAUTH_DIGEST, c.proxy_auth_types
+
+    assert_nothing_raised { c.proxy_auth_types = [FooBadToS.new, :basic] }
+    assert_equal Curl::CURLAUTH_BASIC, c.proxy_auth_types
+  end
+
   def test_max_redirects_01
     c = Curl::Easy.new($TEST_URL)
     

data/tests/tc_curl_easy_cookielist.rb

@@ -46,14 +46,17 @@ class TestCurbCurlEasyCookielist < Test::Unit::TestCase
     expires = (Date.today + 2).to_datetime
     easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies"
     easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c1=v1; domain=localhost; expires=#{expires.httpdate};")
-    easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }])
+    # c2 has no Domain attribute so it is host-only on every libcurl version.
+    # With domain=localhost, curl >= 8.22 built with libpsl makes it host-only
+    # (localhost is a public suffix) while older curl tail-matches (#483).
+    easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', expires: expires.httpdate, path: '/' }])
     easy.perform
     easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
     easy.post_body = nil
     easy.perform
 
     assert !easy.enable_cookies?
-    assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
+    assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", "localhost\tFALSE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
     assert_equal 'c2=v2; c1=v1', easy.body_str
   end
 
@@ -245,14 +248,15 @@ class TestCurbCurlEasyCookielist < Test::Unit::TestCase
       expires = (Date.today + 2).to_datetime
       easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies"
       easy.setopt(Curl::CURLOPT_COOKIELIST, command)
-      easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }])
+      # host-only c2, see test_setopt_cookielist_enables_cookie_engine (#483)
+      easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', expires: expires.httpdate, path: '/' }])
       easy.perform
       easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
       easy.post_body = nil
       easy.perform
 
       assert !easy.enable_cookies?
-      assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
+      assert_equal ["localhost\tFALSE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
       assert_equal 'c2=v2', easy.body_str
     end
   end

data/tests/tc_curl_easy_resolve.rb

@@ -22,6 +22,7 @@ class TestCurbCurlEasyResolve < Test::Unit::TestCase
     mapping = "#{host}:#{TestServlet.port}:127.0.0.1"
 
     @easy.url = "http://#{host}:#{TestServlet.port}#{TestServlet.path}"
+    @easy.proxy_url = ""
     @easy.dns_cache_timeout = 0
     @easy.set(:resolve, [mapping])
 
@@ -39,6 +40,7 @@ class TestCurbCurlEasyResolve < Test::Unit::TestCase
     entry.define_singleton_method(:to_s) { mapping }
 
     @easy.url = "http://#{host}:#{TestServlet.port}#{TestServlet.path}"
+    @easy.proxy_url = ""
     @easy.dns_cache_timeout = 0
     @easy.resolve = [entry]
 

data/tests/tc_curl_multi.rb

@@ -586,6 +586,35 @@ class TestCurbCurlMulti < Test::Unit::TestCase
     end
   end
 
+  # Only easies attached by the raising callback itself are withdrawn. Work
+  # queued by an earlier, successful callback in the same perform was already
+  # attached when the failing callback started, so it keeps running and the
+  # deferred exception is raised once it drains.
+  def test_multi_perform_keeps_work_added_by_earlier_callback_when_later_callback_raises
+    server = CurbTinyHTTPServer.new
+    multi = Curl::Multi.new
+    first = Curl::Easy.new(server.url('/fast'))
+    failing = Curl::Easy.new(server.url('/delay/0.15'))
+    kept = Curl::Easy.new(server.url('/delay/0.4'))
+
+    first.on_complete { multi.add(kept) }
+    failing.on_complete { raise "complete blew up" }
+
+    error = assert_raise(Curl::Err::AbortedByCallbackError) do
+      multi.add(first)
+      multi.add(failing)
+      multi.perform
+    end
+
+    assert_equal "complete blew up", error.message
+    assert_equal 0, kept.last_result
+    assert_equal "delayed", kept.body_str,
+                 "work added by an earlier successful callback should finish, not be withdrawn with the failing callback's additions"
+  ensure
+    multi.close if multi
+    server.close if server
+  end
+
   def test_multi_perform_does_not_start_work_added_within_on_complete_after_on_body_exception
     with_queue_refill_test_server do |port, hits|
       multi = Curl::Multi.new

data/tests/tc_curl_multi_poll.rb

@@ -0,0 +1,223 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+# Coverage for the blocking (non-scheduler) Curl::Multi#perform wait loop:
+# descriptors above FD_SETSIZE, long waits, and interrupting those waits.
+class TestCurbCurlMultiPoll < Test::Unit::TestCase
+  FD_SETSIZE = 1024
+  HIGH_FD_TARGET = FD_SETSIZE + 76
+
+  class PollInterrupt < StandardError; end
+
+  def setup
+    super
+    @server = CurbTinyHTTPServer.new
+    @original_default_timeout = Curl::Multi.default_timeout
+  end
+
+  def teardown
+    Curl::Multi.default_timeout = @original_default_timeout
+    @server.close if @server
+    super
+  end
+
+  def monotonic
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+
+  # Under valgrind these bounds can no longer tell the old fixed-sleep loop
+  # from ordinary overhead. Skip only the bounds; the requests and their
+  # assertions still run under memcheck.
+  def assert_faster_than(limit, elapsed, message)
+    return if curb_under_valgrind?
+    assert_operator elapsed, :<, limit, message
+  end
+
+  # Occupy low descriptor numbers so every socket libcurl opens afterwards is
+  # numbered above FD_SETSIZE and cannot be represented in an fd_set.
+  def with_fds_above_fd_setsize
+    omit('fd numbering differs on Windows') if WINDOWS
+    # Before 7.68 there is no curl_multi_poll and perform falls back to
+    # select(), which cannot represent descriptors above FD_SETSIZE.
+    omit('libcurl < 7.68 has no curl_multi_poll') if Gem::Version.new(Curl::CURL_VERSION[/\A\d+(\.\d+)*/]) < Gem::Version.new('7.68.0')
+
+    soft, hard = Process.getrlimit(:NOFILE)
+    needed = HIGH_FD_TARGET + 256
+    if soft < needed
+      omit("RLIMIT_NOFILE hard limit #{hard} is too low") if hard != Process::RLIM_INFINITY && hard < needed
+      Process.setrlimit(:NOFILE, needed, hard)
+    end
+
+    hog = []
+    hog << File.open(File::NULL) while hog.empty? || hog.last.fileno < HIGH_FD_TARGET
+
+    probe = Socket.new(:INET, :STREAM)
+    assert_operator probe.fileno, :>, FD_SETSIZE, "new sockets should be numbered above FD_SETSIZE"
+    probe.close
+
+    yield
+  ensure
+    hog.each(&:close) if hog
+    Process.setrlimit(:NOFILE, soft, hard) if soft && Process.getrlimit(:NOFILE)[0] != soft
+  end
+
+  # A request whose socket is above FD_SETSIZE used to be invisible to the
+  # curl_multi_fdset/select loop, which then fell back to fixed 100ms sleeps,
+  # so each request took at least 100ms. Ten sequential local requests would
+  # need a full second.
+  def test_easy_perform_with_socket_fds_above_fd_setsize
+    with_fds_above_fd_setsize do
+      started = monotonic
+      10.times do
+        assert_equal "fast", Curl.get(@server.url('/fast')).body_str
+      end
+      elapsed = monotonic - started
+
+      assert_faster_than 0.6, elapsed, "10 local requests took #{elapsed.round(3)}s with high-numbered sockets"
+    end
+  end
+
+  # Batches of concurrent requests share each 100ms sleep of the old loop, so
+  # run several batches to make that fixed floor (>= 1s total) visible.
+  def test_multi_concurrent_requests_with_socket_fds_above_fd_setsize
+    with_fds_above_fd_setsize do
+      started = monotonic
+      5.times do |batch|
+        multi = Curl::Multi.new
+        easies = 20.times.map do |i|
+          easy = Curl::Easy.new(@server.url("/fast?#{batch}-#{i}"))
+          multi.add(easy)
+          easy
+        end
+
+        multi.perform
+
+        easies.each do |easy|
+          assert_equal 200, easy.response_code
+          assert_equal "fast", easy.body_str
+        end
+      ensure
+        multi.close if multi
+      end
+      elapsed = monotonic - started
+
+      assert_faster_than 0.8, elapsed, "5 batches of 20 concurrent local requests took #{elapsed.round(3)}s with high-numbered sockets"
+    end
+  end
+
+  # With a large default_timeout, readiness on the transfer's socket must still
+  # wake the wait immediately rather than waiting out the timeout.
+  def test_long_default_timeout_still_wakes_on_socket_activity
+    Curl::Multi.default_timeout = 5_000
+
+    multi = Curl::Multi.new
+    fast = Curl::Easy.new(@server.url('/fast'))
+    delayed = Curl::Easy.new(@server.url('/delay/0.3'))
+    multi.add(fast)
+    multi.add(delayed)
+
+    started = monotonic
+    multi.perform
+    elapsed = monotonic - started
+
+    assert_equal "fast", fast.body_str
+    assert_equal "delayed", delayed.body_str
+    assert_operator elapsed, :<, 2.0, "perform waited #{elapsed.round(3)}s; the 5s default_timeout should not delay completion"
+  ensure
+    multi.close if multi
+  end
+
+  # The block passed to perform is yielded each time the wait times out, so a
+  # small default_timeout keeps it running while a transfer is idle.
+  def test_perform_block_yields_while_waiting
+    Curl::Multi.default_timeout = 20
+
+    multi = Curl::Multi.new
+    easy = Curl::Easy.new(@server.url('/delay/0.4'))
+    multi.add(easy)
+
+    yields = 0
+    multi.perform { yields += 1 }
+
+    assert_equal "delayed", easy.body_str
+    assert_operator yields, :>=, 5, "perform block ran #{yields} times during a 400ms idle transfer"
+  ensure
+    multi.close if multi
+  end
+
+  def run_interruptible_hang
+    Curl::Multi.default_timeout = 30_000
+    errors = Queue.new
+
+    worker = Thread.new do
+      Thread.current.report_on_exception = false
+      begin
+        Curl.get(@server.url('/hang'))
+        errors << nil
+      rescue Exception => e
+        errors << e
+        raise
+      end
+    end
+
+    assert_equal '/hang', curb_pop_within(@server.requests, 5), "server never received the request"
+    # Give the worker a moment to settle into the blocking wait.
+    sleep 0.1
+
+    started = monotonic
+    yield worker
+    joined = worker.join(3) rescue worker
+    elapsed = monotonic - started
+
+    assert_not_nil joined, "worker was still blocked 3s after being interrupted"
+    assert_operator elapsed, :<, 2.0, "interrupt took #{elapsed.round(3)}s to stop the wait"
+    errors.empty? ? nil : errors.pop
+  ensure
+    if worker&.alive?
+      worker.kill
+      worker.join(1)
+    end
+  end
+
+  def test_thread_raise_interrupts_long_wait
+    error = run_interruptible_hang { |worker| worker.raise(PollInterrupt, "stop waiting") }
+    assert_kind_of PollInterrupt, error
+  end
+
+  def test_thread_kill_interrupts_long_wait
+    run_interruptible_hang { |worker| worker.kill }
+  end
+
+  # A trapped signal must run its handler promptly during a long wait, and the
+  # wait must then resume so the transfer still completes. The signal can
+  # interrupt the poll with EINTR directly, so this guards against treating
+  # that as a failed wait rather than exercising the unblocking function
+  # (the Thread#raise/#kill tests cover that).
+  def test_signal_trap_runs_during_long_wait_and_transfer_completes
+    omit('POSIX signals are not available on Windows') if WINDOWS
+    Curl::Multi.default_timeout = 30_000
+
+    handled_at = nil
+    previous = trap('USR2') { handled_at = monotonic }
+    trapped = true
+    sent_at = nil
+
+    signaller = Thread.new do
+      curb_pop_within(@server.requests, 5)
+      sleep 0.1
+      sent_at = monotonic
+      Process.kill('USR2', Process.pid)
+      sleep 0.3
+      @server.release_hung(1)
+    end
+
+    easy = Curl.get(@server.url('/hang'))
+    signaller.join(5)
+
+    assert_equal "released", easy.body_str
+    assert_not_nil handled_at, "USR2 handler never ran"
+    assert_operator handled_at - sent_at, :<, 0.25, "signal handler was delayed #{(handled_at - sent_at).round(3)}s by the wait"
+  ensure
+    trap('USR2', previous || 'DEFAULT') if trapped
+    signaller&.kill
+  end
+end