curb 1.2.1 → 1.3.0

data/lib/curl.rb

@@ -6,6 +6,199 @@ require 'uri'
 
 # expose shortcut methods
 module Curl
+  def self.scheduler_active?
+    Fiber.respond_to?(:scheduler) && !Fiber.scheduler.nil?
+  end
+
+  def self.deferred_exception_source_id(state)
+    return unless state[:multi].instance_variable_defined?(:@__curb_deferred_exception_source_id)
+
+    state[:multi].instance_variable_get(:@__curb_deferred_exception_source_id)
+  end
+
+  def self.scheduler_waiter_blocking_supported?
+    scheduler = Fiber.scheduler
+    scheduler && scheduler.respond_to?(:block) && scheduler.respond_to?(:unblock)
+  end
+
+  def self.wake_scheduler_waiter(waiter)
+    fiber = waiter[:fiber]
+    scheduler = waiter[:scheduler]
+    return unless fiber&.alive? && scheduler&.respond_to?(:unblock)
+
+    scheduler.unblock(waiter, fiber)
+  end
+
+  def self.complete_scheduler_waiter(waiter)
+    return if waiter[:done]
+
+    waiter[:done] = true
+    wake_scheduler_waiter(waiter)
+  end
+
+  def self.fail_scheduler_waiter(waiter, error)
+    return if waiter[:error]
+
+    waiter[:error] = error
+    wake_scheduler_waiter(waiter)
+  end
+
+  def self.release_scheduler_error(state, error)
+    source_waiter = state[:waiters][deferred_exception_source_id(state)]
+
+    if source_waiter
+      fail_scheduler_waiter(source_waiter, error)
+    else
+      state[:error] = error
+      state[:waiters].each_value { |waiter| wake_scheduler_waiter(waiter) }
+    end
+  end
+
+  def self.block_scheduler_waiter(waiter)
+    unless scheduler_waiter_blocking_supported?
+      sleep 0
+      return
+    end
+
+    waiter[:fiber] = Fiber.current
+    waiter[:scheduler] ||= Fiber.scheduler
+    return if waiter[:done] || waiter[:error]
+
+    waiter[:scheduler].block(waiter, nil)
+  ensure
+    waiter[:fiber] = nil if waiter[:fiber].equal?(Fiber.current)
+  end
+
+  def self.scheduler_yield
+    scheduler = Fiber.scheduler
+
+    if scheduler&.respond_to?(:kernel_sleep)
+      scheduler.kernel_sleep(0)
+    else
+      sleep 0
+    end
+  end
+
+  def self.release_scheduler_waiters(state)
+    source_id = deferred_exception_source_id(state)
+
+    state[:waiters].each do |easy_id, waiter|
+      next if source_id == easy_id
+
+      complete_scheduler_waiter(waiter) if waiter[:completed]
+    end
+  end
+
+  def self.perform_with_scheduler(easy)
+    state = scheduler_state
+    waiter = {completed: false, done: false, error: nil, fiber: nil, scheduler: Fiber.scheduler}
+    state[:waiters][easy.object_id] = waiter
+    previous_complete = easy.on_complete do |completed_easy|
+      previous_complete.call(completed_easy) if previous_complete
+      waiter[:completed] = true
+    end
+
+    state[:pending] << easy
+    ensure_scheduler_driver(state)
+
+    until waiter[:done]
+      raise waiter[:error] if waiter[:error]
+      raise state[:error] if state[:error]
+      block_scheduler_waiter(waiter)
+    end
+
+    while state[:driver_running] && state[:pending].empty? &&
+          state[:waiters].length == 1 && state[:waiters].key?(easy.object_id)
+      scheduler_yield
+    end
+
+    true
+  ensure
+    state[:waiters].delete(easy.object_id) if defined?(state) && state[:waiters]
+    if defined?(previous_complete)
+      if previous_complete
+        easy.on_complete(&previous_complete)
+      else
+        easy.on_complete
+      end
+    end
+  end
+
+  def self.scheduler_state
+    Thread.current.thread_variable_get(:curb_scheduler_state) || begin
+      state = {
+        multi: Curl::Multi.new,
+        pending: [],
+        driver_running: false,
+        error: nil,
+        waiters: {},
+      }
+      Thread.current.thread_variable_set(:curb_scheduler_state, state)
+      state
+    end
+  end
+
+  def self.ensure_scheduler_driver(state)
+    return if state[:driver_running]
+
+    state[:driver_running] = true
+    state[:error] = nil
+
+    runner = proc do
+      begin
+        # Give sibling fibers a chance to enqueue work so the shared multi can
+        # batch scheduler-driven Easy#perform calls together.
+        pending_count = -1
+        until pending_count == state[:pending].size
+          pending_count = state[:pending].size
+          scheduler_yield
+        end
+
+        loop do
+          drain_scheduler_pending(state)
+          break if state[:multi].idle?
+
+          begin
+            state[:multi].perform do
+              drain_scheduler_pending(state)
+              release_scheduler_waiters(state)
+              scheduler_yield
+            end
+          ensure
+            # Release any siblings that completed just before a deferred
+            # callback exception is re-raised.
+            release_scheduler_waiters(state)
+          end
+        end
+      rescue => e
+        release_scheduler_waiters(state)
+        release_scheduler_error(state, e)
+      ensure
+        state[:driver_running] = false
+        ensure_scheduler_driver(state) if state[:error].nil? && !state[:pending].empty?
+      end
+    end
+
+    if Fiber.respond_to?(:schedule)
+      Fiber.schedule(&runner)
+    else
+      Fiber.new(blocking: false, &runner).resume
+    end
+  end
+
+  def self.drain_scheduler_pending(state)
+    pending = state[:pending]
+    until pending.empty?
+      easy = pending.first
+
+      break if state[:multi].instance_variable_defined?(:@__curb_deferred_exception)
+
+      state[:multi].add(easy)
+      break unless state[:multi].requests.key?(easy.object_id)
+
+      pending.shift
+    end
+  end
 
   def self.http(verb, url, post_body=nil, put_data=nil, &block)
     if Thread.current[:curb_curl_yielding]

data/tests/helper.rb

@@ -4,6 +4,8 @@ $CURB_TESTING = true
 require 'uri'
 require 'stringio'
 require 'digest/md5'
+require 'rbconfig'
+require File.join(RbConfig::CONFIG['rubylibdir'], 'timeout')
 
 $TOPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..'))
 $EXTDIR = File.join($TOPDIR, 'ext')
@@ -57,6 +59,7 @@ $TEST_URL = "file://#{'/' if RUBY_DESCRIPTION =~ /mswin|msys|mingw|cygwin|bccwin
 
 require 'thread'
 require 'webrick'
+require 'socket'
 
 # set this to true to avoid testing with multiple threads
 # or to test with multiple threads set it to false
@@ -64,21 +67,39 @@ require 'webrick'
 # on the presence of multiple threads
 TEST_SINGLE_THREADED=false
 
-# keep webrick quiet
-::WEBrick::HTTPServer.send(:remove_method,:access_log) if ::WEBrick::HTTPServer.instance_methods.include?(:access_log)
-::WEBrick::BasicLog.send(:remove_method,:log) if ::WEBrick::BasicLog.instance_methods.include?(:log)
+WEBRICK_TEST_LOG = WEBrick::Log.new(File.open(File::NULL, 'w'), WEBrick::BasicLog::ERROR)
+
+module CurbTestResourceCleanup
+  def teardown
+    super
+  ensure
+    begin
+      if Curl::Easy.respond_to?(:flush_deferred_multi_closes)
+        Curl::Easy.flush_deferred_multi_closes(all_threads: true)
+      end
+    rescue StandardError
+      nil
+    end
+
+    begin
+      ObjectSpace.each_object(Curl::Multi) do |multi|
+        begin
+          next if multi.instance_variable_defined?(:@deferred_close) && multi.instance_variable_get(:@deferred_close)
+          multi.instance_variable_set(:@requests, {})
+          multi._close
+        rescue StandardError
+          nil
+        end
+      end
+    rescue StandardError
+      nil
+    end
 
-::WEBrick::HTTPServer.class_eval do
-  def access_log(config, req, res)
-    # nop
-  end
-end
-::WEBrick::BasicLog.class_eval do
-  def log(level, data)
-    # nop
   end
 end
 
+Test::Unit::TestCase.prepend(CurbTestResourceCleanup)
+
 #
 # Simple test server to record number of times a request is sent/recieved of a specific
 # request type, e.g. GET,POST,PUT,DELETE
@@ -187,7 +208,7 @@ end
 module BugTestServerSetupTeardown
   def setup
     @port ||= 9992
-    @server = WEBrick::HTTPServer.new( :Port => @port )
+    @server = WEBrick::HTTPServer.new(:Port => @port, :Logger => WEBRICK_TEST_LOG, :AccessLog => [])
     @server.mount_proc("/test") do|req,res|
       if @response_proc
         @response_proc.call(res)
@@ -211,14 +232,170 @@ module BugTestServerSetupTeardown
 end
 
 module TestServerMethods
+  def server_responding?(port)
+    socket = TCPSocket.new('127.0.0.1', port)
+    socket.close
+    true
+  rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, IOError
+    false
+  end
+
+  def server_startup_timeout
+    ENV['RUBY_MEMCHECK_RUNNING'] ? 30 : 5
+  end
+
+  def wait_for_server_ready(port, thread: nil)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + server_startup_timeout
+
+    loop do
+      if thread && !thread.alive?
+        return false
+      end
+
+      return true if server_responding?(port)
+
+      raise "Failed to startup test server on port #{port}" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+      sleep 0.01
+    end
+  end
+
+  def server_shutdown_timeout
+    [server_startup_timeout, 0.25].max
+  end
+
+  def wait_for_server_stopped(port, thread: nil)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + server_shutdown_timeout
+
+    loop do
+      return true unless server_responding?(port)
+
+      if thread && !thread.alive?
+        return !server_responding?(port)
+      end
+
+      return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+      sleep 0.01
+    end
+  end
+
   def locked_file
     File.join(File.dirname(__FILE__),"server_lock-#{@__port}")
   end
 
+  def read_server_lock_pid
+    Integer(File.read(locked_file).strip, 10)
+  rescue Errno::ENOENT, ArgumentError, TypeError
+    nil
+  end
+
+  def write_server_lock(pid = Process.pid)
+    File.open(locked_file, 'w') { |f| f << "#{pid}\n" }
+  end
+
+  def process_alive?(pid)
+    return false unless pid && pid.positive?
+
+    Process.kill(0, pid)
+    true
+  rescue Errno::ESRCH
+    false
+  rescue Errno::EPERM
+    true
+  end
+
+  def server_lock_fresh?
+    (Time.now - File.mtime(locked_file)) < server_startup_timeout
+  rescue Errno::ENOENT
+    false
+  end
+
+  def stale_server_lock?(port)
+    return false unless File.exist?(locked_file)
+
+    pid = read_server_lock_pid
+    return false if pid && process_alive?(pid) && server_lock_fresh?
+    return false if server_responding?(port)
+
+    true
+  end
+
+  def clear_stale_server_lock(port)
+    return unless stale_server_lock?(port)
+
+    File.unlink(locked_file)
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def stop_test_server
+    server = instance_variable_defined?(:@server) ? @server : nil
+    pid = instance_variable_defined?(:@__pid) ? @__pid : nil
+    return unless server || pid
+
+    if TEST_SINGLE_THREADED
+      @__pid = nil
+
+      if pid
+        begin
+          Process.kill('INT', pid)
+        rescue Errno::ESRCH
+          nil
+        end
+
+        begin
+          Process.wait(pid)
+        rescue Errno::ECHILD, Errno::ESRCH
+          nil
+        end
+      end
+    else
+      thread = instance_variable_defined?(:@test_thread) ? @test_thread : nil
+      port = instance_variable_defined?(:@__port) ? @__port : nil
+      @server = nil
+      @test_thread = nil
+
+      server.shutdown if server
+      wait_for_server_stopped(port, thread: thread) if port
+      thread.join(server_shutdown_timeout) if thread
+
+      if thread&.alive?
+        thread.kill
+        thread.join(server_shutdown_timeout)
+        wait_for_server_stopped(port) if port
+      end
+    end
+
+    File.unlink(locked_file) if File.exist?(locked_file)
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def teardown
+    super
+  ensure
+    stop_test_server
+  end
+
   def server_setup(port=9129,servlet=TestServlet)
     @__port = port
-    if (@server ||= nil).nil? and !File.exist?(locked_file)
-      File.open(locked_file,'w') {|f| f << 'locked' }
+    @server = nil unless instance_variable_defined?(:@server)
+    @__pid = nil unless instance_variable_defined?(:@__pid)
+    @test_thread = nil unless instance_variable_defined?(:@test_thread)
+    clear_stale_server_lock(port)
+
+    if @server.nil? && File.exist?(locked_file)
+      begin
+        wait_for_server_ready(port)
+        return
+      rescue RuntimeError
+        clear_stale_server_lock(port)
+      end
+    end
+
+    if @server.nil? and !File.exist?(locked_file)
+      write_server_lock
       if TEST_SINGLE_THREADED
         rd, wr = IO.pipe
         @__pid = fork do
@@ -226,33 +403,50 @@ module TestServerMethods
           rd = nil
 
           # start up a webrick server for testing delete
-          server = WEBrick::HTTPServer.new :Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__))
+          server = WEBrick::HTTPServer.new(:Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__)), :Logger => WEBRICK_TEST_LOG, :AccessLog => [])
 
           server.mount(servlet.path, servlet)
           server.mount("/ext", WEBrick::HTTPServlet::FileHandler, File.join(File.dirname(__FILE__),'..','ext'))
 
           trap("INT") { server.shutdown }
           GC.start
-          wr.flush
-          wr.close
-          server.start
+          server_thread = Thread.new { server.start }
+
+          begin
+            if wait_for_server_ready(port, thread: server_thread)
+              wr.write('1')
+            else
+              wr.write('0')
+            end
+          rescue StandardError
+            wr.write('0')
+          ensure
+            wr.flush
+            wr.close
+          end
+
+          server_thread.join
         end
         wr.close
-        rd.read
+        ready = rd.read
         rd.close
+        if ready != '1'
+          STDERR.puts "Failed to startup test server!"
+          exit(1)
+        end
       else
         # start up a webrick server for testing delete
-        @server = WEBrick::HTTPServer.new :Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__))
+        server = WEBrick::HTTPServer.new(:Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__)), :Logger => WEBRICK_TEST_LOG, :AccessLog => [])
 
-        @server.mount(servlet.path, servlet)
-        @server.mount("/ext", WEBrick::HTTPServlet::FileHandler, File.join(File.dirname(__FILE__),'..','ext'))
-        queue = Queue.new # synchronize the thread startup to the main thread
+        server.mount(servlet.path, servlet)
+        server.mount("/ext", WEBrick::HTTPServlet::FileHandler, File.join(File.dirname(__FILE__),'..','ext'))
 
-        @test_thread = Thread.new { queue << 1; @server.start }
+        @server = server
+        # Keep a stable reference inside the thread so helper shutdown can clear
+        # @server without racing the server startup path.
+        @test_thread = Thread.new(server) { |srv| srv.start }
 
-        # wait for the queue
-        value = queue.pop
-        if !value
+        if !wait_for_server_ready(port, thread: @test_thread)
           STDERR.puts "Failed to startup test server!"
           exit(1)
         end
@@ -261,15 +455,7 @@ module TestServerMethods
 
       exit_code = lambda do
         begin
-          if File.exist?(locked_file)
-            File.unlink locked_file
-            if TEST_SINGLE_THREADED
-              Process.kill 'INT', @__pid
-            else
-              @server.shutdown unless @server.nil?
-            end
-          end
-          #@server.shutdown unless @server.nil?
+          stop_test_server
         rescue Object => e
           puts "Error #{__FILE__}:#{__LINE__}\n#{e.message}"
         end