curb 0.8.4 → 1.3.5

data/lib/curl/multi.rb

@@ -1,12 +1,15 @@
+# frozen_string_literal: true
 module Curl
-
   class Multi
+    class DownloadError < RuntimeError
+      attr_accessor :errors
+    end
     class << self
       # call-seq:
-      #   Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
+      #   Curl::Multi.get(['url1','url2','url3','url4','url5'], :follow_location => true) do|easy|
       #     easy
       #   end
-      # 
+      #
       # Blocking call to fetch multiple url's in parallel.
       def get(urls, easy_options={}, multi_options={}, &blk)
         url_confs = []
@@ -22,13 +25,13 @@ module Curl
       #                     {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
       #                     {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
       #                    { :follow_location => true, :multipart_form_post => true },
-      #                    {:pipeline => true }) do|easy|
+      #                    {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
       #     easy_handle_on_request_complete
       #   end
-      # 
+      #
       # Blocking call to POST multiple form's in parallel.
-      # 
-      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      #
+      # urls_with_config: is a hash of url's pointing to the postfields to send
       # easy_options: are a set of common options to set on all easy handles
       # multi_options: options to set on the Curl::Multi handle
       #
@@ -46,13 +49,13 @@ module Curl
       #                    {:url => 'url2', :put_data => IO.read('filepath')},
       #                    {:url => 'url3', :put_data => "maybe another string or socket?"],
       #                    {:follow_location => true},
-      #                    {:pipeline => true }) do|easy|
+      #                    {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
       #     easy_handle_on_request_complete
       #   end
-      # 
+      #
       # Blocking call to POST multiple form's in parallel.
-      # 
-      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      #
+      # urls_with_config: is a hash of url's pointing to the postfields to send
       # easy_options: are a set of common options to set on all easy handles
       # multi_options: options to set on the Curl::Multi handle
       #
@@ -74,12 +77,12 @@ module Curl
       #     :follow_location => true, :max_redirects => 3 },
       #   { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
       #   { :url => 'url4', :method => :head }
-      # ], {:pipeline => true})
+      # ], {:pipeline => Curl::CURLPIPE_HTTP1})
       #
       # Blocking call to issue multiple HTTP requests with varying verb's.
       #
       # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
-      # multi_options: options for the multi handle 
+      # multi_options: options for the multi handle
       # blk: a callback, that yeilds when a handle is completed
       #
       def http(urls_with_config, multi_options={}, &blk)
@@ -92,7 +95,7 @@ module Curl
 
         # configure the multi handle
         multi_options.each { |k,v| m.send("#{k}=", v) }
-        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_redirect,:on_missing,:on_body,:on_header]
 
         add_free_handle = proc do|conf, easy|
           c       = conf.dup # avoid being destructive to input
@@ -128,23 +131,23 @@ module Curl
 
           # headers is a special key
           headers.each {|k,v| easy.headers[k] = v } if headers
- 
+
           #
           # use the remaining options as specific configuration to the easy handle
           # bad options should raise an undefined method error
           #
           c.each { |k,v| easy.send("#{k}=",v) }
 
-          easy.on_complete {|curl,code|
+          easy.on_complete {|curl|
             free_handles << curl
-            blk.call(curl,code,method) if blk
+            blk.call(curl,curl.response_code,method) if blk
           }
           m.add(easy)
         end
 
         max_connects.times do
           conf = urls_with_config.pop
-          add_free_handle.call conf, nil
+          add_free_handle.call(conf, nil) if conf
           break if urls_with_config.empty?
         end
 
@@ -153,21 +156,26 @@ module Curl
           if urls_with_config.size > 0 && free_handles.size > 0
             easy = free_handles.pop
             conf = urls_with_config.pop
-            add_free_handle.call conf, easy
+            add_free_handle.call(conf, easy) if conf
           end
         end
 
-        if urls_with_config.empty?
-          m.perform
-        else
-          until urls_with_config.empty?
-            m.perform do
+        begin
+          if urls_with_config.empty?
+            m.perform
+          else
+            until urls_with_config.empty?
+              m.perform do
+                consume_free_handles.call
+              end
               consume_free_handles.call
             end
-            consume_free_handles.call
+            free_handles = nil
           end
-          free_handles = nil
+        ensure
+          m.close
         end
+
       end
 
       # call-seq:
@@ -175,7 +183,7 @@ module Curl
       # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
       #
       # will create 2 new files file1.txt and file2.txt
-      # 
+      #
       # 2 files will be opened, and remain open until the call completes
       #
       # when using the :post or :put method, urls should be a hash, including the individual post fields per post
@@ -240,9 +248,107 @@ module Curl
             errors << e
           end
         }
-        raise errors unless errors.empty?
+        if errors.any?
+          de = Curl::Multi::DownloadError.new
+          de.errors = errors
+          raise de
+        end
+      end
+    end
+
+    def cancel!
+      requests.each do |_,easy|
+        remove(easy)
+      end
+    end
+
+    def idle?
+      requests.empty?
+    end
+
+    def requests
+      @requests ||= {}
+    end
+
+    def __idle_easy_references
+      @__curb_idle_easy_references ||= ObjectSpace::WeakMap.new
+    end
+    
+    def __register_idle_easy_reference(easy)
+      __idle_easy_references[easy] = true
+      self
+    end
+    
+    def __unregister_idle_easy_reference(easy)
+      return self unless instance_variable_defined?(:@__curb_idle_easy_references)
+
+      if @__curb_idle_easy_references.respond_to?(:delete)
+        @__curb_idle_easy_references.delete(easy)
+      else
+        retained_references = ObjectSpace::WeakMap.new
+        @__curb_idle_easy_references.each_key do |tracked_easy|
+          next if tracked_easy.equal?(easy)
+
+          retained_references[tracked_easy] = true
+        end
+        @__curb_idle_easy_references = retained_references
       end
 
+      self
+    end
+    
+    def __clear_idle_easy_references
+      return unless instance_variable_defined?(:@__curb_idle_easy_references)
+    
+      @__curb_idle_easy_references.keys.each do |easy|
+        easy.multi = nil if easy.multi.equal?(self)
+      end
+      @__curb_idle_easy_references = ObjectSpace::WeakMap.new
+    end
+
+    private :__idle_easy_references, :__register_idle_easy_reference,
+            :__unregister_idle_easy_reference, :__clear_idle_easy_references
+
+    def add(easy)
+      return self if requests[easy.object_id]
+      # Once a deferred callback exception is pending, Multi#perform is
+      # draining existing transfers only and must not start replacement work.
+      return self if instance_variable_defined?(:@__curb_deferred_exception)
+      _add(easy)
+      __unregister_idle_easy_reference(easy)
+      requests[easy.object_id] = easy
+      self
     end
+
+    def remove(easy)
+      return self if !requests[easy.object_id]
+      requests.delete(easy.object_id)
+      _remove(easy)
+      self
+    end
+
+    def close
+      __close(true)
+    end
+
+    def _autoclose
+      __close(false)
+    end
+
+    private :_autoclose
+
+    private
+
+    def __close(permanent)
+      requests.values.each {|easy|
+        _remove(easy)
+      }
+      __clear_idle_easy_references if permanent
+      @requests = {}
+      _close
+      _mark_closed if permanent
+      self
+    end
+
   end
 end

data/lib/curl.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'curb_core'
 require 'curl/easy'
 require 'curl/multi'
@@ -5,13 +6,215 @@ require 'uri'
 
 # expose shortcut methods
 module Curl
-  
+  def self.scheduler_active?
+    Fiber.respond_to?(:scheduler) && !Fiber.scheduler.nil?
+  end
+
+  def self.deferred_exception_source_id(state)
+    return unless state[:multi].instance_variable_defined?(:@__curb_deferred_exception_source_id)
+
+    state[:multi].instance_variable_get(:@__curb_deferred_exception_source_id)
+  end
+
+  def self.scheduler_waiter_blocking_supported?
+    scheduler = Fiber.scheduler
+    scheduler && scheduler.respond_to?(:block) && scheduler.respond_to?(:unblock)
+  end
+
+  def self.wake_scheduler_waiter(waiter)
+    fiber = waiter[:fiber]
+    scheduler = waiter[:scheduler]
+    return unless fiber&.alive? && scheduler&.respond_to?(:unblock)
+
+    scheduler.unblock(waiter, fiber)
+  end
+
+  def self.complete_scheduler_waiter(waiter)
+    return if waiter[:done]
+
+    waiter[:done] = true
+    wake_scheduler_waiter(waiter)
+  end
+
+  def self.fail_scheduler_waiter(waiter, error)
+    return if waiter[:error]
+
+    waiter[:error] = error
+    wake_scheduler_waiter(waiter)
+  end
+
+  def self.release_scheduler_error(state, error)
+    source_waiter = state[:waiters][deferred_exception_source_id(state)]
+
+    if source_waiter
+      fail_scheduler_waiter(source_waiter, error)
+    else
+      state[:error] = error
+      state[:waiters].each_value { |waiter| wake_scheduler_waiter(waiter) }
+    end
+  end
+
+  def self.block_scheduler_waiter(waiter)
+    unless scheduler_waiter_blocking_supported?
+      sleep 0
+      return
+    end
+
+    waiter[:fiber] = Fiber.current
+    waiter[:scheduler] ||= Fiber.scheduler
+    return if waiter[:done] || waiter[:error]
+
+    waiter[:scheduler].block(waiter, nil)
+  ensure
+    waiter[:fiber] = nil if waiter[:fiber].equal?(Fiber.current)
+  end
+
+  def self.scheduler_yield
+    scheduler = Fiber.scheduler
+
+    if scheduler&.respond_to?(:kernel_sleep)
+      scheduler.kernel_sleep(0)
+    else
+      sleep 0
+    end
+  end
+
+  def self.release_scheduler_waiters(state)
+    source_id = deferred_exception_source_id(state)
+
+    state[:waiters].each do |easy_id, waiter|
+      next if source_id == easy_id
+
+      complete_scheduler_waiter(waiter) if waiter[:completed]
+    end
+  end
+
+  def self.perform_with_scheduler(easy)
+    state = scheduler_state
+    waiter = {completed: false, done: false, error: nil, fiber: nil, scheduler: Fiber.scheduler}
+    state[:waiters][easy.object_id] = waiter
+    previous_complete = easy.on_complete do |completed_easy|
+      previous_complete.call(completed_easy) if previous_complete
+      waiter[:completed] = true
+    end
+
+    state[:pending] << easy
+    ensure_scheduler_driver(state)
+
+    until waiter[:done]
+      raise waiter[:error] if waiter[:error]
+      raise state[:error] if state[:error]
+      block_scheduler_waiter(waiter)
+    end
+
+    while state[:driver_running] && state[:pending].empty? &&
+          state[:waiters].length == 1 && state[:waiters].key?(easy.object_id)
+      scheduler_yield
+    end
+
+    true
+  ensure
+    state[:waiters].delete(easy.object_id) if defined?(state) && state[:waiters]
+    if defined?(previous_complete)
+      if previous_complete
+        easy.on_complete(&previous_complete)
+      else
+        easy.on_complete
+      end
+    end
+  end
+
+  def self.scheduler_state
+    Thread.current.thread_variable_get(:curb_scheduler_state) || begin
+      state = {
+        multi: Curl::Multi.new,
+        pending: [],
+        driver_running: false,
+        error: nil,
+        waiters: {},
+      }
+      Thread.current.thread_variable_set(:curb_scheduler_state, state)
+      state
+    end
+  end
+
+  def self.ensure_scheduler_driver(state)
+    return if state[:driver_running]
+
+    state[:driver_running] = true
+    state[:error] = nil
+
+    runner = proc do
+      begin
+        # Give sibling fibers a chance to enqueue work so the shared multi can
+        # batch scheduler-driven Easy#perform calls together.
+        pending_count = -1
+        until pending_count == state[:pending].size
+          pending_count = state[:pending].size
+          scheduler_yield
+        end
+
+        loop do
+          drain_scheduler_pending(state)
+          break if state[:multi].idle?
+
+          begin
+            state[:multi].perform do
+              drain_scheduler_pending(state)
+              release_scheduler_waiters(state)
+              scheduler_yield
+            end
+          ensure
+            # Release any siblings that completed just before a deferred
+            # callback exception is re-raised.
+            release_scheduler_waiters(state)
+          end
+        end
+      rescue => e
+        release_scheduler_waiters(state)
+        release_scheduler_error(state, e)
+      ensure
+        state[:driver_running] = false
+        ensure_scheduler_driver(state) if state[:error].nil? && !state[:pending].empty?
+      end
+    end
+
+    if Fiber.respond_to?(:schedule)
+      Fiber.schedule(&runner)
+    else
+      Fiber.new(blocking: false, &runner).resume
+    end
+  end
+
+  def self.drain_scheduler_pending(state)
+    pending = state[:pending]
+    until pending.empty?
+      easy = pending.first
+
+      break if state[:multi].instance_variable_defined?(:@__curb_deferred_exception)
+
+      state[:multi].add(easy)
+      break unless state[:multi].requests.key?(easy.object_id)
+
+      pending.shift
+    end
+  end
+
   def self.http(verb, url, post_body=nil, put_data=nil, &block)
-    handle = Thread.current[:curb_curl] ||= Curl::Easy.new
+    if Thread.current[:curb_curl_yielding]
+      handle = Curl::Easy.new # we can't reuse this
+    else
+      handle = Thread.current[:curb_curl] ||= Curl::Easy.new
+      handle.reset
+    end
     handle.url = url
     handle.post_body = post_body if post_body
     handle.put_data = put_data if put_data
-    yield handle if block_given?
+    if block_given?
+      Thread.current[:curb_curl_yielding] = true
+      yield handle
+      Thread.current[:curb_curl_yielding] = false
+    end
     handle.http(verb)
     handle
   end
@@ -45,18 +248,21 @@ module Curl
   end
 
   def self.urlalize(url, params={})
-    query_str = params.map {|k,v| "#{URI.escape(k.to_s)}=#{URI.escape(v.to_s)}" }.join('&')
-    if url.match(/\?/)
-      "#{url}&#{query_str}"
-    elsif query_str.size > 0
-      "#{url}?#{query_str}"
-    else
-      url
-    end
+    uri = URI(url)
+    # early return if we didn't specify any extra params
+    return uri.to_s if (params || {}).empty?
+
+    params_query = URI.encode_www_form(params || {})
+    uri.query = [uri.query.to_s, params_query].reject(&:empty?).join('&')
+    uri.to_s
   end
 
   def self.postalize(params={})
     params.respond_to?(:map) ? URI.encode_www_form(params) : (params.respond_to?(:to_s) ? params.to_s : params)
   end
 
+  def self.reset
+    Thread.current[:curb_curl] = Curl::Easy.new
+  end
+
 end

data/tests/bug_crash_on_debug.rb

@@ -1,37 +1,23 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
 
-require 'webrick'
-class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
-class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
-
-require 'curl'
-
 class BugCrashOnDebug < Test::Unit::TestCase
+  include BugTestServerSetupTeardown
 
   def test_on_debug
-    server = WEBrick::HTTPServer.new( :Port => 9999 )
-    server.mount_proc("/test") do|req,res|
-      res.body = "hi"
-      res['Content-Type'] = "text/html"
-    end
-    puts 'a'
-    thread = Thread.new(server) do|srv|
-      srv.start
-    end
-    puts 'b'
-    c = Curl::Easy.new('http://127.0.0.1:9999/test')
-    c.on_debug do|x|
-      puts x.inspect
-      raise "error" # this will get swallowed
+    c = Curl::Easy.new("http://127.0.0.1:#{@port}/test")
+    did_raise = false
+    did_call = false
+    begin
+      c.on_success do|x|
+        did_call = true
+        raise "error" # this will get swallowed
+      end
+      c.perform
+    rescue => e
+      did_raise = true
     end
-    c.perform
-    puts 'c'
-  ensure
-    puts 'd'
-    server.shutdown
-    puts 'e'
-    puts thread.exit
-    puts 'f'
+    assert did_raise
+    assert did_call
   end
 
 end

data/tests/bug_crash_on_progress.rb

@@ -1,33 +1,49 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
-require 'webrick'
-class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
-class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
 
 class BugCrashOnDebug < Test::Unit::TestCase
-  
-  def test_on_debug
-    server = WEBrick::HTTPServer.new( :Port => 9999 )
-    server.mount_proc("/test") do|req,res|
-      res.body = "hi"
-      res['Content-Type'] = "text/html"
-    end
+  include BugTestServerSetupTeardown
 
-    thread = Thread.new(server) do|srv|
-      srv.start
+  def test_on_progress_raise
+    c = Curl::Easy.new("http://127.0.0.1:#{@port}/test")
+    c.on_progress do|x|
+      raise "error"
     end
+    c.perform
 
-    c = Curl::Easy.new('http://127.0.0.1:9999/test')
+    assert false, "should not reach this point"
+
+  rescue => e
+    assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s
+    c.close
+  end
+
+  def test_on_progress_abort
+    # see: https://github.com/taf2/curb/issues/192,
+    # to pass:
+    # 
+    # c = Curl::Easy.new('http://127.0.0.1:9999/test')
+    # c.on_progress do|x|
+    #   puts "we're in the progress callback"
+    #   false
+    # end
+    # c.perform
+    # 
+    #  notice no return keyword
+    #
+    c = Curl::Easy.new("http://127.0.0.1:#{@port}/test")
+    did_progress = false
     c.on_progress do|x|
-      raise "error"
+      did_progress = true
+      return false
     end
     c.perform
+    assert did_progress
 
     assert false, "should not reach this point"
 
   rescue => e
     assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s
     c.close
-  ensure
-    server.shutdown
   end
+
 end

data/tests/bug_curb_easy_blocks_ruby_threads.rb

@@ -1,27 +1,25 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
-require 'webrick'
-class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
-class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
 
 class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase
-  def test_bug
-    server = WEBrick::HTTPServer.new( :Port => 9999 )
-    server.mount_proc("/test") do|req,res|
+  include BugTestServerSetupTeardown
+
+  def setup
+    @port = unused_local_port
+    @response_proc = lambda do|res|
       sleep 0.5
       res.body = "hi"
       res['Content-Type'] = "text/html"
     end
+    super
+  end
 
-    thread = Thread.new(server) do|srv|
-      srv.start
-    end
-
+  def test_bug
     threads = []
     timer = Time.now
 
     5.times do |i|
       t = Thread.new do
-        c = Curl::Easy.perform('http://127.0.0.1:9999/test')
+        c = Curl::Easy.perform("http://127.0.0.1:#{@port}/test")
         c.header_str
       end
       threads << t
@@ -37,7 +35,7 @@ class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase
     timer = Time.now
     single_responses = []
     5.times do |i|
-      c = Curl::Easy.perform('http://127.0.0.1:9999/test')
+      c = Curl::Easy.perform("http://127.0.0.1:#{@port}/test")
       single_responses << c.header_str
     end
     
@@ -45,8 +43,5 @@ class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase
     puts "requested in #{single_time}"
 
     assert single_time > multi_time
-
-    server.shutdown
-    thread.join
   end
 end