curb 0.7.15 → 0.7.18

data/lib/curl/multi.rb

@@ -0,0 +1,244 @@
+module Curl
+
+  class Multi
+    class << self
+      # call-seq:
+      #   Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
+      #     easy
+      #   end
+      # 
+      # Blocking call to fetch multiple url's in parallel.
+      def get(urls, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls.each do|url|
+          url_confs << {:url => url, :method => :get}.merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
+      #                    { :follow_location => true, :multipart_form_post => true },
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def post(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :post).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
+      #                    {:url => 'url2', :put_data => IO.read('filepath')},
+      #                    {:url => 'url3', :put_data => "maybe another string or socket?"],
+      #                    {:follow_location => true},
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def put(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :put).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+
+      # call-seq:
+      #
+      # Curl::Multi.http( [
+      #   { :url => 'url1', :method => :post,
+      #     :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
+      #   { :url => 'url2', :method => :get,
+      #     :follow_location => true, :max_redirects => 3 },
+      #   { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
+      #   { :url => 'url4', :method => :head }
+      # ], {:pipeline => true})
+      #
+      # Blocking call to issue multiple HTTP requests with varying verb's.
+      #
+      # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
+      # multi_options: options for the multi handle 
+      # blk: a callback, that yeilds when a handle is completed
+      #
+      def http(urls_with_config, multi_options={}, &blk)
+        m = Curl::Multi.new
+
+        # maintain a sane number of easy handles
+        multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10
+
+        free_handles = [] # keep a list of free easy handles
+
+        # configure the multi handle
+        multi_options.each { |k,v| m.send("#{k}=", v) }
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
+
+        add_free_handle = proc do|conf, easy|
+          c       = conf.dup # avoid being destructive to input
+          url     = c.delete(:url)
+          method  = c.delete(:method)
+          headers = c.delete(:headers)
+
+          easy    = Curl::Easy.new if easy.nil?
+
+          easy.url = url
+
+          # assign callbacks
+          callbacks.each do |cb|
+            cbproc = c.delete(cb)
+            easy.send(cb,&cbproc) if cbproc
+          end
+
+          case method
+          when :post
+            fields = c.delete(:post_fields)
+            # set the post post using the url fields
+            easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
+          when :put
+            easy.put_data = c.delete(:put_data)
+          when :head
+            easy.head = true
+          when :delete
+            easy.delete = true
+          when :get
+          else
+            # XXX: nil is treated like a GET
+          end
+
+          # headers is a special key
+          headers.each {|k,v| easy.headers[k] = v } if headers
+ 
+          #
+          # use the remaining options as specific configuration to the easy handle
+          # bad options should raise an undefined method error
+          #
+          c.each { |k,v| easy.send("#{k}=",v) }
+
+          easy.on_complete {|curl,code|
+            free_handles << curl
+            blk.call(curl,code,method) if blk
+          }
+          m.add(easy)
+        end
+
+        max_connects.times do
+          conf = urls_with_config.pop
+          add_free_handle.call conf, nil
+          break if urls_with_config.empty?
+        end
+
+        consume_free_handles = proc do
+          # as we idle consume free handles
+          if urls_with_config.size > 0 && free_handles.size > 0
+            easy = free_handles.pop
+            conf = urls_with_config.pop
+            add_free_handle.call conf, easy
+          end
+        end
+
+        until urls_with_config.empty?
+          m.perform do
+            consume_free_handles.call
+          end
+          consume_free_handles.call
+        end
+        free_handles = nil 
+      end
+
+      # call-seq:
+      #
+      # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
+      #
+      # will create 2 new files file1.txt and file2.txt
+      # 
+      # 2 files will be opened, and remain open until the call completes
+      #
+      # when using the :post or :put method, urls should be a hash, including the individual post fields per post
+      #
+      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+        errors = []
+        procs = []
+        files = []
+        urls_with_config = []
+        url_to_download_paths = {}
+
+        urls.each_with_index do|urlcfg,i|
+          if urlcfg.is_a?(Hash)
+            url = url[:url]
+          else
+            url = urlcfg
+          end
+
+          if download_paths and download_paths[i]
+            download_path = download_paths[i]
+          else
+            download_path = File.basename(url)
+          end
+
+          file = lambda do|dp|
+            file = File.open(dp,"wb")
+            procs << (lambda {|data| file.write data; data.size })
+            files << file
+            file
+          end.call(download_path)
+
+          if urlcfg.is_a?(Hash)
+            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+          else
+            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+          end
+          url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
+        end
+
+        if blk
+          # when injecting the block, ensure file is closed before yielding
+          Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
+            info = url_to_download_paths[c.url]
+            begin
+              file = info[:file]
+              files.reject!{|f| f == file }
+              file.close
+            rescue => e
+              errors << e
+            end
+            blk.call(c,info[:path])
+          end
+        else
+          Curl::Multi.http(urls_with_config, multi_options)
+        end
+
+      ensure
+        files.each {|f|
+          begin
+            f.close
+          rescue => e
+            errors << e
+          end
+        }
+        raise errors unless errors.empty?
+      end
+
+    end
+  end
+end

data/tests/bug_crash_on_progress.rb

@@ -0,0 +1,33 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'webrick'
+class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
+class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
+
+class BugCrashOnDebug < Test::Unit::TestCase
+  
+  def test_on_debug
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+
+    c = Curl::Easy.new('http://localhost:9999/test')
+    c.on_progress do|x|
+      raise "error"
+    end
+    c.perform
+
+    assert false, "should not reach this point"
+
+  rescue => e
+    assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s
+    c.close
+  ensure
+    server.shutdown
+  end
+end

data/tests/tc_curl_download.rb

@@ -8,7 +8,7 @@ class TestCurbCurlDownload < Test::Unit::TestCase
   end
   
   def test_download_url_to_file_via_string
-    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+    dl_url = "http://localhost:9129/ext/curb_easy.c"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
 
     curb = Curl::Easy.download(dl_url, dl_path)
@@ -19,7 +19,7 @@ class TestCurbCurlDownload < Test::Unit::TestCase
   end
 
   def test_download_url_to_file_via_file_io
-    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+    dl_url = "http://localhost:9129/ext/curb_easy.c"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
     io = File.open(dl_path, 'wb')
 
@@ -32,7 +32,7 @@ class TestCurbCurlDownload < Test::Unit::TestCase
   end
 
   def test_download_url_to_file_via_io
-    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+    dl_url = "http://localhost:9129/ext/curb_easy.c"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
     reader, writer = IO.pipe
 
@@ -62,7 +62,7 @@ class TestCurbCurlDownload < Test::Unit::TestCase
   end
 
   def test_download_bad_url_gives_404
-    dl_url = "http://127.0.0.1:9129/this_file_does_not_exist.html"
+    dl_url = "http://localhost:9129/this_file_does_not_exist.html"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
 
     curb = Curl::Easy.download(dl_url, dl_path)

data/tests/tc_curl_easy.rb

@@ -461,7 +461,8 @@ class TestCurbCurlEasy < Test::Unit::TestCase
   def test_ssl_verify_host
     c = Curl::Easy.new    
     assert c.ssl_verify_host?
-    assert !c.ssl_verify_host = false
+    c.ssl_verify_host = 0
+    c.ssl_verify_host = false
     assert !c.ssl_verify_host?
   end
   
@@ -507,6 +508,17 @@ class TestCurbCurlEasy < Test::Unit::TestCase
     assert c.ignore_content_length?
   end
 
+  def test_resolve_mode
+    c = Curl::Easy.new
+    assert_equal :auto, c.resolve_mode
+    c.resolve_mode = :ipv4
+    assert_equal :ipv4, c.resolve_mode 
+    c.resolve_mode = :ipv6
+    assert_equal :ipv6, c.resolve_mode 
+
+    assert_raises(ArgumentError) { c.resolve_mode = :bad }
+  end
+
   def test_enable_cookies
     c = Curl::Easy.new
     assert !c.enable_cookies?
@@ -572,13 +584,20 @@ class TestCurbCurlEasy < Test::Unit::TestCase
   def test_post_remote_is_easy_handle
     # see: http://pastie.org/560852 and
     # http://groups.google.com/group/curb---ruby-libcurl-bindings/browse_thread/thread/216bb2d9b037f347?hl=en
-    [:post, :get,:head,:delete].each do |method|
-      count = 0
-      curl = Curl::Easy.send("http_#{method}", TestServlet.url) do|c|
-        count += 1
-        assert_equal Curl::Easy, c.class
+    [:post, :get, :head, :delete].each do |method|
+      retries = 0
+      begin
+        count = 0
+        curl = Curl::Easy.send("http_#{method}", TestServlet.url) do|c|
+          count += 1
+          assert_equal Curl::Easy, c.class
+        end
+        assert_equal 1, count, "For request method: #{method.to_s.upcase}"
+      rescue Curl::Err::HostResolutionError => e # travis-ci.org fails to resolve... try again?
+        retries+=1
+        retry if retries < 3
+        raise e
       end
-      assert_equal 1, count, "For request method: #{method.to_s.upcase}"
     end
   end
   
@@ -664,6 +683,17 @@ class TestCurbCurlEasy < Test::Unit::TestCase
     assert_match /message$/, curl.body_str
   end
 
+  # https://github.com/taf2/curb/issues/101
+  def test_put_data_null_bytes
+    curl = Curl::Easy.new(TestServlet.url)
+    curl.put_data = "a\0b"
+ 
+    curl.perform
+    
+    assert_match /^PUT/, curl.body_str
+    assert_match "a\0b", curl.body_str
+  end
+
   def test_put_nil_data_no_crash
     curl = Curl::Easy.new(TestServlet.url)
     curl.put_data = nil
@@ -700,7 +730,9 @@ class TestCurbCurlEasy < Test::Unit::TestCase
 
   def test_cert_with_password
     curl = Curl::Easy.new(TestServlet.url)
-    curl.cert= File.join(File.dirname(__FILE__),"cert.pem:password")
+    path = File.join(File.dirname(__FILE__),"cert.pem")
+    curl.certpassword = 'password'
+    curl.cert = path
     assert_match /cert.pem$/,curl.cert
   end
 
@@ -888,6 +920,40 @@ class TestCurbCurlEasy < Test::Unit::TestCase
 
   end
 
+  def test_get_set_multi_on_easy
+    easy = Curl::Easy.new
+    assert_nil easy.multi
+    multi = Curl::Multi.new
+    easy.multi = multi
+    assert_not_nil easy.multi
+    assert_equal multi, easy.multi
+  end
+
+  def test_raise_on_progress
+    c = Curl::Easy.new($TEST_URL)
+    c.on_progress {|w,x,y,z| raise "error" }
+    c.perform
+  rescue => e
+    assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s
+    c.close
+  end
+
+  def test_raise_on_success
+    c = Curl::Easy.new($TEST_URL)
+    c.on_success {|x| raise "error" }
+    c.perform
+  rescue => e
+    assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s
+    c.close
+  end
+
+  def test_raise_on_debug
+    c = Curl::Easy.new($TEST_URL)
+    c.on_debug { raise "error" }
+    c.perform
+    assert true, "raise in on debug has no effect"
+  end
+
   include TestServerMethods 
 
   def setup

data/tests/tc_curl_easy_setopt.rb

@@ -0,0 +1,31 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class TestCurbCurlEasySetOpt < Test::Unit::TestCase
+  def setup
+    @easy = Curl::Easy.new
+  end
+
+  def test_opt_verbose
+    @easy.set :verbose, true
+    assert @easy.verbose?
+  end
+
+  def test_opt_header
+    @easy.set :header, true
+  end
+
+  def test_opt_noprogress
+    @easy.set :noprogress, true
+  end
+
+  def test_opt_nosignal
+    @easy.set :nosignal, true
+  end
+
+  def test_opt_url
+    url = "http://google.com/"
+    @easy.set :url, url
+    assert_equal url, @easy.url
+  end
+
+end

metadata

@@ -1,41 +1,36 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: curb
-version: !ruby/object:Gem::Version 
-  hash: 29
+version: !ruby/object:Gem::Version
+  version: 0.7.18
   prerelease: 
-  segments: 
-  - 0
-  - 7
-  - 15
-  version: 0.7.15
 platform: ruby
-authors: 
+authors:
 - Ross Bamford
 - Todd A. Fisher
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-03-20 00:00:00 -04:00
-default_executable: 
+date: 2012-01-16 00:00:00.000000000 Z
 dependencies: []
-
-description: Curb (probably CUrl-RuBy or something) provides Ruby-language bindings for the libcurl(3), a fully-featured client-side URL transfer library. cURL and libcurl live at http://curl.haxx.se/
+description: Curb (probably CUrl-RuBy or something) provides Ruby-language bindings
+  for the libcurl(3), a fully-featured client-side URL transfer library. cURL and
+  libcurl live at http://curl.haxx.se/
 email: todd.fisher@gmail.com
 executables: []
-
-extensions: 
+extensions:
 - ext/extconf.rb
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE
 - README
-files: 
+files:
 - LICENSE
 - README
 - Rakefile
 - doc.rb
 - ext/extconf.rb
 - lib/curb.rb
+- lib/curl/easy.rb
+- lib/curl/multi.rb
 - lib/curl.rb
 - ext/curb.c
 - ext/curb_easy.c
@@ -51,6 +46,7 @@ files:
 - ext/curb_postfield.h
 - ext/curb_upload.h
 - tests/alltests.rb
+- tests/bug_crash_on_progress.rb
 - tests/bug_curb_easy_blocks_ruby_threads.rb
 - tests/bug_curb_easy_post_with_string_no_content_length_header.rb
 - tests/bug_instance_post_differs_from_class_post.rb
@@ -64,49 +60,42 @@ files:
 - tests/require_last_or_segfault_script.rb
 - tests/tc_curl_download.rb
 - tests/tc_curl_easy.rb
+- tests/tc_curl_easy_setopt.rb
 - tests/tc_curl_multi.rb
 - tests/tc_curl_postfield.rb
 - tests/timeout.rb
 - tests/timeout_server.rb
 - tests/unittests.rb
-has_rdoc: true
 homepage: http://curb.rubyforge.org/
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --main
 - README
-require_paths: 
+require_paths:
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: curb
-rubygems_version: 1.4.1
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Ruby libcurl bindings
-test_files: 
+test_files:
 - tests/alltests.rb
+- tests/bug_crash_on_progress.rb
 - tests/bug_curb_easy_blocks_ruby_threads.rb
 - tests/bug_curb_easy_post_with_string_no_content_length_header.rb
 - tests/bug_instance_post_differs_from_class_post.rb
@@ -120,6 +109,7 @@ test_files:
 - tests/require_last_or_segfault_script.rb
 - tests/tc_curl_download.rb
 - tests/tc_curl_easy.rb
+- tests/tc_curl_easy_setopt.rb
 - tests/tc_curl_multi.rb
 - tests/tc_curl_postfield.rb
 - tests/timeout.rb