curb 0.1.4 → 0.7.15

data/lib/curb.rb

@@ -0,0 +1,308 @@
+require 'curb_core'
+
+module Curl
+
+  class Easy
+    class << self
+    
+      # call-seq:
+      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   
+      # Stream the specified url (via perform) and save the data directly to the
+      # supplied filename (defaults to the last component of the URL path, which will 
+      # usually be the filename most simple urls).
+      # 
+      # If a block is supplied, it will be passed the curl instance prior to the
+      # perform call.
+      # 
+      # *Note* that the semantics of the on_body handler are subtly changed when using
+      # download, to account for the automatic routing of data to the specified file: The 
+      # data string is passed to the handler *before* it is written
+      # to the file, allowing the handler to perform mutative operations where 
+      # necessary. As usual, the transfer will be aborted if the on_body handler
+      # returns a size that differs from the data chunk size - in this case, the 
+      # offending chunk will *not* be written to the file, the file will be closed,
+      # and a Curl::Err::AbortedByCallbackError will be raised.
+      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+        curl = Curl::Easy.new(url, &blk)
+        
+        output = if filename.is_a? IO
+          filename.binmode if filename.respond_to?(:binmode)
+          filename
+        else
+          File.open(filename, 'wb')
+        end
+        
+        begin
+          old_on_body = curl.on_body do |data|
+            result = old_on_body ?  old_on_body.call(data) : data.length
+            output << data if result == data.length
+            result
+          end
+          curl.perform
+        ensure
+          output.close rescue IOError
+        end
+        
+        return curl
+      end
+    end
+
+    # Allow the incoming cert string to be file:password
+    # but be careful to not use a colon from a windows file path
+    # as the split point. Mimic what curl's main does
+    alias_method :native_cert=, :cert=
+    def cert=(cert_file)
+      pos = cert_file.rindex(':')
+      if pos && pos > 1
+        self.native_cert= cert_file[0..pos-1]
+        self.certpassword= cert_file[pos+1..-1]
+      else
+        self.native_cert= cert_file
+      end
+      self.cert
+    end
+  end
+
+  class Multi
+    class << self
+      # call-seq:
+      #   Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
+      #     easy
+      #   end
+      # 
+      # Blocking call to fetch multiple url's in parallel.
+      def get(urls, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls.each do|url|
+          url_confs << {:url => url, :method => :get}.merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
+      #                    { :follow_location => true, :multipart_form_post => true },
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def post(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :post).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
+      #                    {:url => 'url2', :put_data => IO.read('filepath')},
+      #                    {:url => 'url3', :put_data => "maybe another string or socket?"],
+      #                    {:follow_location => true},
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def put(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :put).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+
+      # call-seq:
+      #
+      # Curl::Multi.http( [
+      #   { :url => 'url1', :method => :post,
+      #     :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
+      #   { :url => 'url2', :method => :get,
+      #     :follow_location => true, :max_redirects => 3 },
+      #   { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
+      #   { :url => 'url4', :method => :head }
+      # ], {:pipeline => true})
+      #
+      # Blocking call to issue multiple HTTP requests with varying verb's.
+      #
+      # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
+      # multi_options: options for the multi handle 
+      # blk: a callback, that yeilds when a handle is completed
+      #
+      def http(urls_with_config, multi_options={}, &blk)
+        m = Curl::Multi.new
+
+        # maintain a sane number of easy handles
+        multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10
+
+        free_handles = [] # keep a list of free easy handles
+
+        # configure the multi handle
+        multi_options.each { |k,v| m.send("#{k}=", v) }
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
+
+        add_free_handle = proc do|conf, easy|
+          c       = conf.dup # avoid being destructive to input
+          url     = c.delete(:url)
+          method  = c.delete(:method)
+          headers = c.delete(:headers)
+
+          easy    = Curl::Easy.new if easy.nil?
+
+          easy.url = url
+
+          # assign callbacks
+          callbacks.each do |cb|
+            cbproc = c.delete(cb)
+            easy.send(cb,&cbproc) if cbproc
+          end
+
+          case method
+          when :post
+            fields = c.delete(:post_fields)
+            # set the post post using the url fields
+            easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
+          when :put
+            easy.put_data = c.delete(:put_data)
+          when :head
+            easy.head = true
+          when :delete
+            easy.delete = true
+          when :get
+          else
+            # XXX: nil is treated like a GET
+          end
+
+          # headers is a special key
+          headers.each {|k,v| easy.headers[k] = v } if headers
+ 
+          #
+          # use the remaining options as specific configuration to the easy handle
+          # bad options should raise an undefined method error
+          #
+          c.each { |k,v| easy.send("#{k}=",v) }
+
+          easy.on_complete {|curl,code|
+            free_handles << curl
+            blk.call(curl,code,method) if blk
+          }
+          m.add(easy)
+        end
+
+        max_connects.times do
+          conf = urls_with_config.pop
+          add_free_handle.call conf, nil
+          break if urls_with_config.empty?
+        end
+
+        consume_free_handles = proc do
+          # as we idle consume free handles
+          if urls_with_config.size > 0 && free_handles.size > 0
+            easy = free_handles.pop
+            conf = urls_with_config.pop
+            add_free_handle.call conf, easy
+          end
+        end
+
+        until urls_with_config.empty?
+          m.perform do
+            consume_free_handles.call
+          end
+          consume_free_handles.call
+        end
+        free_handles = nil 
+      end
+
+      # call-seq:
+      #
+      # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
+      #
+      # will create 2 new files file1.txt and file2.txt
+      # 
+      # 2 files will be opened, and remain open until the call completes
+      #
+      # when using the :post or :put method, urls should be a hash, including the individual post fields per post
+      #
+      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+        errors = []
+        procs = []
+        files = []
+        urls_with_config = []
+        url_to_download_paths = {}
+
+        urls.each_with_index do|urlcfg,i|
+          if urlcfg.is_a?(Hash)
+            url = url[:url]
+          else
+            url = urlcfg
+          end
+
+          if download_paths and download_paths[i]
+            download_path = download_paths[i]
+          else
+            download_path = File.basename(url)
+          end
+
+          file = lambda do|dp|
+            file = File.open(dp,"wb")
+            procs << (lambda {|data| file.write data; data.size })
+            files << file
+            file
+          end.call(download_path)
+
+          if urlcfg.is_a?(Hash)
+            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+          else
+            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+          end
+          url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
+        end
+
+        if blk
+          # when injecting the block, ensure file is closed before yielding
+          Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
+            info = url_to_download_paths[c.url]
+            begin
+              file = info[:file]
+              files.reject!{|f| f == file }
+              file.close
+            rescue => e
+              errors << e
+            end
+            blk.call(c,info[:path])
+          end
+        else
+          Curl::Multi.http(urls_with_config, multi_options)
+        end
+
+      ensure
+        files.each {|f|
+          begin
+            f.close
+          rescue => e
+            errors << e
+          end
+        }
+        raise errors unless errors.empty?
+      end
+
+    end
+  end
+end

data/{ext → lib}/curl.rb

@@ -1,2 +1 @@
 require 'curb'
-

data/tests/bug_curb_easy_blocks_ruby_threads.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'webrick'
+class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
+class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
+
+class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase
+  def test_bug
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      sleep 0.5
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+
+    threads = []
+    timer = Time.now
+
+    5.times do |i|
+      t = Thread.new do
+        c = Curl::Easy.perform('http://localhost:9999/test')
+        c.header_str
+      end
+      threads << t
+    end
+
+    multi_responses = threads.collect do|t|
+      t.value
+    end
+
+    multi_time = (Time.now - timer)
+    puts "requested in #{multi_time}"
+
+    timer = Time.now
+    single_responses = []
+    5.times do |i|
+      c = Curl::Easy.perform('http://localhost:9999/test')
+      single_responses << c.header_str
+    end
+    
+    single_time = (Time.now - timer)
+    puts "requested in #{single_time}"
+
+    assert single_time > multi_time
+
+    server.shutdown
+    thread.join
+  end
+end

data/tests/bug_curb_easy_post_with_string_no_content_length_header.rb

@@ -0,0 +1,83 @@
+=begin
+From jwhitmire
+Todd, I'm trying to use curb to post data to a REST url. We're using it to post support questions from our iphone app directly to tender. The post looks good to me, but curl is not adding the content-length header so I get a 411 length required response from the server.
+
+Here's my post block, do you see anything obvious? Do I need to manually add the Content-Length header?
+
+c = Curl::Easy.http_post(url) do |curl|
+  curl.headers["User-Agent"] = "Curl/Ruby"
+  if user
+    curl.headers["X-Multipass"] = user.multipass
+  else
+    curl.headers["X-Tender-Auth"] = TOKEN
+  end
+  curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+  curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&')
+
+  curl.verbose = true
+  curl.follow_location = true
+  curl.enable_cookies = true
+end
+Any insight you care to share would be helpful. Thanks.
+=end
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'webrick'
+class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
+class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
+
+class BugCurbEasyPostWithStringNoContentLengthHeader < Test::Unit::TestCase
+  def test_bug_workaround
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      assert_equal '15', req['Content-Length']
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+    params = {:cat => "hat", :foo => "bar"}
+
+    post_body = params.map{|f,k| "#{Curl::Easy.new.escape(f)}=#{Curl::Easy.new.escape(k)}"}.join('&')
+    c = Curl::Easy.http_post("http://127.0.0.1:9999/test",post_body) do |curl|
+      curl.headers["User-Agent"] = "Curl/Ruby"
+      curl.headers["X-Tender-Auth"] = "A Token"
+      curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+      curl.follow_location = true
+      curl.enable_cookies = true
+    end
+
+    server.shutdown
+    thread.join
+  end
+  def test_bug
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      assert_equal '15', req['Content-Length']
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+    params = {:cat => "hat", :foo => "bar"}
+
+    c = Curl::Easy.http_post("http://127.0.0.1:9999/test") do |curl|
+      curl.headers["User-Agent"] = "Curl/Ruby"
+      curl.headers["X-Tender-Auth"] = "A Token"
+      curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+      curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&')
+
+      curl.follow_location = true
+      curl.enable_cookies = true
+    end
+
+    server.shutdown
+    thread.join
+  end
+end

data/tests/bug_instance_post_differs_from_class_post.rb

@@ -23,7 +23,7 @@
 #   instance httppost. This bug is intermittent, but results in an
 #   exception from the first post when it occurs.
 #   
-require File.join(File.dirname(__FILE__), 'helper')
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
 
 class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase
   def test_bug

data/tests/bug_multi_segfault.rb

@@ -0,0 +1,14 @@
+# From safis  http://github.com/taf2/curb/issues#issue/5
+# irb: require 'curb'
+# irb: multi = Curl::Multi.new
+# irb: exit
+# <main>:47140: [BUG] Bus Error
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext'))
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib'))
+require 'curb'
+
+class BugMultiSegfault < Test::Unit::TestCase
+  def test_bug
+    multi = Curl::Multi.new
+  end
+end

data/tests/bug_postfields_crash.rb

@@ -0,0 +1,26 @@
+# From GICodeWarrior:
+#
+# $ ruby crash_curb.rb 
+# crash_curb.rb:7: [BUG] Segmentation fault
+# ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# 
+# Aborted
+# crash_curb.rb:
+# #!/usr/bin/ruby
+# require 'rubygems'
+# require 'curb'
+# 
+# curl = Curl::Easy.new('http://example.com/')
+# curl.multipart_form_post = true
+# curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'})
+# Ubuntu 9.10
+# curb gem version 0.6.2.1
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class BugPostFieldsCrash < Test::Unit::TestCase
+  def test_crash
+    curl = Curl::Easy.new('http://example.com/')
+    curl.multipart_form_post = true
+    curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'})
+  end
+end

data/tests/bug_postfields_crash2.rb

@@ -0,0 +1,57 @@
+# Not sure if this is an IRB bug, but thought you guys should know.
+#
+# ** My Ruby version: ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# ** Version of Rubygems: 1.3.5
+# ** Version of the Curb gem: 0.6.6.0
+#
+#
+# Transcript of IRB session:
+# ------------------------------------------------------------------------------------------------------------------
+# irb(main):001:0>  a =  {
+# irb(main):002:1*     :type => :pie,
+# irb(main):003:1*     :series => {
+# irb(main):004:2*       :names => [:a,:b],
+# irb(main):005:2*       :values => [70,30],
+# irb(main):006:2*       :colors => [:red,:green]
+# irb(main):007:2>     },
+# irb(main):008:1*     :output_format => :png
+# irb(main):009:1>   }
+# => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}}
+# irb(main):010:0> post = []
+# => []
+# irb(main):011:0> require 'rubygems'
+# => true
+# irb(main):012:0> require 'curb'
+# => true
+# irb(main):013:0> include Curl
+# => Object
+# irb(main):014:0> a.each_pair do |k,v|
+# irb(main):015:1* post << PostField.content(k,v)
+# irb(main):016:1> end
+# => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}}
+# irb(main):017:0> post
+# /usr/lib/ruby/1.8/irb.rb:302: [BUG] Segmentation fault
+# ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+#
+# Aborted
+# ------------------------------------------------------------------------------------------------------------------
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class BugPostFieldsCrash2 < Test::Unit::TestCase
+  def test_crash
+    a =  {
+       :type => :pie,
+       :series => {
+         :names => [:a,:b],
+         :values => [70,30],
+         :colors => [:red,:green]
+       },
+       :output_format => :png
+    }
+    post = []
+    a.each_pair do |k,v|
+      post << Curl::PostField.content(k,v)
+    end
+    post.inspect
+  end
+end

data/tests/bugtests.rb

@@ -0,0 +1,9 @@
+$: << $TESTDIR = File.expand_path(File.dirname(__FILE__))
+puts "start"
+begin
+Dir[File.join($TESTDIR, 'bug_*.rb')].each { |lib| require lib }
+rescue Object => e
+  puts e.message
+ensure
+puts "done"
+end