ghazel-curb 0.6.2.3 → 0.7.9.1

data/lib/curb.rb

@@ -26,16 +26,24 @@ module Curl
       def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
         curl = Curl::Easy.new(url, &blk)
         
-        File.open(filename, "wb") do |output|
-          old_on_body = curl.on_body do |data| 
+        output = if filename.is_a? IO
+          filename.binmode if filename.respond_to?(:binmode)
+          filename
+        else
+          File.open(filename, 'wb')
+        end
+        
+        begin
+          old_on_body = curl.on_body do |data|
             result = old_on_body ?  old_on_body.call(data) : data.length
             output << data if result == data.length
             result
           end
-          
           curl.perform
-        end        
-
+        ensure
+          output.close rescue IOError
+        end
+        
         return curl
       end
     end
@@ -142,6 +150,7 @@ module Curl
         m = Curl::Multi.new
         # configure the multi handle
         multi_options.each { |k,v| m.send("#{k}=", v) }
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
 
         urls_with_config.each do|conf|
           c = conf.dup # avoid being destructive to input
@@ -151,6 +160,12 @@ module Curl
 
           easy    = Curl::Easy.new(url)
 
+          # assign callbacks
+          callbacks.each do |cb|
+            cbproc = c.delete(cb)
+            easy.send(cb,&cbproc) if cbproc
+          end
+
           case method
           when :post
             fields = c.delete(:post_fields)
@@ -181,6 +196,80 @@ module Curl
         end
         m.perform
       end
+
+      # call-seq:
+      #
+      # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
+      #
+      # will create 2 new files file1.txt and file2.txt
+      # 
+      # 2 files will be opened, and remain open until the call completes
+      #
+      # when using the :post or :put method, urls should be a hash, including the individual post fields per post
+      #
+      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+        errors = []
+        procs = []
+        files = []
+        urls_with_config = []
+        url_to_download_paths = {}
+
+        urls.each_with_index do|urlcfg,i|
+          if urlcfg.is_a?(Hash)
+            url = url[:url]
+          else
+            url = urlcfg
+          end
+
+          if download_paths and download_paths[i]
+            download_path = download_paths[i]
+          else
+            download_path = File.basename(url)
+          end
+
+          file = lambda do|dp|
+            file = File.open(dp,"wb")
+            procs << (lambda {|data| file.write data; data.size })
+            files << file
+            file
+          end.call(download_path)
+
+          if urlcfg.is_a?(Hash)
+            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+          else
+            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+          end
+          url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
+        end
+
+        if blk
+          # when injecting the block, ensure file is closed before yielding
+          Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
+            info = url_to_download_paths[c.url]
+            begin
+              file = info[:file]
+              files.reject!{|f| f == file }
+              file.close
+            rescue => e
+              errors << e
+            end
+            blk.call(c,info[:path])
+          end
+        else
+          Curl::Multi.http(urls_with_config, multi_options)
+        end
+
+      ensure
+        files.each {|f|
+          begin
+            f.close
+          rescue => e
+            errors << e
+          end
+        }
+        raise errors unless errors.empty?
+      end
+
     end
   end
 end

data/tests/bug_curb_easy_post_with_string_no_content_length_header.rb

@@ -0,0 +1,83 @@
+=begin
+From jwhitmire
+Todd, I'm trying to use curb to post data to a REST url. We're using it to post support questions from our iphone app directly to tender. The post looks good to me, but curl is not adding the content-length header so I get a 411 length required response from the server.
+
+Here's my post block, do you see anything obvious? Do I need to manually add the Content-Length header?
+
+c = Curl::Easy.http_post(url) do |curl|
+  curl.headers["User-Agent"] = "Curl/Ruby"
+  if user
+    curl.headers["X-Multipass"] = user.multipass
+  else
+    curl.headers["X-Tender-Auth"] = TOKEN
+  end
+  curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+  curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&')
+
+  curl.verbose = true
+  curl.follow_location = true
+  curl.enable_cookies = true
+end
+Any insight you care to share would be helpful. Thanks.
+=end
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'webrick'
+class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
+class ::WEBrick::BasicLog ; def log(level, data) ; end ; end
+
+class BugCurbEasyPostWithStringNoContentLengthHeader < Test::Unit::TestCase
+  def test_bug_workaround
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      assert_equal '15', req['Content-Length']
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+    params = {:cat => "hat", :foo => "bar"}
+
+    post_body = params.map{|f,k| "#{Curl::Easy.new.escape(f)}=#{Curl::Easy.new.escape(k)}"}.join('&')
+    c = Curl::Easy.http_post("http://127.0.0.1:9999/test",post_body) do |curl|
+      curl.headers["User-Agent"] = "Curl/Ruby"
+      curl.headers["X-Tender-Auth"] = "A Token"
+      curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+      curl.follow_location = true
+      curl.enable_cookies = true
+    end
+
+    server.shutdown
+    thread.join
+  end
+  def test_bug
+    server = WEBrick::HTTPServer.new( :Port => 9999 )
+    server.mount_proc("/test") do|req,res|
+      assert_equal '15', req['Content-Length']
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+
+    thread = Thread.new(server) do|srv|
+      srv.start
+    end
+    params = {:cat => "hat", :foo => "bar"}
+
+    c = Curl::Easy.http_post("http://127.0.0.1:9999/test") do |curl|
+      curl.headers["User-Agent"] = "Curl/Ruby"
+      curl.headers["X-Tender-Auth"] = "A Token"
+      curl.headers["Accept"] = "application/vnd.tender-v1+json"
+
+      curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&')
+
+      curl.follow_location = true
+      curl.enable_cookies = true
+    end
+
+    server.shutdown
+    thread.join
+  end
+end

data/tests/bug_multi_segfault.rb

@@ -6,5 +6,9 @@
 $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext'))
 $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib'))
 require 'curb'
-multi = Curl::Multi.new
-exit
+
+class BugMultiSegfault < Test::Unit::TestCase
+  def test_bug
+    multi = Curl::Multi.new
+  end
+end

data/tests/bug_postfields_crash.rb

@@ -0,0 +1,26 @@
+# From GICodeWarrior:
+#
+# $ ruby crash_curb.rb 
+# crash_curb.rb:7: [BUG] Segmentation fault
+# ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# 
+# Aborted
+# crash_curb.rb:
+# #!/usr/bin/ruby
+# require 'rubygems'
+# require 'curb'
+# 
+# curl = Curl::Easy.new('http://example.com/')
+# curl.multipart_form_post = true
+# curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'})
+# Ubuntu 9.10
+# curb gem version 0.6.2.1
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class BugPostFieldsCrash < Test::Unit::TestCase
+  def test_crash
+    curl = Curl::Easy.new('http://example.com/')
+    curl.multipart_form_post = true
+    curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'})
+  end
+end

data/tests/bug_postfields_crash2.rb

@@ -0,0 +1,57 @@
+# Not sure if this is an IRB bug, but thought you guys should know.
+#
+# ** My Ruby version: ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# ** Version of Rubygems: 1.3.5
+# ** Version of the Curb gem: 0.6.6.0
+#
+#
+# Transcript of IRB session:
+# ------------------------------------------------------------------------------------------------------------------
+# irb(main):001:0>  a =  {
+# irb(main):002:1*     :type => :pie,
+# irb(main):003:1*     :series => {
+# irb(main):004:2*       :names => [:a,:b],
+# irb(main):005:2*       :values => [70,30],
+# irb(main):006:2*       :colors => [:red,:green]
+# irb(main):007:2>     },
+# irb(main):008:1*     :output_format => :png
+# irb(main):009:1>   }
+# => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}}
+# irb(main):010:0> post = []
+# => []
+# irb(main):011:0> require 'rubygems'
+# => true
+# irb(main):012:0> require 'curb'
+# => true
+# irb(main):013:0> include Curl
+# => Object
+# irb(main):014:0> a.each_pair do |k,v|
+# irb(main):015:1* post << PostField.content(k,v)
+# irb(main):016:1> end
+# => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}}
+# irb(main):017:0> post
+# /usr/lib/ruby/1.8/irb.rb:302: [BUG] Segmentation fault
+# ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+#
+# Aborted
+# ------------------------------------------------------------------------------------------------------------------
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class BugPostFieldsCrash2 < Test::Unit::TestCase
+  def test_crash
+    a =  {
+       :type => :pie,
+       :series => {
+         :names => [:a,:b],
+         :values => [70,30],
+         :colors => [:red,:green]
+       },
+       :output_format => :png
+    }
+    post = []
+    a.each_pair do |k,v|
+      post << Curl::PostField.content(k,v)
+    end
+    post.inspect
+  end
+end

data/tests/bugtests.rb

@@ -0,0 +1,9 @@
+$: << $TESTDIR = File.expand_path(File.dirname(__FILE__))
+puts "start"
+begin
+Dir[File.join($TESTDIR, 'bug_*.rb')].each { |lib| require lib }
+rescue Object => e
+  puts e.message
+ensure
+puts "done"
+end

data/tests/helper.rb

@@ -11,6 +11,7 @@ $:.unshift($EXTDIR)
 
 require 'curb'
 require 'test/unit'
+require 'fileutils'
 
 $TEST_URL = "file://#{URI.escape(File.expand_path(__FILE__).tr('\\','/').tr(':','|'))}"
 
@@ -73,14 +74,18 @@ class TestServlet < WEBrick::HTTPServlet::AbstractServlet
   end
 
   def do_POST(req,res)
-    if req.body
-      params = {}
-      req.body.split('&').map{|s| k,v=s.split('='); params[k] = v }
-    end
-    if params and params['s'] == '500'
-      res.status = 500
+    if req.query['filename'].nil?
+      if req.body
+        params = {}
+        req.body.split('&').map{|s| k,v=s.split('='); params[k] = v }
+      end
+      if params and params['s'] == '500'
+        res.status = 500
+      else
+        respond_with("POST\n#{req.body}",req,res)
+      end
     else
-      respond_with("POST\n#{req.body}",req,res)
+      respond_with(req.query['filename'],req,res)
     end
   end
 
@@ -93,6 +98,14 @@ class TestServlet < WEBrick::HTTPServlet::AbstractServlet
     respond_with(:DELETE,req,res)
   end
 
+  def do_PURGE(req,res)
+    respond_with(:PURGE,req,res)
+  end
+
+  def do_COPY(req,res)
+    respond_with(:COPY,req,res)
+  end
+
 end
 
 module TestServerMethods
@@ -165,5 +178,6 @@ module TestServerMethods
       at_exit{exit_code.call}
 
     end
+  rescue Errno::EADDRINUSE
   end
 end

data/tests/mem_check.rb

@@ -0,0 +1,65 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+#require 'rubygems'
+#require 'rmem'
+
+#
+# Run some tests to measure the memory usage of curb, these tests require fork and ps
+#
+class TestCurbMemory < Test::Unit::TestCase
+
+  def test_easy_memory
+    easy_avg, easy_std = measure_object_memory(Curl::Easy)
+    printf "Easy average: %.2f kilobytes +/- %.2f kilobytes\n", easy_avg.to_f, easy_std.to_f
+
+    multi_avg, multi_std = measure_object_memory(Curl::Multi)
+    printf "Multi average: %.2f kilobytes +/- %.2f kilobytes\n", multi_avg.to_f, multi_std.to_f
+
+    # now that we have the average size of an easy handle lets see how much a multi request consumes with 10 requests
+  end
+
+  def c_avg(report)
+    sum = 0
+    report.each {|r| sum += r.last }
+    (sum.to_f / report.size)
+  end
+
+  def c_std(report,avg)
+    var = 0
+    report.each {|r| var += (r.last-avg)*(r.last-avg) }
+    Math.sqrt(var / (report.size-1))
+  end
+
+  def measure_object_memory(klass)
+    report = []
+    200.times do
+      res = mem_check do
+        obj = klass.new
+      end
+      report << res
+    end
+    avg = c_avg(report)
+    std = c_std(report,avg)
+    [avg,std]
+  end
+
+  def mem_check
+    # see: http://gist.github.com/264060 for inspiration of ps command line
+    rd, wr = IO.pipe
+    memory_usage = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes
+    fork do
+      before = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes
+      rd.close
+      yield
+      after = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes
+      wr.write((after - before))
+      wr.flush
+      wr.close
+    end
+    wr.close
+    total = rd.read.to_i
+    rd.close
+    Process.wait
+    # return the delta and the total
+    [memory_usage, total]
+  end
+end

data/tests/tc_curl_download.rb

@@ -7,7 +7,7 @@ class TestCurbCurlDownload < Test::Unit::TestCase
     server_setup
   end
   
-  def test_download_url_to_file
+  def test_download_url_to_file_via_string
     dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
 
@@ -18,6 +18,49 @@ class TestCurbCurlDownload < Test::Unit::TestCase
     File.unlink(dl_path) if File.exist?(dl_path)
   end
 
+  def test_download_url_to_file_via_file_io
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+    dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
+    io = File.open(dl_path, 'wb')
+
+    curb = Curl::Easy.download(dl_url, io)
+    assert io.closed?
+    assert File.exist?(dl_path)
+    assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path)
+  ensure
+    File.unlink(dl_path) if File.exist?(dl_path)
+  end
+
+  def test_download_url_to_file_via_io
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+    dl_path = File.join(Dir::tmpdir, "dl_url_test.file")
+    reader, writer = IO.pipe
+
+    # Write to local file
+    fork do
+      begin
+        writer.close
+        File.open(dl_path, 'wb') { |file| file << reader.read }
+      ensure
+        reader.close rescue IOError # if the stream has already been closed
+      end
+    end
+
+    # Download remote source
+    begin
+      reader.close
+      curb = Curl::Easy.download(dl_url, writer)
+      Process.wait
+    ensure
+      writer.close rescue IOError # if the stream has already been closed, which occurs in Easy::download
+    end
+
+    assert File.exist?(dl_path)
+    assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path)
+  ensure
+    File.unlink(dl_path) if File.exist?(dl_path)
+  end
+
   def test_download_bad_url_gives_404
     dl_url = "http://127.0.0.1:9129/this_file_does_not_exist.html"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")