curb 0.7.15 → 0.8.4

data/lib/curl/easy.rb

@@ -0,0 +1,468 @@
+module Curl
+  class Easy
+
+    alias post http_post
+    alias put http_put
+
+    #
+    # call-seq:
+    #   easy.status  => String
+    #
+    def status
+      parts = self.header_str.split(/\s/)
+      status = []
+      parts.shift
+      while parts.size > 0 && parts.first != ''
+        status << parts.shift
+      end
+      status.join(' ')
+    end
+
+    #
+    # call-seq:
+    #   easy.set :sym|Fixnum, value
+    # 
+    # set options on the curl easy handle see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
+    #
+    def set(opt,val)
+      if opt.is_a?(Symbol)
+        setopt(sym2curl(opt), val)
+      else
+        setopt(opt.to_i, val)
+      end
+    end
+
+    #
+    # call-seq:
+    #   easy.sym2curl :symbol => Fixnum
+    #
+    #  translates ruby symbols to libcurl options
+    #
+    def sym2curl(opt)
+      Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
+    end
+
+    #
+    # call-seq:
+    #   easy.perform                                     => true
+    #
+    # Transfer the currently configured URL using the options set for this
+    # Curl::Easy instance. If this is an HTTP URL, it will be transferred via
+    # the configured HTTP Verb.
+    #
+    def perform
+      self.multi = Curl::Multi.new if self.multi.nil?
+      self.multi.add self
+      ret = self.multi.perform
+
+      if self.last_result != 0 && self.on_failure.nil?
+        error = Curl::Easy.error(self.last_result) 
+        raise error.first
+      end
+
+      ret
+    end
+
+    #
+    # call-seq:
+    #
+    # easy = Curl::Easy.new
+    # easy.nosignal = true
+    #
+    def nosignal=(onoff)
+      set :nosignal, !!onoff
+    end
+
+    #
+    # call-seq:
+    #   easy = Curl::Easy.new("url") do|c|
+    #    c.delete = true
+    #   end
+    #   easy.perform
+    #
+    def delete=(onoff)
+      set :customrequest, onoff ? 'delete' : nil
+      onoff
+    end
+    # 
+    # call-seq:
+    # 
+    #  easy = Curl::Easy.new("url")
+    #  easy.version = Curl::HTTP_1_1
+    #  easy.version = Curl::HTTP_1_0
+    #  easy.version = Curl::HTTP_NONE
+    # 
+    def version=(http_version)
+      set :http_version, http_version
+    end
+
+    # 
+    # call-seq:
+    #   easy.url = "http://some.url/"                    => "http://some.url/"
+    # 
+    # Set the URL for subsequent calls to +perform+. It is acceptable
+    # (and even recommended) to reuse Curl::Easy instances by reassigning
+    # the URL between calls to +perform+.
+    # 
+    def url=(u)
+      set :url, u
+    end
+
+    # 
+    # call-seq:
+    #   easy.proxy_url = string                          => string
+    # 
+    # Set the URL of the HTTP proxy to use for subsequent calls to +perform+.
+    # The URL should specify the the host name or dotted IP address. To specify
+    # port number in this string, append :[port] to the end of the host name.
+    # The proxy string may be prefixed with [protocol]:// since any such prefix
+    # will be ignored. The proxy's port number may optionally be specified with
+    # the separate option proxy_port .
+    # 
+    # When you tell the library to use an HTTP proxy, libcurl will transparently
+    # convert operations to HTTP even if you specify an FTP URL etc. This may have
+    # an impact on what other features of the library you can use, such as
+    # FTP specifics that don't work unless you tunnel through the HTTP proxy. Such
+    # tunneling is activated with proxy_tunnel = true.
+    # 
+    # libcurl respects the environment variables *http_proxy*, *ftp_proxy*,
+    # *all_proxy* etc, if any of those is set. The proxy_url option does however
+    # override any possibly set environment variables.
+    # 
+    # Starting with libcurl 7.14.1, the proxy host string given in environment
+    # variables can be specified the exact same way as the proxy can be set with
+    # proxy_url, including protocol prefix (http://) and embedded user + password.
+    #
+    def proxy_url=(url)
+      set :proxy, url
+    end
+
+    def ssl_verify_host=(value)
+      value = 1 if value.class == TrueClass
+      value = 0 if value.class == FalseClass
+      self.ssl_verify_host_integer=value
+    end
+
+    # 
+    # call-seq:
+    #   easy.ssl_verify_host?                            => boolean
+    # 
+    # Deprecated: call easy.ssl_verify_host instead
+    # can be one of [0,1,2]
+    # 
+    # Determine whether this Curl instance will verify that the server cert
+    # is for the server it is known as.
+    # 
+    def ssl_verify_host?
+      ssl_verify_host.nil? ? false : (ssl_verify_host > 0)
+    end
+
+    # 
+    # call-seq:
+    #   easy.interface = string                          => string
+    # 
+    # Set the interface name to use as the outgoing network interface.
+    # The name can be an interface name, an IP address or a host name.
+    # 
+    def interface=(value)
+      set :interface, value
+    end
+
+    # 
+    # call-seq:
+    #   easy.userpwd = string                            => string
+    # 
+    # Set the username/password string to use for subsequent calls to +perform+.
+    # The supplied string should have the form "username:password"
+    # 
+    def userpwd=(value)
+      set :userpwd, value
+    end
+
+    # 
+    # call-seq:
+    #   easy.proxypwd = string                           => string
+    # 
+    # Set the username/password string to use for proxy connection during
+    # subsequent calls to +perform+. The supplied string should have the
+    # form "username:password"
+    # 
+    def proxypwd=(value)
+      set :proxyuserpwd, value
+    end
+
+    # 
+    # call-seq:
+    #   easy.cookies = "name1=content1; name2=content2;" => string
+    # 
+    # Set cookies to be sent by this Curl::Easy instance. The format of the string should
+    # be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
+    # Set multiple cookies in one string like this: "name1=content1; name2=content2;" etc.
+    # 
+    def cookies=(value)
+      set :cookie, value
+    end
+
+    # 
+    # call-seq:
+    #   easy.cookiefile = string                         => string
+    # 
+    # Set a file that contains cookies to be sent in subsequent requests by this Curl::Easy instance.
+    # 
+    # *Note* that you must set enable_cookies true to enable the cookie
+    # engine, or this option will be ignored.
+    # 
+    def cookiefile=(value)
+      set :cookiefile, value
+    end
+
+    # 
+    # call-seq:
+    #   easy.cookiejar = string                          => string
+    # 
+    # Set a cookiejar file to use for this Curl::Easy instance.
+    # Cookies from the response will be written into this file.
+    # 
+    # *Note* that you must set enable_cookies true to enable the cookie
+    # engine, or this option will be ignored.
+    # 
+    def cookiejar=(value)
+      set :cookiejar, value
+    end
+
+    #
+    # call-seq:
+    #  easy = Curl::Easy.new("url") do|c|
+    #   c.head = true
+    #  end
+    #  easy.perform
+    #
+    def head=(onoff)
+      set :nobody, onoff
+    end
+
+    #
+    # call-seq:
+    #   easy.follow_location = boolean                   => boolean
+    #
+    # Configure whether this Curl instance will follow Location: headers
+    # in HTTP responses. Redirects will only be followed to the extent
+    # specified by +max_redirects+.
+    #
+    def follow_location=(onoff)
+      set :followlocation, onoff
+    end
+
+    #
+    # call-seq:
+    #   easy.http_head                                   => true
+    #
+    # Request headers from the currently configured URL using the HEAD
+    # method and current options set for this Curl::Easy instance. This
+    # method always returns true, or raises an exception (defined under
+    # Curl::Err) on error.
+    #
+    def http_head
+      set :nobody, true
+      ret = self.perform
+      set :nobody, false
+      ret
+    end
+
+    #
+    # call-seq:
+    #   easy.http_get                                    => true
+    #
+    # GET the currently configured URL using the current options set for
+    # this Curl::Easy instance. This method always returns true, or raises
+    # an exception (defined under Curl::Err) on error.
+    #
+    def http_get
+      set :httpget, true
+      http :GET
+    end
+    alias get http_get
+
+    #
+    # call-seq:
+    #   easy.http_delete
+    #
+    # DELETE the currently configured URL using the current options set for
+    # this Curl::Easy instance. This method always returns true, or raises
+    # an exception (defined under Curl::Err) on error.
+    #
+    def http_delete
+      self.http :DELETE
+    end
+    alias delete http_delete
+
+    class << self
+
+      #
+      # call-seq:
+      #   Curl::Easy.perform(url) { |easy| ... }           => #&lt;Curl::Easy...&gt;
+      #
+      # Convenience method that creates a new Curl::Easy instance with
+      # the specified URL and calls the general +perform+ method, before returning
+      # the new instance. For HTTP URLs, this is equivalent to calling +http_get+.
+      #
+      # If a block is supplied, the new instance will be yielded just prior to
+      # the +http_get+ call.
+      #
+      def perform(*args)
+        c = Curl::Easy.new(*args)
+        yield c if block_given?
+        c.perform
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_get(url) { |easy| ... }          => #&lt;Curl::Easy...&gt;
+      #
+      # Convenience method that creates a new Curl::Easy instance with
+      # the specified URL and calls +http_get+, before returning the new instance.
+      #
+      # If a block is supplied, the new instance will be yielded just prior to
+      # the +http_get+ call.
+      #
+      def http_get(*args)
+        c = Curl::Easy.new(*args)
+        yield c if block_given?
+        c.http_get
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_head(url) { |easy| ... }         => #&lt;Curl::Easy...&gt;
+      #
+      # Convenience method that creates a new Curl::Easy instance with
+      # the specified URL and calls +http_head+, before returning the new instance.
+      #
+      # If a block is supplied, the new instance will be yielded just prior to
+      # the +http_head+ call.
+      #
+      def http_head(*args)
+        c = Curl::Easy.new(*args)
+        yield c if block_given?
+        c.http_head
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_put(url, data) {|c| ... }
+      #
+      # see easy.http_put
+      #
+      def http_put(url, data)
+        c = Curl::Easy.new url
+        yield c if block_given?
+        c.http_put data
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_post(url, "some=urlencoded%20form%20data&and=so%20on") => true
+      #   Curl::Easy.http_post(url, "some=urlencoded%20form%20data", "and=so%20on", ...) => true
+      #   Curl::Easy.http_post(url, "some=urlencoded%20form%20data", Curl::PostField, "and=so%20on", ...) => true
+      #   Curl::Easy.http_post(url, Curl::PostField, Curl::PostField ..., Curl::PostField) => true
+      #
+      # POST the specified formdata to the currently configured URL using
+      # the current options set for this Curl::Easy instance. This method
+      # always returns true, or raises an exception (defined under
+      # Curl::Err) on error.
+      #
+      # If you wish to use multipart form encoding, you'll need to supply a block
+      # in order to set multipart_form_post true. See #http_post for more
+      # information.
+      #
+      def http_post(*args)
+        url = args.shift
+        c = Curl::Easy.new url
+        yield c if block_given?
+        c.http_post(*args)
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_delete(url) { |easy| ... }       => #&lt;Curl::Easy...&gt;
+      #
+      # Convenience method that creates a new Curl::Easy instance with
+      # the specified URL and calls +http_delete+, before returning the new instance.
+      #
+      # If a block is supplied, the new instance will be yielded just prior to
+      # the +http_delete+ call.
+      #
+      def http_delete(*args)
+        c = Curl::Easy.new(*args)
+        yield c if block_given?
+        c.http_delete
+        c
+      end
+    
+      # call-seq:
+      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   
+      # Stream the specified url (via perform) and save the data directly to the
+      # supplied filename (defaults to the last component of the URL path, which will 
+      # usually be the filename most simple urls).
+      # 
+      # If a block is supplied, it will be passed the curl instance prior to the
+      # perform call.
+      # 
+      # *Note* that the semantics of the on_body handler are subtly changed when using
+      # download, to account for the automatic routing of data to the specified file: The 
+      # data string is passed to the handler *before* it is written
+      # to the file, allowing the handler to perform mutative operations where 
+      # necessary. As usual, the transfer will be aborted if the on_body handler
+      # returns a size that differs from the data chunk size - in this case, the 
+      # offending chunk will *not* be written to the file, the file will be closed,
+      # and a Curl::Err::AbortedByCallbackError will be raised.
+      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+        curl = Curl::Easy.new(url, &blk)
+        
+        output = if filename.is_a? IO
+          filename.binmode if filename.respond_to?(:binmode)
+          filename
+        else
+          File.open(filename, 'wb')
+        end
+        
+        begin
+          old_on_body = curl.on_body do |data|
+            result = old_on_body ?  old_on_body.call(data) : data.length
+            output << data if result == data.length
+            result
+          end
+          curl.perform
+        ensure
+          output.close rescue IOError
+        end
+        
+        return curl
+      end
+    end
+
+    # Allow the incoming cert string to be file:password
+    # but be careful to not use a colon from a windows file path
+    # as the split point. Mimic what curl's main does
+    if respond_to?(:cert=)
+    alias_method :native_cert=, :cert=
+    def cert=(cert_file)
+      pos = cert_file.rindex(':')
+      if pos && pos > 1
+        self.native_cert= cert_file[0..pos-1]
+        self.certpassword= cert_file[pos+1..-1]
+      else
+        self.native_cert= cert_file
+      end
+      self.cert
+    end
+    end
+
+  end
+end

data/lib/curl/multi.rb

@@ -0,0 +1,248 @@
+module Curl
+
+  class Multi
+    class << self
+      # call-seq:
+      #   Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
+      #     easy
+      #   end
+      # 
+      # Blocking call to fetch multiple url's in parallel.
+      def get(urls, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls.each do|url|
+          url_confs << {:url => url, :method => :get}.merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
+      #                    { :follow_location => true, :multipart_form_post => true },
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def post(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :post).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
+      #                    {:url => 'url2', :put_data => IO.read('filepath')},
+      #                    {:url => 'url3', :put_data => "maybe another string or socket?"],
+      #                    {:follow_location => true},
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def put(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :put).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+
+      # call-seq:
+      #
+      # Curl::Multi.http( [
+      #   { :url => 'url1', :method => :post,
+      #     :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
+      #   { :url => 'url2', :method => :get,
+      #     :follow_location => true, :max_redirects => 3 },
+      #   { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
+      #   { :url => 'url4', :method => :head }
+      # ], {:pipeline => true})
+      #
+      # Blocking call to issue multiple HTTP requests with varying verb's.
+      #
+      # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
+      # multi_options: options for the multi handle 
+      # blk: a callback, that yeilds when a handle is completed
+      #
+      def http(urls_with_config, multi_options={}, &blk)
+        m = Curl::Multi.new
+
+        # maintain a sane number of easy handles
+        multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10
+
+        free_handles = [] # keep a list of free easy handles
+
+        # configure the multi handle
+        multi_options.each { |k,v| m.send("#{k}=", v) }
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
+
+        add_free_handle = proc do|conf, easy|
+          c       = conf.dup # avoid being destructive to input
+          url     = c.delete(:url)
+          method  = c.delete(:method)
+          headers = c.delete(:headers)
+
+          easy    = Curl::Easy.new if easy.nil?
+
+          easy.url = url
+
+          # assign callbacks
+          callbacks.each do |cb|
+            cbproc = c.delete(cb)
+            easy.send(cb,&cbproc) if cbproc
+          end
+
+          case method
+          when :post
+            fields = c.delete(:post_fields)
+            # set the post post using the url fields
+            easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
+          when :put
+            easy.put_data = c.delete(:put_data)
+          when :head
+            easy.head = true
+          when :delete
+            easy.delete = true
+          when :get
+          else
+            # XXX: nil is treated like a GET
+          end
+
+          # headers is a special key
+          headers.each {|k,v| easy.headers[k] = v } if headers
+ 
+          #
+          # use the remaining options as specific configuration to the easy handle
+          # bad options should raise an undefined method error
+          #
+          c.each { |k,v| easy.send("#{k}=",v) }
+
+          easy.on_complete {|curl,code|
+            free_handles << curl
+            blk.call(curl,code,method) if blk
+          }
+          m.add(easy)
+        end
+
+        max_connects.times do
+          conf = urls_with_config.pop
+          add_free_handle.call conf, nil
+          break if urls_with_config.empty?
+        end
+
+        consume_free_handles = proc do
+          # as we idle consume free handles
+          if urls_with_config.size > 0 && free_handles.size > 0
+            easy = free_handles.pop
+            conf = urls_with_config.pop
+            add_free_handle.call conf, easy
+          end
+        end
+
+        if urls_with_config.empty?
+          m.perform
+        else
+          until urls_with_config.empty?
+            m.perform do
+              consume_free_handles.call
+            end
+            consume_free_handles.call
+          end
+          free_handles = nil
+        end
+      end
+
+      # call-seq:
+      #
+      # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
+      #
+      # will create 2 new files file1.txt and file2.txt
+      # 
+      # 2 files will be opened, and remain open until the call completes
+      #
+      # when using the :post or :put method, urls should be a hash, including the individual post fields per post
+      #
+      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+        errors = []
+        procs = []
+        files = []
+        urls_with_config = []
+        url_to_download_paths = {}
+
+        urls.each_with_index do|urlcfg,i|
+          if urlcfg.is_a?(Hash)
+            url = url[:url]
+          else
+            url = urlcfg
+          end
+
+          if download_paths and download_paths[i]
+            download_path = download_paths[i]
+          else
+            download_path = File.basename(url)
+          end
+
+          file = lambda do|dp|
+            file = File.open(dp,"wb")
+            procs << (lambda {|data| file.write data; data.size })
+            files << file
+            file
+          end.call(download_path)
+
+          if urlcfg.is_a?(Hash)
+            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+          else
+            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+          end
+          url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
+        end
+
+        if blk
+          # when injecting the block, ensure file is closed before yielding
+          Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
+            info = url_to_download_paths[c.url]
+            begin
+              file = info[:file]
+              files.reject!{|f| f == file }
+              file.close
+            rescue => e
+              errors << e
+            end
+            blk.call(c,info[:path])
+          end
+        else
+          Curl::Multi.http(urls_with_config, multi_options)
+        end
+
+      ensure
+        files.each {|f|
+          begin
+            f.close
+          rescue => e
+            errors << e
+          end
+        }
+        raise errors unless errors.empty?
+      end
+
+    end
+  end
+end