fotonauts-curb 0.7.7

data/ext/curb_postfield.h

@@ -0,0 +1,40 @@
+/* curb_postfield.h - Field class for POST method
+ * Copyright (c)2006 Ross Bamford. 
+ * Licensed under the Ruby License. See LICENSE for details.
+ * 
+ * $Id: curb_postfield.h 4 2006-11-17 18:35:31Z roscopeco $
+ */
+#ifndef __CURB_POSTFIELD_H
+#define __CURB_POSTFIELD_H
+
+#include "curb.h"
+
+/* 
+ * postfield doesn't actually wrap a curl_httppost - instead,
+ * it just holds together some ruby objects and has a C-side 
+ * method to add it to a given form list during the perform.
+ */
+typedef struct {  
+  /* Objects we associate */
+  VALUE name;  
+  VALUE content;
+  VALUE content_type;
+  VALUE content_proc;
+  VALUE local_file;
+  VALUE remote_file;
+  
+  /* this will sometimes hold a string, which is the result
+   * of the content_proc invocation. We need it to hang around.
+   */
+  VALUE buffer_str;
+} ruby_curl_postfield;
+
+extern VALUE cCurlPostField;
+
+void append_to_form(VALUE self, 
+                    struct curl_httppost **first, 
+                    struct curl_httppost **last);
+
+void init_curb_postfield();
+
+#endif

data/ext/curb_upload.c

@@ -0,0 +1,80 @@
+/* curb_upload.c - Curl upload handle
+ * Copyright (c)2009 Todd A Fisher. 
+ * Licensed under the Ruby License. See LICENSE for details.
+ */
+#include "curb_upload.h"
+extern VALUE mCurl;
+VALUE cCurlUpload;
+
+#ifdef RDOC_NEVER_DEFINED
+  mCurl = rb_define_module("Curl");
+#endif
+
+static void curl_upload_mark(ruby_curl_upload *rbcu) {
+  if (rbcu->stream) rb_gc_mark(rbcu->stream);
+}
+static void curl_upload_free(ruby_curl_upload *rbcu) {
+  free(rbcu);
+}
+
+/*
+ * call-seq:
+ *   internal class for sending large file uploads
+ */
+VALUE ruby_curl_upload_new(VALUE klass) {
+  VALUE upload;
+  ruby_curl_upload *rbcu = ALLOC(ruby_curl_upload);
+  rbcu->stream = Qnil;
+  rbcu->offset = 0;
+  upload = Data_Wrap_Struct(klass, curl_upload_mark, curl_upload_free, rbcu);
+  return upload;
+}
+
+/*
+ * call-seq:
+ *   internal class for sending large file uploads
+ */
+VALUE ruby_curl_upload_stream_set(VALUE self, VALUE stream) {
+  ruby_curl_upload *rbcu;
+  Data_Get_Struct(self, ruby_curl_upload, rbcu);
+  rbcu->stream = stream;
+  return stream;
+}
+/*
+ * call-seq:
+ *   internal class for sending large file uploads
+ */
+VALUE ruby_curl_upload_stream_get(VALUE self) {
+  ruby_curl_upload *rbcu;
+  Data_Get_Struct(self, ruby_curl_upload, rbcu);
+  return rbcu->stream;
+}
+/*
+ * call-seq:
+ *   internal class for sending large file uploads
+ */
+VALUE ruby_curl_upload_offset_set(VALUE self, VALUE offset) {
+  ruby_curl_upload *rbcu;
+  Data_Get_Struct(self, ruby_curl_upload, rbcu);
+  rbcu->offset = FIX2INT(offset);
+  return offset;
+}
+/*
+ * call-seq:
+ *   internal class for sending large file uploads
+ */
+VALUE ruby_curl_upload_offset_get(VALUE self) {
+  ruby_curl_upload *rbcu;
+  Data_Get_Struct(self, ruby_curl_upload, rbcu);
+  return INT2FIX(rbcu->offset);
+}
+
+/* =================== INIT LIB =====================*/
+void init_curb_upload() {
+  cCurlUpload = rb_define_class_under(mCurl, "Upload", rb_cObject);
+  rb_define_singleton_method(cCurlUpload, "new", ruby_curl_upload_new, 0);
+  rb_define_method(cCurlUpload, "stream=", ruby_curl_upload_stream_set, 1);
+  rb_define_method(cCurlUpload, "stream", ruby_curl_upload_stream_get, 0);
+  rb_define_method(cCurlUpload, "offset=", ruby_curl_upload_offset_set, 1);
+  rb_define_method(cCurlUpload, "offset", ruby_curl_upload_offset_get, 0);
+}

data/ext/curb_upload.h

@@ -0,0 +1,30 @@
+/* curb_upload.h - Curl upload handle
+ * Copyright (c)2009 Todd A Fisher. 
+ * Licensed under the Ruby License. See LICENSE for details.
+ */
+#ifndef __CURB_UPLOAD_H
+#define __CURB_UPLOAD_H
+
+#include "curb.h"
+
+#include <curl/easy.h>
+
+/*
+ * Maintain the state of an upload e.g. for putting large streams with very little memory
+ * out to a server. via PUT requests
+ */
+typedef struct {
+  VALUE stream;
+  size_t offset;
+} ruby_curl_upload;
+
+extern VALUE cCurlUpload;
+void init_curb_upload();
+
+VALUE ruby_curl_upload_new(VALUE klass);
+VALUE ruby_curl_upload_stream_set(VALUE self, VALUE stream);
+VALUE ruby_curl_upload_stream_get(VALUE self);
+VALUE ruby_curl_upload_offset_set(VALUE self, VALUE offset);
+VALUE ruby_curl_upload_offset_get(VALUE self);
+
+#endif

data/ext/extconf.rb

@@ -0,0 +1,177 @@
+require 'mkmf'
+puts $CFLAGS.inspect
+
+dir_config('curl')
+
+if find_executable('curl-config')
+  $CFLAGS << " #{`curl-config --cflags`.strip} -g"
+  if ENV['STATIC_BUILD']
+    $LIBS << " #{`curl-config --static-libs`.strip}"
+  else
+    $LIBS << " #{`curl-config --libs`.strip}"
+  end
+  ca_bundle_path=`curl-config --ca`.strip
+  if !ca_bundle_path.nil? and ca_bundle_path != ''
+    $defs.push( %{-D HAVE_CURL_CONFIG_CA} )
+    $defs.push( %{-D CURL_CONFIG_CA='#{ca_bundle_path.inspect}'} )
+  end
+elsif !have_library('curl') or !have_header('curl/curl.h')
+  fail <<-EOM
+  Can't find libcurl or curl/curl.h
+
+  Try passing --with-curl-dir or --with-curl-lib and --with-curl-include
+  options to extconf.
+  EOM
+end
+
+# Check arch flags
+# TODO: detect mismatched arch types when libcurl mac ports is mixed with native mac ruby or vice versa
+#archs = $CFLAGS.scan(/-arch\s(.*?)\s/).first # get the first arch flag
+#if archs and archs.size >= 1
+#  # need to reduce the number of archs...
+#  # guess the first one is correct... at least the first one is probably the ruby installed arch...
+#  # this could lead to compiled binaries that crash at runtime...
+#  $CFLAGS.gsub!(/-arch\s(.*?)\s/,' ')
+#  $CFLAGS << " -arch #{archs.first}"
+#  puts "Selected arch: #{archs.first}"
+#end
+
+def define(s)
+  $defs.push( format("-D HAVE_%s", s.to_s.upcase) )
+end
+
+def have_constant(name)
+  checking_for name do
+    src = %{
+      #include <curl/curl.h>
+      int main() {
+        int test = (int)#{name.upcase};
+        return 0;
+      }
+    }
+    if try_compile(src,"#{$CFLAGS} #{$LIBS}")
+      define name
+      true
+    else
+      false
+    end
+  end
+end
+
+have_constant "curlinfo_redirect_time"
+have_constant "curlinfo_response_code"
+have_constant "curlinfo_filetime"
+have_constant "curlinfo_redirect_count"
+have_constant "curlinfo_os_errno"
+have_constant "curlinfo_num_connects"
+have_constant "curlinfo_ftp_entry_path"
+have_constant "curl_version_ssl"
+have_constant "curl_version_libz"
+have_constant "curl_version_ntlm"
+have_constant "curl_version_gssnegotiate"
+have_constant "curl_version_debug"
+have_constant "curl_version_asynchdns"
+have_constant "curl_version_spnego"
+have_constant "curl_version_largefile"
+have_constant "curl_version_idn"
+have_constant "curl_version_sspi"
+have_constant "curl_version_conv"
+have_constant "curlproxy_http"
+have_constant "curlproxy_socks4"
+have_constant "curlproxy_socks5"
+have_constant "curlauth_basic"
+have_constant "curlauth_digest"
+have_constant "curlauth_gssnegotiate"
+have_constant "curlauth_ntlm"
+have_constant "curlauth_anysafe"
+have_constant "curlauth_any"
+have_constant "curle_tftp_notfound"
+have_constant "curle_tftp_perm"
+have_constant "curle_tftp_diskfull"
+have_constant "curle_tftp_illegal"
+have_constant "curle_tftp_unknownid"
+have_constant "curle_tftp_exists"
+have_constant "curle_tftp_nosuchuser"
+# older versions of libcurl 7.12
+have_constant "curle_send_fail_rewind"
+have_constant "curle_ssl_engine_initfailed"
+have_constant "curle_login_denied"
+
+# older than 7.16.3
+have_constant "curlmopt_maxconnects"
+
+# additional consts
+have_constant "curle_conv_failed"
+have_constant "curle_conv_reqd"
+have_constant "curle_ssl_cacert_badfile"
+have_constant "curle_remote_file_not_found"
+have_constant "curle_ssh"
+have_constant "curle_ssl_shutdown_failed"
+have_constant "curle_again"
+have_constant "curle_ssl_crl_badfile"
+have_constant "curle_ssl_issuer_error"
+
+# username/password added in 7.19.1
+have_constant "curlopt_username"
+have_constant "curlopt_password"
+have_constant "curlinfo_primary_ip"
+
+# ie quirk added in 7.19.3
+have_constant "curlauth_digest_ie"
+
+# added in 7.15.1
+have_constant "curlftpmethod_multicwd"
+have_constant "curlftpmethod_nocwd"
+have_constant "curlftpmethod_singlecwd"
+
+# centos 4.5 build of libcurl
+have_constant "curlm_bad_socket"
+have_constant "curlm_unknown_option"
+have_func("curl_multi_timeout")
+have_func("curl_multi_fdset")
+have_func("curl_multi_perform")
+
+if try_compile('int main() { return 0; }','-Wall')
+  $CFLAGS << ' -Wall'
+end
+
+# do some checking to detect ruby 1.8 hash.c vs ruby 1.9 hash.c
+def test_for(name, const, src)
+  checking_for name do
+    if try_compile(src,"#{$CFLAGS} #{$LIBS}")
+      define const
+      true
+    else
+      false
+    end
+  end
+end
+test_for("Ruby 1.9 Hash", "RUBY19_HASH", %{
+  #include <ruby.h>
+  int main() {
+    VALUE hash = rb_hash_new();
+    if( RHASH(hash)->ntbl->num_entries ) {
+      return 0;
+    }
+    return 1;
+  }
+})
+test_for("Ruby 1.9 st.h", "RUBY19_ST_H", %{
+  #include <ruby.h>
+  #include <ruby/st.h>
+  int main() {
+    return 0;
+  }
+})
+
+test_for("curl_easy_escape", "CURL_EASY_ESCAPE", %{
+  #include <curl/curl.h>
+  int main() {
+    CURL *easy = curl_easy_init();
+    curl_easy_escape(easy,"hello",5);
+    return 0;
+  }
+})
+
+create_header('curb_config.h')
+create_makefile('curb_core')

data/lib/curb.rb

@@ -0,0 +1,255 @@
+require 'curb_core'
+
+module Curl
+
+  class Easy
+    class << self
+    
+      # call-seq:
+      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   
+      # Stream the specified url (via perform) and save the data directly to the
+      # supplied filename (defaults to the last component of the URL path, which will 
+      # usually be the filename most simple urls).
+      # 
+      # If a block is supplied, it will be passed the curl instance prior to the
+      # perform call.
+      # 
+      # *Note* that the semantics of the on_body handler are subtly changed when using
+      # download, to account for the automatic routing of data to the specified file: The 
+      # data string is passed to the handler *before* it is written
+      # to the file, allowing the handler to perform mutative operations where 
+      # necessary. As usual, the transfer will be aborted if the on_body handler
+      # returns a size that differs from the data chunk size - in this case, the 
+      # offending chunk will *not* be written to the file, the file will be closed,
+      # and a Curl::Err::AbortedByCallbackError will be raised.
+      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+        curl = Curl::Easy.new(url, &blk)
+        
+        File.open(filename, "wb") do |output|
+          old_on_body = curl.on_body do |data| 
+            result = old_on_body ?  old_on_body.call(data) : data.length
+            output << data if result == data.length
+            result
+          end
+          
+          curl.perform
+        end        
+
+        return curl
+      end
+    end
+
+    # Allow the incoming cert string to be file:password
+    # but be careful to not use a colon from a windows file path
+    # as the split point. Mimic what curl's main does
+    alias_method :native_cert=, :cert=
+    def cert=(cert_file)
+      pos = cert_file.rindex(':')
+      if pos && pos > 1
+        self.native_cert= cert_file[0..pos-1]
+        self.certpassword= cert_file[pos+1..-1]
+      else
+        self.native_cert= cert_file
+      end
+      self.cert
+    end
+  end
+
+  class Multi
+    class << self
+      # call-seq:
+      #   Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
+      #     easy
+      #   end
+      # 
+      # Blocking call to fetch multiple url's in parallel.
+      def get(urls, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls.each do|url|
+          url_confs << {:url => url, :method => :get}.merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
+      #                     {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
+      #                    { :follow_location => true, :multipart_form_post => true },
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def post(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :post).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+      # call-seq:
+      #
+      #   Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
+      #                    {:url => 'url2', :put_data => IO.read('filepath')},
+      #                    {:url => 'url3', :put_data => "maybe another string or socket?"],
+      #                    {:follow_location => true},
+      #                    {:pipeline => true }) do|easy|
+      #     easy_handle_on_request_complete
+      #   end
+      # 
+      # Blocking call to POST multiple form's in parallel.
+      # 
+      # urls_with_config: is a hash of url's pointing to the postfields to send 
+      # easy_options: are a set of common options to set on all easy handles
+      # multi_options: options to set on the Curl::Multi handle
+      #
+      def put(urls_with_config, easy_options={}, multi_options={}, &blk)
+        url_confs = []
+        urls_with_config.each do|uconf|
+          url_confs << uconf.merge(:method => :put).merge(easy_options)
+        end
+        self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
+      end
+
+
+      # call-seq:
+      #
+      # Curl::Multi.http( [
+      #   { :url => 'url1', :method => :post,
+      #     :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
+      #   { :url => 'url2', :method => :get,
+      #     :follow_location => true, :max_redirects => 3 },
+      #   { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
+      #   { :url => 'url4', :method => :head }
+      # ], {:pipeline => true})
+      #
+      # Blocking call to issue multiple HTTP requests with varying verb's.
+      #
+      # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
+      # multi_options: options for the multi handle 
+      # blk: a callback, that yeilds when a handle is completed
+      #
+      def http(urls_with_config, multi_options={}, &blk)
+        m = Curl::Multi.new
+        # configure the multi handle
+        multi_options.each { |k,v| m.send("#{k}=", v) }
+        callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
+
+        urls_with_config.each do|conf|
+          c = conf.dup # avoid being destructive to input
+          url     = c.delete(:url)
+          method  = c.delete(:method)
+          headers = c.delete(:headers)
+
+          easy    = Curl::Easy.new(url)
+
+          # assign callbacks
+          callbacks.each do |cb|
+            cbproc = c.delete(cb)
+            easy.send(cb,&cbproc) if cbproc
+          end
+
+          case method
+          when :post
+            fields = c.delete(:post_fields)
+            # set the post post using the url fields
+            easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
+          when :put
+            easy.put_data = c.delete(:put_data)
+          when :head
+            easy.head = true
+          when :delete
+            easy.delete = true
+          when :get
+          else
+            # XXX: nil is treated like a GET
+          end
+
+          # headers is a special key
+          headers.each {|k,v| easy.headers[k] = v } if headers
+ 
+          #
+          # use the remaining options as specific configuration to the easy handle
+          # bad options should raise an undefined method error
+          #
+          c.each { |k,v| easy.send("#{k}=",v) }
+
+          easy.on_complete {|curl,code| blk.call(curl,code,method) } if blk
+          m.add(easy)
+        end
+        m.perform
+      end
+
+      # call-seq:
+      #
+      # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
+      #
+      # will create 2 new files file1.txt and file2.txt
+      # 
+      # 2 files will be opened, and remain open until the call completes
+      #
+      # when using the :post or :put method, urls should be a hash, including the individual post fields per post
+      #
+      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+        procs = []
+        files = []
+        urls_with_config = []
+        url_to_download_paths = {}
+
+        urls.each_with_index do|urlcfg,i|
+          if urlcfg.is_a?(Hash)
+            url = url[:url]
+          else
+            url = urlcfg
+          end
+
+          if download_paths and download_paths[i]
+            download_path = download_paths[i]
+          else
+            download_path = File.basename(url)
+          end
+
+          lambda do|dp|
+            file = File.open(dp,"wb")
+            procs << (lambda {|data| file.write data; data.size })
+            files << file
+          end.call(download_path)
+
+          if urlcfg.is_a?(Hash)
+            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+          else
+            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+          end
+          url_to_download_paths[url] = download_path # store for later
+        end
+
+        if blk
+          Curl::Multi.http(urls_with_config, multi_options) {|c,code,method| blk.call(c,url_to_download_paths[c.url]) }
+        else
+          Curl::Multi.http(urls_with_config, multi_options)
+        end
+
+      ensure
+        errors = []
+        files.each {|f|
+          begin
+            f.close
+          rescue => e
+            errors << e
+          end
+        }
+        raise errors unless errors.empty?
+      end
+
+    end
+  end
+end