typhoeus_curly 0.1.14

data/ext/typhoeus/typhoeus_easy.h

@@ -0,0 +1,19 @@
+#ifndef TYPHOEUS_EASY
+#define TYPHOEUS_EASY
+
+#include <native.h>
+
+void init_typhoeus_easy();
+typedef struct {
+  const char *memory;
+  int size;
+  int read;
+} RequestChunk;
+
+typedef struct {
+  RequestChunk *request_chunk;
+  CURL *curl;
+  struct curl_slist *headers;
+} CurlEasy;
+
+#endif

data/ext/typhoeus/typhoeus_multi.c

@@ -0,0 +1,225 @@
+#include <typhoeus_multi.h>
+
+static void multi_read_info(VALUE self, CURLM *multi_handle);
+
+static void dealloc(CurlMulti *curl_multi) {
+  curl_multi_cleanup(curl_multi->multi);
+  free(curl_multi);
+}
+
+static VALUE multi_add_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  CURLMcode mcode;
+
+  mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
+  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
+    rb_raise((VALUE)mcode, "An error occured adding the handle");
+  }
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
+  curl_multi->active++;
+
+  if (mcode == CURLM_CALL_MULTI_PERFORM) {
+    curl_multi_perform(curl_multi->multi, &(curl_multi->running));
+  }
+  // 
+  // if (curl_multi->running) {
+  //     printf("call read_info on add<br/>");
+  //   multi_read_info(self, curl_multi->multi);
+  // }
+
+  return easy;
+}
+
+static VALUE multi_remove_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi->active--;
+  curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);
+
+  return easy;
+}
+
+static void multi_read_info(VALUE self, CURLM *multi_handle) {
+  int msgs_left, result;
+  CURLMsg *msg;
+  CURLcode ecode;
+  CURL *easy_handle;
+  VALUE easy;
+
+  /* check for finished easy handles and remove from the multi handle */
+  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
+
+    if (msg->msg != CURLMSG_DONE) {
+      continue;
+    }
+
+    easy_handle = msg->easy_handle;
+    result = msg->data.result;
+    if (easy_handle) {
+      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
+      if (ecode != 0) {
+        rb_raise(ecode, "error getting easy object");
+      }
+
+      long response_code = -1;
+      curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
+
+      // TODO: find out what the real problem is here and fix it.
+      // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
+      // fail intermittently and return this result number. However, it will succeed if you try it a few
+      // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!
+
+      // Sandofsky says:
+      // This is caused by OS X first attempting to resolve using IPV6.
+      // Hack solution: connect to yourself with 127.0.0.1, not localhost
+      // http://curl.haxx.se/mail/tracker-2009-09/0018.html
+      if (result == 7) {
+        VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
+        if (max_retries != Qtrue) {
+          multi_remove_handle(self, easy);
+          multi_add_handle(self, easy);
+          CurlMulti *curl_multi;
+          Data_Get_Struct(self, CurlMulti, curl_multi);
+          curl_multi_perform(curl_multi->multi, &(curl_multi->running));
+
+          rb_funcall(easy, rb_intern("increment_retries"), 0);
+
+          continue;
+        }
+      }
+      multi_remove_handle(self, easy);
+
+      if (result != 0) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+      else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
+        rb_funcall(easy, rb_intern("success"), 0);
+      }
+      else if (response_code >= 300 && response_code < 600) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+    }
+  }
+}
+
+/* called by multi_perform and fire_and_forget */
+static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
+  CURLMcode mcode;
+
+  do {
+    mcode = curl_multi_perform(multi_handle, still_running);
+  } while (mcode == CURLM_CALL_MULTI_PERFORM);
+
+  if (mcode != CURLM_OK) {
+    rb_raise((VALUE)mcode, "an error occured while running perform");
+  }
+
+  multi_read_info( self, multi_handle );
+}
+
+static VALUE fire_and_forget(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+}
+
+static VALUE multi_perform(VALUE self) {
+  CURLMcode mcode;
+  CurlMulti *curl_multi;
+  int maxfd, rc;
+  fd_set fdread, fdwrite, fdexcep;
+
+  long timeout;
+  struct timeval tv = {0, 0};
+
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+  while(curl_multi->running) {
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
+
+    /* get the curl suggested time out */
+    mcode = curl_multi_timeout(curl_multi->multi, &timeout);
+    if (mcode != CURLM_OK) {
+      rb_raise((VALUE)mcode, "an error occured getting the timeout");
+          }
+
+    if (timeout == 0) { /* no delay */
+      rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+      continue;
+    }
+    else if (timeout < 0) {
+      timeout = 1;
+    }
+
+    tv.tv_sec = timeout / 1000;
+    tv.tv_usec = (timeout * 1000) % 1000000;
+
+    /* load the fd sets from the multi handle */
+    mcode = curl_multi_fdset(curl_multi->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
+    if (mcode != CURLM_OK) {
+      rb_raise((VALUE)mcode, "an error occured getting the fdset");
+    }
+
+    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
+    if (rc < 0) {
+      rb_raise(rb_eRuntimeError, "error on thread select");
+    }
+    rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+
+  }
+
+  return Qnil;
+}
+
+static VALUE active_handle_count(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  return INT2NUM(curl_multi->active);
+}
+
+static VALUE multi_cleanup(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi_cleanup(curl_multi->multi);
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  return Qnil;
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CurlMulti *curl_multi = ALLOC(CurlMulti);
+  curl_multi->multi = curl_multi_init();
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  VALUE multi = Data_Wrap_Struct(cTyphoeusMulti, 0, dealloc, curl_multi);
+
+  rb_obj_call_init(multi, argc, argv);
+
+  return multi;
+}
+
+void init_typhoeus_multi() {
+  VALUE klass = cTyphoeusMulti = rb_define_class_under(mTyphoeus, "Multi", rb_cObject);
+
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_private_method(klass, "multi_add_handle", multi_add_handle, 1);
+  rb_define_private_method(klass, "multi_remove_handle", multi_remove_handle, 1);
+  rb_define_private_method(klass, "multi_perform", multi_perform, 0);
+  rb_define_private_method(klass, "multi_cleanup", multi_cleanup, 0);
+  rb_define_private_method(klass, "active_handle_count", active_handle_count, 0);
+	rb_define_method(klass, "fire_and_forget", fire_and_forget, 0);
+}

data/ext/typhoeus/typhoeus_multi.h

@@ -0,0 +1,16 @@
+#ifndef TYPHOEUS_MULTI
+#define TYPHOEUS_MULTI
+
+#include <native.h>
+#include <typhoeus_easy.h>
+
+VALUE cTyphoeusMulti;
+typedef struct {
+  int running;
+  int active;
+  CURLM *multi;
+} CurlMulti;
+
+void init_typhoeus_multi();
+
+#endif

data/lib/typhoeus.rb

@@ -0,0 +1,55 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+require 'rack/utils'
+require 'digest/sha2'
+require 'typhoeus/easy'
+require 'typhoeus/multi'
+require 'typhoeus/native'
+require 'typhoeus/filter'
+require 'typhoeus/remote_method'
+require 'typhoeus/remote'
+require 'typhoeus/remote_proxy_object'
+require 'typhoeus/response'
+require 'typhoeus/request'
+require 'typhoeus/hydra'
+
+module Typhoeus
+  VERSION = "0.1.14"
+
+  def self.easy_object_pool
+    @easy_objects ||= []
+  end
+
+  def self.init_easy_object_pool
+    20.times do
+      easy_object_pool << Typhoeus::Easy.new
+    end
+  end
+
+  def self.release_easy_object(easy)
+    easy.reset
+    easy_object_pool << easy
+  end
+
+  def self.get_easy_object
+    if easy_object_pool.empty?
+      Typhoeus::Easy.new
+    else
+      easy_object_pool.pop
+    end
+  end
+
+  def self.add_easy_request(easy_object)
+    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
+    Thread.current[:curl_multi].add(easy_object)
+  end
+
+  def self.perform_easy_requests
+    multi = Thread.current[:curl_multi]
+    start_time = Time.now
+    multi.easy_handles.each do |easy|
+      easy.start_time = start_time
+    end
+    multi.perform
+  end
+end

data/lib/typhoeus/easy.rb

@@ -0,0 +1,240 @@
+module Typhoeus
+  class Easy
+    attr_reader :response_body, :response_header, :method, :headers, :url
+    attr_accessor :start_time
+
+    CURLINFO_STRING = 1048576
+    OPTION_VALUES = {
+      :CURLOPT_URL            => 10002,
+      :CURLOPT_HTTPGET        => 80,
+      :CURLOPT_HTTPPOST       => 10024,
+      :CURLOPT_UPLOAD         => 46,
+      :CURLOPT_CUSTOMREQUEST  => 10036,
+      :CURLOPT_POSTFIELDS     => 10015,
+      :CURLOPT_POSTFIELDSIZE  => 60,
+      :CURLOPT_USERAGENT      => 10018,
+      :CURLOPT_TIMEOUT_MS     => 155,
+      :CURLOPT_NOSIGNAL       => 99,
+      :CURLOPT_HTTPHEADER     => 10023,
+      :CURLOPT_FOLLOWLOCATION => 52,
+      :CURLOPT_MAXREDIRS      => 68,
+      :CURLOPT_HTTPAUTH       => 107,
+      :CURLOPT_USERPWD        => 10000 + 5,
+      :CURLOPT_VERBOSE        => 41
+    }
+    INFO_VALUES = {
+      :CURLINFO_RESPONSE_CODE => 2097154,
+      :CURLINFO_TOTAL_TIME    => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23
+    }
+    AUTH_TYPES = {
+      :CURLAUTH_BASIC         => 1,
+      :CURLAUTH_DIGEST        => 2,
+      :CURLAUTH_GSSNEGOTIATE  => 4,
+      :CURLAUTH_NTLM          => 8,
+      :CURLAUTH_DIGEST_IE     => 16
+    }
+    
+    def initialize
+      @method = :get
+      @post_dat_set = nil
+      @headers = {}
+    end
+    
+    def headers=(hash)
+      @headers = hash
+    end
+    
+    def auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_USERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_HTTPAUTH], authinfo[:method]) if authinfo[:method]
+    end
+    
+    def auth_methods
+      get_info_long(INFO_VALUES[:CURLINFO_HTTPAUTH_AVAIL])
+    end
+    
+    def verbose=(boolean)
+      set_option(OPTION_VALUES[:CURLOPT_VERBOSE], !!boolean ? 1 : 0)
+    end
+    
+    def total_time_taken
+      get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
+    end
+    
+    def response_code
+      get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
+    end
+    
+    def follow_location=(boolean)
+      if boolean
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
+      end
+    end
+
+    def max_redirects=(redirects)
+      set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
+    end
+    
+    def timeout=(milliseconds)
+      @timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
+    end
+    
+    def timed_out?
+      @timeout && total_time_taken > @timeout && response_code == 0
+    end
+    
+    def request_body=(request_body)
+      @request_body = request_body
+      if @method == :put
+        easy_set_request_body(@request_body)
+        headers["Transfer-Encoding"] = ""
+        headers["Expect"] = ""
+      else
+        self.post_data = request_body
+      end
+    end
+    
+    def user_agent=(user_agent)
+      set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
+    end
+          
+    def url=(url)
+      @url = url
+      set_option(OPTION_VALUES[:CURLOPT_URL], url)
+    end
+    
+    def method=(method)
+      @method = method
+      if method == :get
+        set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
+      elsif method == :post
+        set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
+        self.post_data = ""
+      elsif method == :put
+        set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
+        self.request_body = "" unless @request_body
+      else
+        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
+      end
+    end
+    
+    def post_data=(data)
+      @post_data_set = true
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
+    end
+    
+    def params=(params)
+      params_string = params.keys.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
+        elsif value.is_a? Array
+          key = Rack::Utils.escape(k.to_s)
+          value.collect { |v| "#{key}=#{Rack::Utils.escape(v.to_s)}" }.join('&')
+        else
+          "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+      
+      if method == :post
+        self.post_data = params_string
+      else
+        self.url = "#{url}?#{params_string}"
+      end
+    end
+    
+    def set_option(option, value)
+      if value.class == String
+        easy_setopt_string(option, value)
+      else
+        easy_setopt_long(option, value)
+      end
+    end
+    
+    def perform
+      set_headers()
+      easy_perform()
+      response_code()
+    end
+    
+    def set_headers
+      headers.each_pair do |key, value|
+        easy_add_header("#{key}: #{value}")
+      end
+      easy_set_headers() unless headers.empty?
+    end
+    
+    # gets called when finished and response code is 200-299
+    def success
+      @success.call(self) if @success
+    end
+    
+    def on_success(&block)
+      @success = block
+    end
+    
+    def on_success=(block)
+      @success = block
+    end
+    
+    # gets called when finished and response code is 300-599
+    def failure
+      @failure.call(self) if @failure
+    end
+    
+    def on_failure(&block)
+      @failure = block
+    end
+    
+    def on_failure=(block)
+      @failure = block
+    end
+
+    def retries
+      @retries ||= 0
+    end
+    
+    def increment_retries
+      @retries ||= 0
+      @retries += 1
+    end
+    
+    def max_retries
+      @max_retries ||= 40
+    end
+    
+    def max_retries?
+      retries >= max_retries
+    end
+    
+    def reset
+      @retries = 0
+      @response_code = 0
+      @response_header = ""
+      @response_body = ""
+      easy_reset()
+    end
+    
+    def get_info_string(option)
+      easy_getinfo_string(option)
+    end
+    
+    def get_info_long(option)
+      easy_getinfo_long(option)
+    end
+    
+    def get_info_double(option)
+      easy_getinfo_double(option)
+    end
+    
+    def curl_version
+      version
+    end
+  end
+end