arachni-typhoeus 0.2.0

data/ext/typhoeus/typhoeus_easy.h

@@ -0,0 +1,19 @@
+#ifndef TYPHOEUS_EASY
+#define TYPHOEUS_EASY
+
+#include <native.h>
+
+void init_typhoeus_easy();
+typedef struct {
+  const char *memory;
+  int size;
+  int read;
+} RequestChunk;
+
+typedef struct {
+  RequestChunk *request_chunk;
+  CURL *curl;
+  struct curl_slist *headers;
+} CurlEasy;
+
+#endif

data/ext/typhoeus/typhoeus_multi.c

@@ -0,0 +1,211 @@
+#include <typhoeus_multi.h>
+
+static void multi_read_info(VALUE self, CURLM *multi_handle);
+
+static void dealloc(CurlMulti *curl_multi) {
+  curl_multi_cleanup(curl_multi->multi);
+  free(curl_multi);
+}
+
+static VALUE multi_add_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  CURLMcode mcode;
+
+  mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
+  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
+    rb_raise(rb_eRuntimeError, "An error occured adding the handle: %d: %s", mcode, curl_multi_strerror(mcode));
+  }
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
+  curl_multi->active++;
+
+  VALUE easy_handles = rb_iv_get(self, "@easy_handles");
+  rb_ary_push(easy_handles, easy);
+
+  if (mcode == CURLM_CALL_MULTI_PERFORM) {
+    curl_multi_perform(curl_multi->multi, &(curl_multi->running));
+  }
+  // 
+  // if (curl_multi->running) {
+  //     printf("call read_info on add<br/>");
+  //   multi_read_info(self, curl_multi->multi);
+  // }
+
+  return easy;
+}
+
+static VALUE multi_remove_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi->active--;
+  curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);
+
+  VALUE easy_handles = rb_iv_get(self, "@easy_handles");
+  rb_ary_delete(easy_handles, easy);
+
+  return easy;
+}
+
+static void multi_read_info(VALUE self, CURLM *multi_handle) {
+  int msgs_left, result;
+  CURLMsg *msg;
+  CURLcode ecode;
+  CURL *easy_handle;
+  VALUE easy;
+
+  /* check for finished easy handles and remove from the multi handle */
+  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
+
+    if (msg->msg != CURLMSG_DONE) {
+      continue;
+    }
+
+    easy_handle = msg->easy_handle;
+    result = msg->data.result;
+    if (easy_handle) {
+      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
+      if (ecode != 0) {
+        rb_raise(rb_eRuntimeError, "error getting easy object: %d: %s", ecode, curl_easy_strerror(ecode));
+      }
+
+      long response_code = -1;
+      curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
+
+      multi_remove_handle(self, easy);
+      rb_iv_set(easy, "@curl_return_code", INT2FIX(result));
+
+      if (result != 0) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+      else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
+        rb_funcall(easy, rb_intern("success"), 0);
+      }
+      else if (response_code >= 300 && response_code < 600) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+    }
+  }
+}
+
+/* called by multi_perform and fire_and_forget */
+static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
+  CURLMcode mcode;
+
+  do {
+    mcode = curl_multi_perform(multi_handle, still_running);
+  } while (mcode == CURLM_CALL_MULTI_PERFORM);
+
+  if (mcode != CURLM_OK) {
+    rb_raise(rb_eRuntimeError, "an error occured while running perform: %d: %s", mcode, curl_multi_strerror(mcode));
+  }
+
+  multi_read_info( self, multi_handle );
+}
+
+static VALUE fire_and_forget(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+
+  return Qnil;
+}
+
+static VALUE multi_perform(VALUE self) {
+  CURLMcode mcode;
+  CurlMulti *curl_multi;
+  int maxfd, rc;
+  fd_set fdread, fdwrite, fdexcep;
+
+  long timeout;
+  struct timeval tv = {0, 0};
+
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+  while(curl_multi->running) {
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
+
+    /* get the curl suggested time out */
+    mcode = curl_multi_timeout(curl_multi->multi, &timeout);
+    if (mcode != CURLM_OK) {
+      rb_raise(rb_eRuntimeError, "an error occured getting the timeout: %d: %s", mcode, curl_multi_strerror(mcode));
+          }
+
+    if (timeout == 0) { /* no delay */
+      rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+      continue;
+    }
+    else if (timeout < 0) {
+      timeout = 1;
+    }
+
+    tv.tv_sec = timeout / 1000;
+    tv.tv_usec = (timeout * 1000) % 1000000;
+
+    /* load the fd sets from the multi handle */
+    mcode = curl_multi_fdset(curl_multi->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
+    if (mcode != CURLM_OK) {
+      rb_raise(rb_eRuntimeError, "an error occured getting the fdset: %d: %s", mcode, curl_multi_strerror(mcode));
+    }
+
+    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
+    if (rc < 0) {
+      rb_raise(rb_eRuntimeError, "error on thread select");
+    }
+    rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+
+  }
+
+  return Qnil;
+}
+
+static VALUE active_handle_count(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  return INT2FIX(curl_multi->active);
+}
+
+static VALUE multi_cleanup(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi_cleanup(curl_multi->multi);
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  return Qnil;
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CurlMulti *curl_multi = ALLOC(CurlMulti);
+  curl_multi->multi = curl_multi_init();
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  VALUE multi = Data_Wrap_Struct(cTyphoeusMulti, 0, dealloc, curl_multi);
+
+  rb_obj_call_init(multi, argc, argv);
+
+  return multi;
+}
+
+void init_typhoeus_multi() {
+  VALUE klass = cTyphoeusMulti = rb_define_class_under(mTyphoeus, "Multi", rb_cObject);
+
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_private_method(klass, "multi_add_handle", multi_add_handle, 1);
+  rb_define_private_method(klass, "multi_remove_handle", multi_remove_handle, 1);
+  rb_define_private_method(klass, "multi_perform", multi_perform, 0);
+  rb_define_private_method(klass, "multi_cleanup", multi_cleanup, 0);
+  rb_define_private_method(klass, "active_handle_count", active_handle_count, 0);
+	rb_define_method(klass, "fire_and_forget", fire_and_forget, 0);
+}

data/ext/typhoeus/typhoeus_multi.h

@@ -0,0 +1,16 @@
+#ifndef TYPHOEUS_MULTI
+#define TYPHOEUS_MULTI
+
+#include <native.h>
+#include <typhoeus_easy.h>
+
+VALUE cTyphoeusMulti;
+typedef struct {
+  int running;
+  int active;
+  CURLM *multi;
+} CurlMulti;
+
+void init_typhoeus_multi();
+
+#endif

data/lib/typhoeus.rb

@@ -0,0 +1,58 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__) + "/../ext")
+
+require 'digest/sha2'
+require 'typhoeus/utils'
+require 'typhoeus/normalized_header_hash'
+require 'typhoeus/easy'
+require 'typhoeus/multi'
+require 'typhoeus/native'
+require 'typhoeus/filter'
+require 'typhoeus/remote_method'
+require 'typhoeus/remote'
+require 'typhoeus/remote_proxy_object'
+require 'typhoeus/response'
+require 'typhoeus/request'
+require 'typhoeus/hydra'
+require 'typhoeus/hydra_mock'
+
+module Typhoeus
+  VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").chomp
+
+  def self.easy_object_pool
+    @easy_objects ||= []
+  end
+
+  def self.init_easy_object_pool
+    20.times do
+      easy_object_pool << Typhoeus::Easy.new
+    end
+  end
+
+  def self.release_easy_object(easy)
+    easy.reset
+    easy_object_pool << easy
+  end
+
+  def self.get_easy_object
+    if easy_object_pool.empty?
+      Typhoeus::Easy.new
+    else
+      easy_object_pool.pop
+    end
+  end
+
+  def self.add_easy_request(easy_object)
+    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
+    Thread.current[:curl_multi].add(easy_object)
+  end
+
+  def self.perform_easy_requests
+    multi = Thread.current[:curl_multi]
+    start_time = Time.now
+    multi.easy_handles.each do |easy|
+      easy.start_time = start_time
+    end
+    multi.perform
+  end
+end

data/lib/typhoeus/.gitignore

@@ -0,0 +1 @@
+native.bundle

data/lib/typhoeus/easy.rb

@@ -0,0 +1,366 @@
+module Typhoeus
+  class Easy
+    attr_reader :response_body, :response_header, :method, :headers, :url, :params, :curl_return_code
+    attr_accessor :start_time
+
+    # These integer codes are available in curl/curl.h
+    CURLINFO_STRING = 1048576
+    OPTION_VALUES = {
+      :CURLOPT_URL            => 10002,
+      :CURLOPT_HTTPGET        => 80,
+      :CURLOPT_HTTPPOST       => 10024,
+      :CURLOPT_UPLOAD         => 46,
+      :CURLOPT_CUSTOMREQUEST  => 10036,
+      :CURLOPT_POSTFIELDS     => 10015,
+      :CURLOPT_COPYPOSTFIELDS     => 10165,
+      :CURLOPT_POSTFIELDSIZE  => 60,
+      :CURLOPT_USERAGENT      => 10018,
+      :CURLOPT_TIMEOUT_MS     => 155,
+      # Time-out connect operations after this amount of milliseconds.
+      # [Only works on unix-style/SIGALRM operating systems. IOW, does
+      # not work on Windows.
+      :CURLOPT_CONNECTTIMEOUT_MS  => 156,
+      :CURLOPT_NOSIGNAL       => 99,
+      :CURLOPT_HTTPHEADER     => 10023,
+      :CURLOPT_FOLLOWLOCATION => 52,
+      :CURLOPT_MAXREDIRS      => 68,
+      :CURLOPT_HTTPAUTH       => 107,
+      :CURLOPT_USERPWD        => 10000 + 5,
+      :CURLOPT_VERBOSE        => 41,
+      :CURLOPT_PROXY          => 10004,
+      :CURLOPT_PROXYUSERPWD   => 10000 + 6,
+      :CURLOPT_PROXYTYPE      => 101,
+      :CURLOPT_PROXYAUTH      => 111,
+      :CURLOPT_VERIFYPEER     => 64,
+      :CURLOPT_NOBODY         => 44,
+      :CURLOPT_ENCODING       => 10000 + 102,
+      :CURLOPT_SSLCERT        => 10025,
+      :CURLOPT_SSLCERTTYPE    => 10086,
+      :CURLOPT_SSLKEY         => 10087,
+      :CURLOPT_SSLKEYTYPE     => 10088,
+      :CURLOPT_KEYPASSWD      => 10026,
+      :CURLOPT_CAINFO         => 10065,
+      :CURLOPT_CAPATH         => 10097
+    }
+    INFO_VALUES = {
+      :CURLINFO_RESPONSE_CODE      => 2097154,
+      :CURLINFO_TOTAL_TIME         => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL     => 0x200000 + 23,
+      :CURLINFO_EFFECTIVE_URL      => 0x100000 + 1,
+      :CURLINFO_NAMELOOKUP_TIME    => 0x300000 + 4,
+      :CURLINFO_CONNECT_TIME       => 0x300000 + 5,
+      :CURLINFO_PRETRANSFER_TIME   => 0x300000 + 6,
+      :CURLINFO_STARTTRANSFER_TIME => 0x300000 + 17,
+      :CURLINFO_APPCONNECT_TIME    => 0x300000 + 33,
+
+    }
+    AUTH_TYPES = {
+      :CURLAUTH_BASIC         => 1,
+      :CURLAUTH_DIGEST        => 2,
+      :CURLAUTH_GSSNEGOTIATE  => 4,
+      :CURLAUTH_NTLM          => 8,
+      :CURLAUTH_DIGEST_IE     => 16,
+      :CURLAUTH_AUTO          => 16 | 8 | 4 | 2 | 1
+    }
+    PROXY_TYPES = {
+      :CURLPROXY_HTTP         => 0,
+      :CURLPROXY_HTTP_1_0     => 1,
+      :CURLPROXY_SOCKS4       => 4,
+      :CURLPROXY_SOCKS5       => 5,
+      :CURLPROXY_SOCKS4A      => 6,
+    }
+
+
+    def initialize
+      @method = :get
+      @headers = {}
+
+      # Enable encoding/compression support
+      set_option(OPTION_VALUES[:CURLOPT_ENCODING], '')
+    end
+
+    def headers=(hash)
+      @headers = hash
+    end
+
+    def proxy=(proxy)
+      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy[:server])
+      set_option(OPTION_VALUES[:CURLOPT_PROXYTYPE], proxy[:type]) if proxy[:type]
+    end
+
+    def proxy_auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_PROXYUSERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_PROXYAUTH], authinfo[:method]) if authinfo[:method]
+    end
+
+    def auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_USERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_HTTPAUTH], authinfo[:method]) if authinfo[:method]
+    end
+
+    def auth_methods
+      get_info_long(INFO_VALUES[:CURLINFO_HTTPAUTH_AVAIL])
+    end
+
+    def verbose=(boolean)
+      set_option(OPTION_VALUES[:CURLOPT_VERBOSE], !!boolean ? 1 : 0)
+    end
+
+    def total_time_taken
+      get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
+    end
+
+    def start_transfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_STARTTRANSFER_TIME])
+    end
+
+    def app_connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_APPCONNECT_TIME])
+    end
+
+    def pretransfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_PRETRANSFER_TIME])
+    end
+
+    def connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_CONNECT_TIME])
+    end
+
+    def name_lookup_time
+      get_info_double(INFO_VALUES[:CURLINFO_NAMELOOKUP_TIME])
+    end
+
+    def effective_url
+      get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
+    end
+
+    def response_code
+      get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
+    end
+
+    def follow_location=(boolean)
+      if boolean
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
+      end
+    end
+
+    def max_redirects=(redirects)
+      set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
+    end
+
+    def connect_timeout=(milliseconds)
+      @connect_timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_CONNECTTIMEOUT_MS], milliseconds)
+    end
+
+    def timeout=(milliseconds)
+      @timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
+    end
+
+    def timed_out?
+      curl_return_code == 28
+    end
+
+    def supports_zlib?
+      !!(curl_version.match(/zlib/))
+    end
+
+    def request_body=(request_body)
+      @request_body = request_body
+      if @method == :put
+        easy_set_request_body(@request_body)
+        headers["Transfer-Encoding"] = ""
+        headers["Expect"] = ""
+      else
+        self.post_data = request_body
+      end
+    end
+
+    def user_agent=(user_agent)
+      set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
+    end
+
+    def url=(url)
+      @url = url
+      set_option(OPTION_VALUES[:CURLOPT_URL], url)
+    end
+
+    def disable_ssl_peer_verification
+      set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
+    end
+
+    def method=(method)
+      @method = method
+      if method == :get
+        set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
+      elsif method == :post
+        set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
+        self.post_data = ""
+      elsif method == :put
+        set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
+        self.request_body = "" unless @request_body
+      elsif method == :head
+        set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], method.to_s.upcase)
+      end
+    end
+
+    def post_data=(data)
+      @post_data_set = true
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
+      set_option(OPTION_VALUES[:CURLOPT_COPYPOSTFIELDS], data)
+    end
+
+    def params=(params)
+      @params = params
+      params_string = params.keys.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.collect {|sk| Typhoeus::Utils.escape("#{k}[#{sk}]") + "=" + Typhoeus::Utils.escape(value[sk].to_s)}
+        elsif value.is_a? Array
+          key = Typhoeus::Utils.escape(k.to_s)
+          value.collect { |v| "#{key}=#{Typhoeus::Utils.escape(v.to_s)}" }.join('&')
+        else
+          "#{Typhoeus::Utils.escape(k.to_s)}=#{Typhoeus::Utils.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+
+      if method == :post
+        self.post_data = params_string
+      else
+        self.url = "#{url}?#{params_string}"
+      end
+    end
+
+    # Set SSL certificate
+    # " The string should be the file name of your certificate. "
+    # The default format is "PEM" and can be changed with ssl_cert_type=
+    def ssl_cert=(cert)
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERT], cert)
+    end
+
+    # Set SSL certificate type
+    # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
+    def ssl_cert_type=(cert_type)
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type)
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
+    end
+
+    # Set SSL Key file
+    # " The string should be the file name of your private key. "
+    # The default format is "PEM" and can be changed with ssl_key_type=
+    #
+    def ssl_key=(key)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEY], key)
+    end
+
+    # Set SSL Key type
+    # " The string should be the format of your private key. Supported formats are "PEM", "DER" and "ENG". "
+    #
+    def ssl_key_type=(key_type)
+      raise "Invalid ssl key type : '#{key_type}'..." if key_type and !%w(PEM DER ENG).include?(key_type)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEYTYPE], key_type)
+    end
+
+    def ssl_key_password=(key_password)
+      set_option(OPTION_VALUES[:CURLOPT_KEYPASSWD], key_password)
+    end
+
+    # Set SSL CACERT
+    # " File holding one or more certificates to verify the peer with. "
+    #
+    def ssl_cacert=(cacert)
+      set_option(OPTION_VALUES[:CURLOPT_CAINFO], cacert)
+    end
+
+    # Set CAPATH
+    # " directory holding multiple CA certificates to verify the peer with. The certificate directory must be prepared using the openssl c_rehash utility. "
+    #
+    def ssl_capath=(capath)
+      set_option(OPTION_VALUES[:CURLOPT_CAPATH], capath)
+    end
+
+    def set_option(option, value)
+      if value.class == String
+        easy_setopt_string(option, value)
+      elsif value
+        easy_setopt_long(option, value)
+      end
+    end
+
+    def perform
+      set_headers()
+      easy_perform()
+      resp_code = response_code()
+      if resp_code >= 200 && resp_code <= 299
+        success
+      else
+        failure
+      end
+      resp_code
+    end
+
+    def set_headers
+      headers.each_pair do |key, value|
+        easy_add_header("#{key}: #{value}")
+      end
+      easy_set_headers() unless headers.empty?
+    end
+
+    # gets called when finished and response code is 200-299
+    def success
+      @success.call(self) if @success
+    end
+
+    def on_success(&block)
+      @success = block
+    end
+
+    def on_success=(block)
+      @success = block
+    end
+
+    # gets called when finished and response code is 300-599 or curl returns an error code
+    def failure
+      @failure.call(self) if @failure
+    end
+
+    def on_failure(&block)
+      @failure = block
+    end
+
+    def on_failure=(block)
+      @failure = block
+    end
+
+    def reset
+      @response_code = 0
+      @response_header = ""
+      @response_body = ""
+      easy_reset()
+    end
+
+    def get_info_string(option)
+      easy_getinfo_string(option)
+    end
+
+    def get_info_long(option)
+      easy_getinfo_long(option)
+    end
+
+    def get_info_double(option)
+      easy_getinfo_double(option)
+    end
+
+    def curl_version
+      version
+    end
+
+  end
+end