fblee-typhoeus 0.1.31

data/ext/typhoeus/extconf.rb

@@ -0,0 +1,65 @@
+ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
+
+# :stopdoc:
+
+require 'mkmf'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+specified_curl = ARGV[0] =~ /^--with-curl/ ? ARGV[0].split("=")[1] : nil
+LIBDIR = specified_curl ? "#{specified_curl}/lib": Config::CONFIG['libdir']
+INCLUDEDIR = specified_curl ? "#{specified_curl}/include" : Config::CONFIG['includedir']
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'macruby'
+  $LIBRUBYARG_STATIC.gsub!(/-static/, '')
+end
+
+$CFLAGS << " #{ENV["CFLAGS"]}"
+if Config::CONFIG['target_os'] == 'mingw32'
+  $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF"
+elsif Config::CONFIG['target_os'] == 'solaris2'
+  $CFLAGS << " -DUSE_INCLUDED_VASPRINTF"
+else
+  $CFLAGS << " -g -DXP_UNIX"
+end
+
+use_macports = !(defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby')
+$LIBPATH << "/opt/local/lib" if use_macports
+
+$CFLAGS << " -O3 -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  header = File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'include')
+  unless find_header('curl/curl.h', header)
+    abort "need libcurl"
+  end
+else
+  HEADER_DIRS = [
+    File.join(INCLUDEDIR, "curl"),
+    INCLUDEDIR,
+    '/usr/include/curl',
+    '/usr/local/include/curl'
+  ]
+
+  [
+    '/opt/local/include/curl',
+    '/opt/local/include',
+  ].each { |x| HEADER_DIRS.unshift(x) } if use_macports
+
+  unless find_header('curl/curl.h', *HEADER_DIRS)
+    abort "need libcurl"
+  end
+end
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  find_library('curl', 'curl_easy_init',
+               File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'bin'))
+else
+  find_library('curl', 'curl_easy_init',
+               LIBDIR,
+               '/opt/local/lib',
+               '/usr/local/lib',
+               '/usr/lib'
+    )
+end
+
+create_makefile("typhoeus/native")

data/ext/typhoeus/native.c

@@ -0,0 +1,11 @@
+#include <native.h>
+
+VALUE mTyphoeus;
+
+void Init_native()
+{
+  mTyphoeus = rb_const_get(rb_cObject, rb_intern("Typhoeus"));
+
+  init_typhoeus_easy();
+  init_typhoeus_multi();
+}

data/ext/typhoeus/native.h

@@ -0,0 +1,21 @@
+#ifndef TYPHOEUS_NATIVE
+#define TYPHOEUS_NATIVE
+
+#include <ruby.h>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <curl/multi.h>
+
+void Init_native();
+extern VALUE mTyphoeus;
+extern void init_typhoeus_easy();
+extern void init_typhoeus_multi();
+
+#endif
+
+#ifndef RSTRING_PTR
+
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+
+#endif

data/ext/typhoeus/typhoeus_easy.c

@@ -0,0 +1,207 @@
+#include <typhoeus_easy.h>
+
+static VALUE idAppend;
+VALUE cTyphoeusEasy;
+
+static void dealloc(CurlEasy *curl_easy) {
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+  }
+
+  curl_easy_cleanup(curl_easy->curl);
+
+  free(curl_easy);
+}
+
+static VALUE easy_setopt_string(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  long opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, StringValuePtr(parameter));
+  return opt_name;
+}
+
+static VALUE easy_setopt_long(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  long opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, NUM2LONG(parameter));
+  return opt_name;
+}
+
+static VALUE easy_getinfo_string(VALUE self, VALUE info) {
+  char *info_string;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  long opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_string);
+
+  return rb_str_new2(info_string);
+}
+
+static VALUE easy_getinfo_long(VALUE self, VALUE info) {
+  long info_long;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  long opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_long);
+
+  return LONG2NUM(info_long);
+}
+
+static VALUE easy_getinfo_double(VALUE self, VALUE info) {
+  double info_double = 0;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  long opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_double);
+
+  return rb_float_new(info_double);
+}
+
+static VALUE easy_perform(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+  curl_easy_perform(curl_easy->curl);
+
+  return Qnil;
+}
+
+static size_t write_data_handler(char *stream, size_t size, size_t nmemb, VALUE val) {
+  long stream_size = (long)(size * nmemb);
+  rb_funcall(val, idAppend, 1, rb_str_new(stream, stream_size));
+  return size * nmemb;
+}
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *data) {
+  size_t realsize = size * nmemb;
+  RequestChunk *mem = (RequestChunk *)data;
+
+  if (realsize > mem->size - mem->read) {
+    realsize = mem->size - mem->read;
+  }
+
+  if (realsize != 0) {
+    memcpy(ptr, &(mem->memory[mem->read]), realsize);
+    mem->read += realsize;
+  }
+
+  return realsize;
+}
+
+static void set_response_handlers(VALUE easy, CURL *curl) {
+  rb_iv_set(easy, "@response_body", rb_str_new2(""));
+  rb_iv_set(easy, "@response_header", rb_str_new2(""));
+
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, rb_iv_get(easy, "@response_body"));
+  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_HEADERDATA, rb_iv_get(easy, "@response_header"));
+}
+
+static VALUE easy_reset(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+    curl_easy->request_chunk = NULL;
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+    curl_easy->headers = NULL;
+  }
+
+  curl_easy_reset(curl_easy->curl);
+
+  set_response_handlers(self, curl_easy->curl);
+
+  return Qnil;
+}
+
+static VALUE easy_add_header(VALUE self, VALUE header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->headers = curl_slist_append(curl_easy->headers, RSTRING_PTR(header));
+  return header;
+}
+
+static VALUE easy_set_headers(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_HTTPHEADER, curl_easy->headers);
+
+  return Qnil;
+}
+
+static VALUE easy_set_request_body(VALUE self, VALUE data, VALUE content_length_header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->request_chunk = ALLOC(RequestChunk);
+  curl_easy->request_chunk->size = RSTRING_LEN(data);
+  curl_easy->request_chunk->memory = StringValuePtr(data);
+  curl_easy->request_chunk->read = 0;
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READFUNCTION, (curl_read_callback)read_callback);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READDATA, curl_easy->request_chunk);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_INFILESIZE, RSTRING_LEN(data));
+
+  return Qnil;
+}
+
+static VALUE easy_escape(VALUE self, VALUE data, VALUE length) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  return rb_str_new2(curl_easy_escape(curl_easy->curl, StringValuePtr(data), NUM2INT(length)));
+}
+
+static VALUE version(VALUE self) {
+  return rb_str_new2(curl_version());
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CURL *curl = curl_easy_init();
+  CurlEasy *curl_easy = ALLOC(CurlEasy);
+  curl_easy->curl = curl;
+  curl_easy->headers = NULL;
+  curl_easy->request_chunk = NULL;
+  VALUE easy = Data_Wrap_Struct(cTyphoeusEasy, 0, dealloc, curl_easy);
+
+  set_response_handlers(easy, curl);
+
+  rb_obj_call_init(easy, argc, argv);
+
+  return easy;
+}
+
+void init_typhoeus_easy() {
+  VALUE klass = cTyphoeusEasy = rb_define_class_under(mTyphoeus, "Easy", rb_cObject);
+  idAppend = rb_intern("<<");
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_private_method(klass, "easy_setopt_string", easy_setopt_string, 2);
+  rb_define_private_method(klass, "easy_setopt_long", easy_setopt_long, 2);
+  rb_define_private_method(klass, "easy_getinfo_string", easy_getinfo_string, 1);
+  rb_define_private_method(klass, "easy_getinfo_long", easy_getinfo_long, 1);
+  rb_define_private_method(klass, "easy_getinfo_double", easy_getinfo_double, 1);
+  rb_define_private_method(klass, "easy_perform", easy_perform, 0);
+  rb_define_private_method(klass, "easy_reset", easy_reset, 0);
+  rb_define_private_method(klass, "easy_set_request_body", easy_set_request_body, 1);
+  rb_define_private_method(klass, "easy_set_headers", easy_set_headers, 0);
+  rb_define_private_method(klass, "easy_add_header", easy_add_header, 1);
+  rb_define_private_method(klass, "easy_escape", easy_escape, 2);
+  rb_define_private_method(klass, "version", version, 0);
+}

data/ext/typhoeus/typhoeus_easy.h

@@ -0,0 +1,19 @@
+#ifndef TYPHOEUS_EASY
+#define TYPHOEUS_EASY
+
+#include <native.h>
+
+void init_typhoeus_easy();
+typedef struct {
+  const char *memory;
+  int size;
+  int read;
+} RequestChunk;
+
+typedef struct {
+  RequestChunk *request_chunk;
+  CURL *curl;
+  struct curl_slist *headers;
+} CurlEasy;
+
+#endif

data/ext/typhoeus/typhoeus_multi.c

@@ -0,0 +1,225 @@
+#include <typhoeus_multi.h>
+
+static void multi_read_info(VALUE self, CURLM *multi_handle);
+
+static void dealloc(CurlMulti *curl_multi) {
+  curl_multi_cleanup(curl_multi->multi);
+  free(curl_multi);
+}
+
+static VALUE multi_add_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  CURLMcode mcode;
+
+  mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
+  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
+    rb_raise((VALUE)mcode, "An error occured adding the handle");
+  }
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
+  curl_multi->active++;
+
+  if (mcode == CURLM_CALL_MULTI_PERFORM) {
+    curl_multi_perform(curl_multi->multi, &(curl_multi->running));
+  }
+  // 
+  // if (curl_multi->running) {
+  //     printf("call read_info on add<br/>");
+  //   multi_read_info(self, curl_multi->multi);
+  // }
+
+  return easy;
+}
+
+static VALUE multi_remove_handle(VALUE self, VALUE easy) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(easy, CurlEasy, curl_easy);
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi->active--;
+  curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);
+
+  return easy;
+}
+
+static void multi_read_info(VALUE self, CURLM *multi_handle) {
+  int msgs_left, result;
+  CURLMsg *msg;
+  CURLcode ecode;
+  CURL *easy_handle;
+  VALUE easy;
+
+  /* check for finished easy handles and remove from the multi handle */
+  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
+
+    if (msg->msg != CURLMSG_DONE) {
+      continue;
+    }
+
+    easy_handle = msg->easy_handle;
+    result = msg->data.result;
+    if (easy_handle) {
+      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
+      if (ecode != 0) {
+        rb_raise(ecode, "error getting easy object");
+      }
+
+      long response_code = -1;
+      curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
+
+      // TODO: find out what the real problem is here and fix it.
+      // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
+      // fail intermittently and return this result number. However, it will succeed if you try it a few
+      // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!
+
+      // Sandofsky says:
+      // This is caused by OS X first attempting to resolve using IPV6.
+      // Hack solution: connect to yourself with 127.0.0.1, not localhost
+      // http://curl.haxx.se/mail/tracker-2009-09/0018.html
+      if (result == 7) {
+        VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
+        if (max_retries != Qtrue) {
+          multi_remove_handle(self, easy);
+          multi_add_handle(self, easy);
+          CurlMulti *curl_multi;
+          Data_Get_Struct(self, CurlMulti, curl_multi);
+          curl_multi_perform(curl_multi->multi, &(curl_multi->running));
+
+          rb_funcall(easy, rb_intern("increment_retries"), 0);
+
+          continue;
+        }
+      }
+      multi_remove_handle(self, easy);
+
+      if (result != 0) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+      else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
+        rb_funcall(easy, rb_intern("success"), 0);
+      }
+      else if (response_code >= 300 && response_code < 600) {
+        rb_funcall(easy, rb_intern("failure"), 0);
+      }
+    }
+  }
+}
+
+/* called by multi_perform and fire_and_forget */
+static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
+  CURLMcode mcode;
+
+  do {
+    mcode = curl_multi_perform(multi_handle, still_running);
+  } while (mcode == CURLM_CALL_MULTI_PERFORM);
+
+  if (mcode != CURLM_OK) {
+    rb_raise((VALUE)mcode, "an error occured while running perform");
+  }
+
+  multi_read_info( self, multi_handle );
+}
+
+static VALUE fire_and_forget(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+}
+
+static VALUE multi_perform(VALUE self) {
+  CURLMcode mcode;
+  CurlMulti *curl_multi;
+  int maxfd, rc;
+  fd_set fdread, fdwrite, fdexcep;
+
+  long timeout;
+  struct timeval tv = {0, 0};
+
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+  while(curl_multi->running) {
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
+
+    /* get the curl suggested time out */
+    mcode = curl_multi_timeout(curl_multi->multi, &timeout);
+    if (mcode != CURLM_OK) {
+      rb_raise((VALUE)mcode, "an error occured getting the timeout");
+          }
+
+    if (timeout == 0) { /* no delay */
+      rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+      continue;
+    }
+    else if (timeout < 0) {
+      timeout = 1;
+    }
+
+    tv.tv_sec = timeout / 1000;
+    tv.tv_usec = (timeout * 1000) % 1000000;
+
+    /* load the fd sets from the multi handle */
+    mcode = curl_multi_fdset(curl_multi->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
+    if (mcode != CURLM_OK) {
+      rb_raise((VALUE)mcode, "an error occured getting the fdset");
+    }
+
+    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
+    if (rc < 0) {
+      rb_raise(rb_eRuntimeError, "error on thread select");
+    }
+    rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+
+  }
+
+  return Qnil;
+}
+
+static VALUE active_handle_count(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  return INT2NUM(curl_multi->active);
+}
+
+static VALUE multi_cleanup(VALUE self) {
+  CurlMulti *curl_multi;
+  Data_Get_Struct(self, CurlMulti, curl_multi);
+
+  curl_multi_cleanup(curl_multi->multi);
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  return Qnil;
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CurlMulti *curl_multi = ALLOC(CurlMulti);
+  curl_multi->multi = curl_multi_init();
+  curl_multi->active = 0;
+  curl_multi->running = 0;
+
+  VALUE multi = Data_Wrap_Struct(cTyphoeusMulti, 0, dealloc, curl_multi);
+
+  rb_obj_call_init(multi, argc, argv);
+
+  return multi;
+}
+
+void init_typhoeus_multi() {
+  VALUE klass = cTyphoeusMulti = rb_define_class_under(mTyphoeus, "Multi", rb_cObject);
+
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_private_method(klass, "multi_add_handle", multi_add_handle, 1);
+  rb_define_private_method(klass, "multi_remove_handle", multi_remove_handle, 1);
+  rb_define_private_method(klass, "multi_perform", multi_perform, 0);
+  rb_define_private_method(klass, "multi_cleanup", multi_cleanup, 0);
+  rb_define_private_method(klass, "active_handle_count", active_handle_count, 0);
+	rb_define_method(klass, "fire_and_forget", fire_and_forget, 0);
+}

data/ext/typhoeus/typhoeus_multi.h

@@ -0,0 +1,16 @@
+#ifndef TYPHOEUS_MULTI
+#define TYPHOEUS_MULTI
+
+#include <native.h>
+#include <typhoeus_easy.h>
+
+VALUE cTyphoeusMulti;
+typedef struct {
+  int running;
+  int active;
+  CURLM *multi;
+} CurlMulti;
+
+void init_typhoeus_multi();
+
+#endif

data/lib/typhoeus/.gitignore

@@ -0,0 +1 @@
+native.bundle