xenda-typhoeus 0.2.4

data/Rakefile

@@ -0,0 +1,56 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require "spec"
+require "spec/rake/spectask"
+require "lib/typhoeus/version"
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "typhoeus"
+    gemspec.summary = "A library for interacting with web services (and building SOAs) at blinding speed."
+    gemspec.description = "Like a modern code version of the mythical beast with 100 serpent heads, Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic."
+    gemspec.email = "dbalatero@gmail.com"
+    gemspec.homepage = "http://github.com/dbalatero/typhoeus"
+    gemspec.authors = ["Paul Dix", "David Balatero"]
+    gemspec.add_dependency "mime-types"
+    gemspec.add_development_dependency "rspec"
+    gemspec.add_development_dependency "jeweler"
+    gemspec.add_development_dependency "diff-lcs"
+    gemspec.add_development_dependency "sinatra"
+    gemspec.add_development_dependency "json"
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+task :install do
+  rm_rf "*.gem"
+  puts `gem build typhoeus.gemspec`
+  puts `gem install typhoeus-#{Typhoeus::VERSION}.gem`
+end
+
+desc "Builds the native code"
+task :build_native do
+  system("cd ext/typhoeus && ruby extconf.rb && make")
+end
+
+desc "Start up the test servers"
+task :start_test_servers do
+  puts "Starting 3 test servers"
+  (3000..3002).map do |port|
+    Thread.new do
+      system("ruby spec/servers/app.rb -p #{port}")
+    end
+  end.each(&:join)
+end
+
+desc "Run all the tests"
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.2.4

data/benchmarks/profile.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'rubygems'
+require 'ruby-prof'
+
+calls = 20
+@klass = Class.new do
+  include Typhoeus
+end
+
+Typhoeus.init_easy_objects
+
+RubyProf.start
+
+responses = []
+calls.times do |i|
+  responses << @klass.get("http://127.0.0.1:3000/#{i}")
+end
+
+responses.each {|r| }#raise unless r.response_body == "whatever"}
+
+result = RubyProf.stop
+
+ # Print a flat profile to text
+ printer = RubyProf::FlatPrinter.new(result)
+ printer.print(STDOUT, 0)

data/benchmarks/vs_nethttp.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'open-uri'
+require 'benchmark'
+include Benchmark
+
+
+calls = 20
+@klass = Class.new do
+  include Typhoeus
+end
+
+Typhoeus.init_easy_object_pool
+
+benchmark do |t|    
+  t.report("net::http") do
+    responses = []
+    
+    calls.times do |i|
+      responses << open("http://127.0.0.1:3000/#{i}").read
+    end
+    
+    responses.each {|r| raise unless r == "whatever"}    
+  end
+  
+  t.report("typhoeus") do
+    responses = []
+    
+    calls.times do |i|
+      responses << @klass.get("http://127.0.0.1:3000/#{i}")
+    end
+    
+    responses.each {|r| raise unless r.body == "whatever"}
+  end
+end

data/examples/file.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+
+
+response = Typhoeus::Request.post(
+  "http://video-feed.local",
+  :params => {
+    :file => File.new("file.rb")
+  }
+)
+
+puts response.inspect

data/examples/times.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+
+hydra = Typhoeus::Hydra.new
+hydra.disable_memoization
+
+urls = [
+    'http://google.com',
+    'http://testphp.vulnweb.com',
+    'http://demo.testfire.net',
+    'http://example.net',
+]
+
+10.times {
+    |i|
+
+    req = Typhoeus::Request.new( urls[ i % urls.size] )
+    req.on_complete {
+        |res|
+        puts 'URL:     ' + res.effective_url
+        puts 'Time:    ' + res.time.to_s
+        puts 'Connect time: ' + res.connect_time.to_s
+        puts 'App connect time:    ' + res.app_connect_time.to_s
+        puts 'Start transfer time: ' + res.start_transfer_time.to_s
+        puts 'Pre transfer time:   ' + res.pretransfer_time.to_s
+        puts '-------------'
+    }
+
+    hydra.queue( req )
+    puts 'Queued: ' + req.url
+}
+
+puts
+puts 'Harvesting responses...'
+puts
+
+hydra.run
+
+puts
+puts 'Done.'
+puts

data/examples/twitter.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'json'
+
+class Twitter
+  include Typhoeus
+  remote_defaults :on_success => lambda {|response| JSON.parse(response.body)},
+                  :on_failure => lambda {|response| puts "error code: #{response.code}"},
+                  :base_uri   => "http://search.twitter.com"
+
+  define_remote_method :search, :path => '/search.json'
+  define_remote_method :trends, :path => '/trends/:time_frame.json'
+end
+
+tweets = Twitter.search(:params => {:q => "railsconf"})
+trends = Twitter.trends(:time_frame => :current)
+
+# and then the calls don't actually happen until the first time you
+# call a method on one of the objects returned from the remote_method
+
+puts tweets.keys # it's a hash from parsed JSON

data/ext/typhoeus/.gitignore

@@ -0,0 +1,7 @@
+conftest.dSYM/*
+conftest.dSYM
+mkmf.log
+*.o
+native.bundle
+Makefile
+*.bundle

data/ext/typhoeus/extconf.rb

@@ -0,0 +1,65 @@
+ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
+
+# :stopdoc:
+
+require 'mkmf'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+specified_curl = ARGV[0] =~ /^--with-curl/ ? ARGV[0].split("=")[1] : nil
+LIBDIR = specified_curl ? "#{specified_curl}/lib": Config::CONFIG['libdir']
+INCLUDEDIR = specified_curl ? "#{specified_curl}/include" : Config::CONFIG['includedir']
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'macruby'
+  $LIBRUBYARG_STATIC.gsub!(/-static/, '')
+end
+
+$CFLAGS << " #{ENV["CFLAGS"]}"
+if Config::CONFIG['target_os'] == 'mingw32'
+  $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF"
+elsif Config::CONFIG['target_os'] == 'solaris2'
+  $CFLAGS << " -DUSE_INCLUDED_VASPRINTF"
+else
+  $CFLAGS << " -g -DXP_UNIX"
+end
+
+use_macports = !(defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby')
+$LIBPATH << "/opt/local/lib" if use_macports
+
+$CFLAGS << " -O3 -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  header = File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'include')
+  unless find_header('curl/curl.h', header)
+    abort "need libcurl"
+  end
+else
+  HEADER_DIRS = [
+    File.join(INCLUDEDIR, "curl"),
+    INCLUDEDIR,
+    '/usr/include/curl',
+    '/usr/local/include/curl'
+  ]
+
+  [
+    '/opt/local/include/curl',
+    '/opt/local/include',
+  ].each { |x| HEADER_DIRS.unshift(x) } if use_macports
+
+  unless find_header('curl/curl.h', *HEADER_DIRS)
+    abort "need libcurl"
+  end
+end
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  find_library('curl', 'curl_easy_init',
+               File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'bin'))
+else
+  find_library('curl', 'curl_easy_init',
+               LIBDIR,
+               '/opt/local/lib',
+               '/usr/local/lib',
+               '/usr/lib'
+    )
+end
+
+create_makefile("typhoeus/native")

data/ext/typhoeus/native.c

@@ -0,0 +1,12 @@
+#include <native.h>
+
+VALUE mTyphoeus;
+
+void Init_native()
+{
+  mTyphoeus = rb_const_get(rb_cObject, rb_intern("Typhoeus"));
+
+  init_typhoeus_easy();
+  init_typhoeus_multi();
+  init_typhoeus_form();
+}

data/ext/typhoeus/native.h

@@ -0,0 +1,22 @@
+#ifndef TYPHOEUS_NATIVE
+#define TYPHOEUS_NATIVE
+
+#include <ruby.h>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <curl/multi.h>
+
+void Init_native();
+extern VALUE mTyphoeus;
+extern void init_typhoeus_easy();
+extern void init_typhoeus_multi();
+extern void init_typhoeus_form();
+
+#endif
+
+#ifndef RSTRING_PTR
+
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+
+#endif

data/ext/typhoeus/typhoeus_easy.c

@@ -0,0 +1,232 @@
+#include <typhoeus_easy.h>
+
+static VALUE idAppend;
+VALUE cTyphoeusEasy;
+
+static void dealloc(CurlEasy *curl_easy) {
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+  }
+
+  curl_easy_cleanup(curl_easy->curl);
+
+  free(curl_easy);
+}
+
+static VALUE easy_setopt_string(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, StringValuePtr(parameter));
+  return opt_name;
+}
+
+static VALUE easy_setopt_form(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  CurlForm *curl_form;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+  Data_Get_Struct(parameter, CurlForm, curl_form);
+
+  CURLoption opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, curl_form->first);
+  return opt_name;
+}
+
+static VALUE easy_setopt_long(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, NUM2LONG(parameter));
+  return opt_name;
+}
+
+static VALUE easy_getinfo_string(VALUE self, VALUE info) {
+  char *info_string;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_string);
+
+  return rb_str_new2(info_string);
+}
+
+static VALUE easy_getinfo_long(VALUE self, VALUE info) {
+  long info_long;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_long);
+
+  return LONG2NUM(info_long);
+}
+
+static VALUE easy_getinfo_double(VALUE self, VALUE info) {
+  double info_double = 0;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_double);
+
+  return rb_float_new(info_double);
+}
+
+static VALUE easy_perform(VALUE self) {
+  CurlEasy *curl_easy;
+  CURLcode return_code;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+  return_code = curl_easy_perform(curl_easy->curl);
+  rb_iv_set(self, "@curl_return_code", INT2FIX(return_code));
+
+  return Qnil;
+}
+
+static size_t write_data_handler(char *stream, size_t size, size_t nmemb, VALUE val) {
+  long stream_size = (long)(size * nmemb);
+  rb_funcall(val, idAppend, 1, rb_str_new(stream, stream_size));
+  return size * nmemb;
+}
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *data) {
+  size_t realsize = size * nmemb;
+  RequestChunk *mem = (RequestChunk *)data;
+
+  if (realsize > mem->size - mem->read) {
+    realsize = mem->size - mem->read;
+  }
+
+  if (realsize != 0) {
+    memcpy(ptr, &(mem->memory[mem->read]), realsize);
+    mem->read += realsize;
+  }
+
+  return realsize;
+}
+
+static void set_response_handlers(VALUE easy, CURL *curl) {
+  rb_iv_set(easy, "@response_body", rb_str_new2(""));
+  rb_iv_set(easy, "@response_header", rb_str_new2(""));
+
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, rb_iv_get(easy, "@response_body"));
+  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_HEADERDATA, rb_iv_get(easy, "@response_header"));
+}
+
+static VALUE easy_reset(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+    curl_easy->request_chunk = NULL;
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+    curl_easy->headers = NULL;
+  }
+
+  curl_easy_reset(curl_easy->curl);
+
+  set_response_handlers(self, curl_easy->curl);
+
+  return Qnil;
+}
+
+static VALUE easy_add_header(VALUE self, VALUE header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->headers = curl_slist_append(curl_easy->headers, RSTRING_PTR(header));
+  return header;
+}
+
+static VALUE easy_set_headers(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_HTTPHEADER, curl_easy->headers);
+
+  return Qnil;
+}
+
+static VALUE easy_set_request_body(VALUE self, VALUE data, VALUE content_length_header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->request_chunk = ALLOC(RequestChunk);
+  curl_easy->request_chunk->size = RSTRING_LEN(data);
+  curl_easy->request_chunk->memory = StringValuePtr(data);
+  curl_easy->request_chunk->read = 0;
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READFUNCTION, (curl_read_callback)read_callback);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READDATA, curl_easy->request_chunk);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_INFILESIZE, RSTRING_LEN(data));
+
+  return Qnil;
+}
+
+static VALUE easy_escape(VALUE self, VALUE data, VALUE length) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  return rb_str_new2(curl_easy_escape(curl_easy->curl, StringValuePtr(data), (int)NUM2INT(length)));
+}
+
+static VALUE version(VALUE self) {
+  return rb_str_new2(curl_version());
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CURL *curl = curl_easy_init();
+  CurlEasy *curl_easy = ALLOC(CurlEasy);
+  curl_easy->curl = curl;
+  curl_easy->headers = NULL;
+  curl_easy->request_chunk = NULL;
+  VALUE easy = Data_Wrap_Struct(cTyphoeusEasy, 0, dealloc, curl_easy);
+
+  set_response_handlers(easy, curl);
+
+  rb_obj_call_init(easy, argc, argv);
+
+  return easy;
+}
+
+static VALUE curl_error_message(VALUE self) {
+  VALUE return_code = rb_iv_get(self, "@curl_return_code");
+  if (return_code == Qnil)
+    return Qnil;
+  else {
+    CURLcode rc = (CURLcode)FIX2INT(return_code);
+    return rb_str_new2(curl_easy_strerror(rc));
+  }
+}
+
+void init_typhoeus_easy() {
+  VALUE klass = cTyphoeusEasy = rb_define_class_under(mTyphoeus, "Easy", rb_cObject);
+  idAppend = rb_intern("<<");
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_method(klass, "curl_error_message", curl_error_message, 0);
+  rb_define_private_method(klass, "easy_setopt_string", easy_setopt_string, 2);
+  rb_define_private_method(klass, "easy_setopt_form", easy_setopt_form, 2);
+  rb_define_private_method(klass, "easy_setopt_long", easy_setopt_long, 2);
+  rb_define_private_method(klass, "easy_getinfo_string", easy_getinfo_string, 1);
+  rb_define_private_method(klass, "easy_getinfo_long", easy_getinfo_long, 1);
+  rb_define_private_method(klass, "easy_getinfo_double", easy_getinfo_double, 1);
+  rb_define_private_method(klass, "easy_perform", easy_perform, 0);
+  rb_define_private_method(klass, "easy_reset", easy_reset, 0);
+  rb_define_private_method(klass, "easy_set_request_body", easy_set_request_body, 1);
+  rb_define_private_method(klass, "easy_set_headers", easy_set_headers, 0);
+  rb_define_private_method(klass, "easy_add_header", easy_add_header, 1);
+  rb_define_private_method(klass, "easy_escape", easy_escape, 2);
+  rb_define_private_method(klass, "version", version, 0);
+}