arachni-typhoeus 0.2.0

data/.gitignore

@@ -0,0 +1,3 @@
+.*.sw?
+*.gem
+.bundle

data/CHANGELOG.markdown

@@ -0,0 +1,43 @@
+Current Master
+--------------
+* Added extended proxy support [Zapotek, GH-46]
+* eliminated compile time warnings by using proper type declarations [skaes, GH-54]
+* fixed broken calls to rb_raise [skaes, GH-54]
+* prevent leaking of curl easy handles when exceptions are raised (either from typhoeus itself or user callbacks) [skaes, GH-54]
+* fixed Easy#timed_out? using curl return codes [skaes, GH-54]
+* provide curl return codes and corresponding curl error messages on classes Easy and Request [skaes, GH-54]
+* allow VCR to whitelist hosts in Typhoeus stubbing/mocking [myronmarston, GH-57]
+* added timed_out? documentation, method to Response [dbalatero, GH-34]
+* added abort to Hydra to prematurely stop a hydra.run [Zapotek]
+
+0.2.0
+------
+* Fix warning in Request#headers from attr_accessor
+* Params with array values were not parsing into the format that rack expects
+[GH-39, smartocci]
+* Removed Rack as a dependency [GH-45]
+* Added integration hooks for VCR!
+
+0.1.31
+------
+* Fixed bug in setting compression encoding [morhekil]
+* Exposed authentication control methods through Request interface [morhekil]
+
+0.1.30
+-----------
+* Exposed CURLOPT_CONNECTTIMEOUT_MS to Requests [balexis]
+
+0.1.29
+------
+* Fixed a memory corruption with using CURLOPT_POSTFIELDS [gravis,
+32531d0821aecc4]
+
+0.1.28
+----------------
+* Added SSL cert options for Typhoeus::Easy [GH-25, gravis]
+* Ported SSL cert options to Typhoeus::Request interface [gravis]
+* Added support for any HTTP method (purge for Varnish) [ryana]
+
+0.1.27
+------
+* Added rack as dependency, added dev dependencies to Rakefile [GH-21]

data/Gemfile

@@ -0,0 +1,9 @@
+source :rubygems
+
+group :test do
+  gem 'rspec', '1.3.1'
+  gem 'jeweler'
+  gem 'json'
+  gem 'sinatra'
+  gem 'diff-lcs'
+end

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    gemcutter (0.6.1)
+    git (1.2.5)
+    jeweler (1.4.0)
+      gemcutter (>= 0.1.0)
+      git (>= 1.2.5)
+      rubyforge (>= 2.0.0)
+    json (1.4.6)
+    json_pure (1.4.6)
+    rack (1.2.1)
+    rspec (1.3.1)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+    sinatra (1.1.0)
+      rack (~> 1.1)
+      tilt (~> 1.1)
+    tilt (1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  diff-lcs
+  jeweler
+  json
+  rspec (= 1.3.1)
+  sinatra

data/README.textile

@@ -0,0 +1,6 @@
+h1. Info
+
+This branch holds the Arachni sanctioned version of Typhoeus.
+Typhoeus 0.2.1 doesn't agree with Arachni on some matters so the gem built using this branch remedies the situation.
+
+Original Typhoeus repo @  https://github.com/dbalatero/typhoeus

data/Rakefile

@@ -0,0 +1,40 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require "spec"
+require "spec/rake/spectask"
+require 'lib/typhoeus'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "arachni-typhoeus"
+    gemspec.summary = "Typhoeus gem tailored for Arachni."
+    gemspec.description = "Original @ https://github.com/dbalatero/typhoeus."
+    gemspec.email = "tasos.laskos@gmail.com"
+    gemspec.homepage = "https://github.com/Zapotek/typhoeus/tree/arachni-typhoeus"
+    gemspec.authors = ["Paul Dix", "Tasos Laskos"]
+    gemspec.add_development_dependency "rspec"
+    gemspec.add_development_dependency "jeweler"
+    gemspec.add_development_dependency "diff-lcs"
+    gemspec.add_development_dependency "sinatra"
+    gemspec.add_development_dependency "json"
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+task :install do
+  rm_rf "*.gem"
+  puts `gem build typhoeus.gemspec`
+  puts `sudo gem install typhoeus-#{Typhoeus::VERSION}.gem`
+end
+
+desc "Run all the tests"
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/benchmarks/profile.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'rubygems'
+require 'ruby-prof'
+
+calls = 20
+@klass = Class.new do
+  include Typhoeus
+end
+
+Typhoeus.init_easy_objects
+
+RubyProf.start
+
+responses = []
+calls.times do |i|
+  responses << @klass.get("http://127.0.0.1:3000/#{i}")
+end
+
+responses.each {|r| }#raise unless r.response_body == "whatever"}
+
+result = RubyProf.stop
+
+ # Print a flat profile to text
+ printer = RubyProf::FlatPrinter.new(result)
+ printer.print(STDOUT, 0)

data/benchmarks/vs_nethttp.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'open-uri'
+require 'benchmark'
+include Benchmark
+
+
+calls = 20
+@klass = Class.new do
+  include Typhoeus
+end
+
+Typhoeus.init_easy_object_pool
+
+benchmark do |t|    
+  t.report("net::http") do
+    responses = []
+    
+    calls.times do |i|
+      responses << open("http://127.0.0.1:3000/#{i}").read
+    end
+    
+    responses.each {|r| raise unless r == "whatever"}    
+  end
+  
+  t.report("typhoeus") do
+    responses = []
+    
+    calls.times do |i|
+      responses << @klass.get("http://127.0.0.1:3000/#{i}")
+    end
+    
+    responses.each {|r| raise unless r.body == "whatever"}
+  end
+end

data/examples/twitter.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/typhoeus.rb'
+require 'json'
+
+class Twitter
+  include Typhoeus
+  remote_defaults :on_success => lambda {|response| JSON.parse(response.body)},
+                  :on_failure => lambda {|response| puts "error code: #{response.code}"},
+                  :base_uri   => "http://search.twitter.com"
+
+  define_remote_method :search, :path => '/search.json'
+  define_remote_method :trends, :path => '/trends/:time_frame.json'
+end
+
+tweets = Twitter.search(:params => {:q => "railsconf"})
+trends = Twitter.trends(:time_frame => :current)
+
+# and then the calls don't actually happen until the first time you
+# call a method on one of the objects returned from the remote_method
+
+puts tweets.keys # it's a hash from parsed JSON

data/ext/typhoeus/.gitignore

@@ -0,0 +1,7 @@
+conftest.dSYM/*
+conftest.dSYM
+mkmf.log
+*.o
+native.bundle
+Makefile
+*.bundle

data/ext/typhoeus/extconf.rb

@@ -0,0 +1,65 @@
+ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
+
+# :stopdoc:
+
+require 'mkmf'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+specified_curl = ARGV[0] =~ /^--with-curl/ ? ARGV[0].split("=")[1] : nil
+LIBDIR = specified_curl ? "#{specified_curl}/lib": Config::CONFIG['libdir']
+INCLUDEDIR = specified_curl ? "#{specified_curl}/include" : Config::CONFIG['includedir']
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'macruby'
+  $LIBRUBYARG_STATIC.gsub!(/-static/, '')
+end
+
+$CFLAGS << " #{ENV["CFLAGS"]}"
+if Config::CONFIG['target_os'] == 'mingw32'
+  $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF"
+elsif Config::CONFIG['target_os'] == 'solaris2'
+  $CFLAGS << " -DUSE_INCLUDED_VASPRINTF"
+else
+  $CFLAGS << " -g -DXP_UNIX"
+end
+
+use_macports = !(defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby')
+$LIBPATH << "/opt/local/lib" if use_macports
+
+$CFLAGS << " -O3 -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  header = File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'include')
+  unless find_header('curl/curl.h', header)
+    abort "need libcurl"
+  end
+else
+  HEADER_DIRS = [
+    File.join(INCLUDEDIR, "curl"),
+    INCLUDEDIR,
+    '/usr/include/curl',
+    '/usr/local/include/curl'
+  ]
+
+  [
+    '/opt/local/include/curl',
+    '/opt/local/include',
+  ].each { |x| HEADER_DIRS.unshift(x) } if use_macports
+
+  unless find_header('curl/curl.h', *HEADER_DIRS)
+    abort "need libcurl"
+  end
+end
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  find_library('curl', 'curl_easy_init',
+               File.join(ROOT, 'cross', 'curl-7.19.4.win32', 'bin'))
+else
+  find_library('curl', 'curl_easy_init',
+               LIBDIR,
+               '/opt/local/lib',
+               '/usr/local/lib',
+               '/usr/lib'
+    )
+end
+
+create_makefile("typhoeus/native")

data/ext/typhoeus/native.c

@@ -0,0 +1,11 @@
+#include <native.h>
+
+VALUE mTyphoeus;
+
+void Init_native()
+{
+  mTyphoeus = rb_const_get(rb_cObject, rb_intern("Typhoeus"));
+
+  init_typhoeus_easy();
+  init_typhoeus_multi();
+}

data/ext/typhoeus/native.h

@@ -0,0 +1,21 @@
+#ifndef TYPHOEUS_NATIVE
+#define TYPHOEUS_NATIVE
+
+#include <ruby.h>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <curl/multi.h>
+
+void Init_native();
+extern VALUE mTyphoeus;
+extern void init_typhoeus_easy();
+extern void init_typhoeus_multi();
+
+#endif
+
+#ifndef RSTRING_PTR
+
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+
+#endif

data/ext/typhoeus/typhoeus_easy.c

@@ -0,0 +1,220 @@
+#include <typhoeus_easy.h>
+
+static VALUE idAppend;
+VALUE cTyphoeusEasy;
+
+static void dealloc(CurlEasy *curl_easy) {
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+  }
+
+  curl_easy_cleanup(curl_easy->curl);
+
+  free(curl_easy);
+}
+
+static VALUE easy_setopt_string(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, StringValuePtr(parameter));
+  return opt_name;
+}
+
+static VALUE easy_setopt_long(VALUE self, VALUE opt_name, VALUE parameter) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(opt_name);
+  curl_easy_setopt(curl_easy->curl, opt, NUM2LONG(parameter));
+  return opt_name;
+}
+
+static VALUE easy_getinfo_string(VALUE self, VALUE info) {
+  char *info_string;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_string);
+
+  return rb_str_new2(info_string);
+}
+
+static VALUE easy_getinfo_long(VALUE self, VALUE info) {
+  long info_long;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_long);
+
+  return LONG2NUM(info_long);
+}
+
+static VALUE easy_getinfo_double(VALUE self, VALUE info) {
+  double info_double = 0;
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  CURLoption opt = NUM2LONG(info);
+  curl_easy_getinfo(curl_easy->curl, opt, &info_double);
+
+  return rb_float_new(info_double);
+}
+
+static VALUE easy_perform(VALUE self) {
+  CurlEasy *curl_easy;
+  CURLcode return_code;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+  return_code = curl_easy_perform(curl_easy->curl);
+  rb_iv_set(self, "@curl_return_code", INT2FIX(return_code));
+
+  return Qnil;
+}
+
+static size_t write_data_handler(char *stream, size_t size, size_t nmemb, VALUE val) {
+  long stream_size = (long)(size * nmemb);
+  rb_funcall(val, idAppend, 1, rb_str_new(stream, stream_size));
+  return size * nmemb;
+}
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *data) {
+  size_t realsize = size * nmemb;
+  RequestChunk *mem = (RequestChunk *)data;
+
+  if (realsize > mem->size - mem->read) {
+    realsize = mem->size - mem->read;
+  }
+
+  if (realsize != 0) {
+    memcpy(ptr, &(mem->memory[mem->read]), realsize);
+    mem->read += realsize;
+  }
+
+  return realsize;
+}
+
+static void set_response_handlers(VALUE easy, CURL *curl) {
+  rb_iv_set(easy, "@response_body", rb_str_new2(""));
+  rb_iv_set(easy, "@response_header", rb_str_new2(""));
+
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, rb_iv_get(easy, "@response_body"));
+  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (curl_write_callback)&write_data_handler);
+  curl_easy_setopt(curl, CURLOPT_HEADERDATA, rb_iv_get(easy, "@response_header"));
+}
+
+static VALUE easy_reset(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  if (curl_easy->request_chunk != NULL) {
+    free(curl_easy->request_chunk);
+    curl_easy->request_chunk = NULL;
+  }
+
+  if (curl_easy->headers != NULL) {
+    curl_slist_free_all(curl_easy->headers);
+    curl_easy->headers = NULL;
+  }
+
+  curl_easy_reset(curl_easy->curl);
+
+  set_response_handlers(self, curl_easy->curl);
+
+  return Qnil;
+}
+
+static VALUE easy_add_header(VALUE self, VALUE header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->headers = curl_slist_append(curl_easy->headers, RSTRING_PTR(header));
+  return header;
+}
+
+static VALUE easy_set_headers(VALUE self) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_HTTPHEADER, curl_easy->headers);
+
+  return Qnil;
+}
+
+static VALUE easy_set_request_body(VALUE self, VALUE data, VALUE content_length_header) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  curl_easy->request_chunk = ALLOC(RequestChunk);
+  curl_easy->request_chunk->size = RSTRING_LEN(data);
+  curl_easy->request_chunk->memory = StringValuePtr(data);
+  curl_easy->request_chunk->read = 0;
+
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READFUNCTION, (curl_read_callback)read_callback);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_READDATA, curl_easy->request_chunk);
+  curl_easy_setopt(curl_easy->curl, CURLOPT_INFILESIZE, RSTRING_LEN(data));
+
+  return Qnil;
+}
+
+static VALUE easy_escape(VALUE self, VALUE data, VALUE length) {
+  CurlEasy *curl_easy;
+  Data_Get_Struct(self, CurlEasy, curl_easy);
+
+  return rb_str_new2(curl_easy_escape(curl_easy->curl, StringValuePtr(data), (int)NUM2INT(length)));
+}
+
+static VALUE version(VALUE self) {
+  return rb_str_new2(curl_version());
+}
+
+static VALUE new(int argc, VALUE *argv, VALUE klass) {
+  CURL *curl = curl_easy_init();
+  CurlEasy *curl_easy = ALLOC(CurlEasy);
+  curl_easy->curl = curl;
+  curl_easy->headers = NULL;
+  curl_easy->request_chunk = NULL;
+  VALUE easy = Data_Wrap_Struct(cTyphoeusEasy, 0, dealloc, curl_easy);
+
+  set_response_handlers(easy, curl);
+
+  rb_obj_call_init(easy, argc, argv);
+
+  return easy;
+}
+
+static VALUE curl_error_message(VALUE self) {
+  VALUE return_code = rb_iv_get(self, "@curl_return_code");
+  if (return_code == Qnil)
+    return Qnil;
+  else {
+    CURLcode rc = (CURLcode)FIX2INT(return_code);
+    return rb_str_new2(curl_easy_strerror(rc));
+  }
+}
+
+void init_typhoeus_easy() {
+  VALUE klass = cTyphoeusEasy = rb_define_class_under(mTyphoeus, "Easy", rb_cObject);
+  idAppend = rb_intern("<<");
+  rb_define_singleton_method(klass, "new", new, -1);
+  rb_define_method(klass, "curl_error_message", curl_error_message, 0);
+  rb_define_private_method(klass, "easy_setopt_string", easy_setopt_string, 2);
+  rb_define_private_method(klass, "easy_setopt_long", easy_setopt_long, 2);
+  rb_define_private_method(klass, "easy_getinfo_string", easy_getinfo_string, 1);
+  rb_define_private_method(klass, "easy_getinfo_long", easy_getinfo_long, 1);
+  rb_define_private_method(klass, "easy_getinfo_double", easy_getinfo_double, 1);
+  rb_define_private_method(klass, "easy_perform", easy_perform, 0);
+  rb_define_private_method(klass, "easy_reset", easy_reset, 0);
+  rb_define_private_method(klass, "easy_set_request_body", easy_set_request_body, 1);
+  rb_define_private_method(klass, "easy_set_headers", easy_set_headers, 0);
+  rb_define_private_method(klass, "easy_add_header", easy_add_header, 1);
+  rb_define_private_method(klass, "easy_escape", easy_escape, 2);
+  rb_define_private_method(klass, "version", version, 0);
+}