scrapey 0.0.5 → 0.0.7

data/README.md

@@ -17,12 +17,13 @@ A simple framework for solving common scraping problems
 ## Examples
 
 ### CSV
+By default scrapey will save as 'output.csv'
+You can change this with:
+
+    @output = 'mycsv.csv'
 
 ```ruby
 require 'scrapey'
-# By default scrapey will save as 'output.csv'
-# You can change this with:
-# @output = 'mycsv.csv'
 
 page = get 'http://www.alexa.com/topsites'
 page.search('li.site-listing').each do |li|
@@ -31,10 +32,9 @@ end
 ```
 
 ### Database
+if you created a scrapey project you can fill out the database connection information in config/config.yml
 ```ruby
 require 'scrapey'
-# if you created a scrapey project you can fill out the database connection
-# information in config/config.yml
 
 tables 'Movie', 'Actor' # create ActiveRecord  models
 
@@ -90,14 +90,13 @@ get 'some_throttled_website_url'
 Scrapey will ensure that the callbacks are threadsafe
 ```ruby
 require 'scrapey'
-require 'scrapey/multi'
 
 fields 'url', 'title'
 
-def scrape url, response, header
+def scrape url, response
   doc = Nokogiri::HTML response
   save({'url' => url, 'title' => doc.at('title').text})
 end
 
-multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], :threads => 3, :callback => :scrape
+multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], :threads => 3, :on_success => :scrape
 ```

data/examples/multi.rb

@@ -1,11 +1,10 @@
 require 'scrapey'
-require 'scrapey/multi'
 
 fields 'url', 'title'
 
-def scrape url, response, header
+def scrape url, response
   doc = Nokogiri::HTML response
-  save({'url' => url, 'title' => doc.at('title').text})
+  save({'url' => url, 'title' => doc.title})
 end
 
-multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], :threads => 3, :callback => :scrape
+multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], :threads => 3, :on_success => :scrape

data/examples/multi2.rb

@@ -1,25 +1,24 @@
 require 'scrapey'
-require 'scrapey/multi'
 
 fields 'url', 'title'
 
 def scrape url, response, header
   doc = Nokogiri::HTML response
-  save({'url' => url, 'title' => doc.at('title').text})
+  save({'url' => url, 'title' => doc.title})
   puts "scraped #{url}."
 end
 
 options = {
   :threads => 3,
-  :callback => :scrape,
-  :proxy => {:host => 'localhost', :port => 8888},
-  :head => {
+  :on_success => :scrape,
+  :proxy => 'http://localhost:8888',
+  :headers => {
     "Accept" => "*/*",
-    #"User-Agent" => "Scrapey #{Scrapey::VERSION}",
-    "Keep-alive" => "true"
+    "Keep-alive" => "true",
+    "Cookie" => "foo=bar"
   }
 }
 
-multi_get ['http://www.yahoo.com/', 'http://www.google.com/', 'http://www.bing.com/'], options
+multi_get ["https://twitter.com/", 'http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], options
 
 puts "this happens after all callbacks."

data/examples/output.csv

@@ -0,0 +1,2 @@
+url,title
+https://twitter.com/,Twitter

data/examples/status_check.rb

@@ -1,15 +1,14 @@
 require 'scrapey'
 require 'scrapey/multi'
-require 'pry'
 
 fields 'url', 'status'
 
-def on_success url, response, header
-  save({'url' => url, 'status' => header.status})
+def on_success url, response
+  save({'url' => url, 'status' => response.status_code})
 end
 
 def on_error url, e
-  save({'url' => url, 'status' => e})
+  save({'url' => url, 'status' => e.message})
 end
 
 multi_head ['http://locahlost2/foo', 'http://www.google.com/', 'http://www.bing.com/', 'http://www.bing.com/404.html']

data/lib/scrapey.rb

@@ -7,6 +7,7 @@ require "scrapey/scrapey"
 require "scrapey/constants"
 require "scrapey/cache"
 require "scrapey/database"
+require "scrapey/multi"
 
 include Scrapey
 

data/lib/scrapey/cache/disk.rb

@@ -12,10 +12,14 @@ module Scrapey
     filename = cache_filename url
     return nil unless File::exists?(filename)
     debug "Loading #{filename} from cache"
-    Nokogiri::HTML Marshal.load(File.read(filename))
+    begin
+      Nokogiri::HTML Marshal.load(File.read(filename))
+    rescue Exception => e
+      puts e.message
+    end
   end
 
   def save_cache url, doc, options = {}
-    File.open(cache_filename(url), "w") {|f| f << Marshal.dump(doc) }
+    File.open(cache_filename(url), "wb") {|f| f << Marshal.dump(doc) }
   end
 end

data/lib/scrapey/cache/redis.rb

@@ -9,9 +9,7 @@ module Scrapey
   def load_cache url
     debug "Loading #{url} from cache"
     return nil unless str = @redis.get(url)
-    debug "found it"
-    #binding.pry
-    Nokogiri::HTML Marshal.load(str)
+    Nokogiri::HTML Marshal.load(str) rescue nil
   end
 
   def save_cache url, body, options = {}

data/lib/scrapey/constants.rb

@@ -1,6 +1,5 @@
 module Scrapey
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
-  #ENV['SSL_FILE'] = "#{Gem.dir}/gems/scrapey-#{Scrapey::VERSION}/ssl/cacert.pem"
 end

data/lib/scrapey/multi.rb

@@ -1,47 +1,42 @@
-require 'em-http-request'
+require 'httpclient'
 
 module Scrapey
   def multi_get_or_post method, all_urls, options = {}
-    head = options.delete(:head) || {}
-    request_options = {:redirects => 10, :head => {"User-Agent" => "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"}.merge(head)}
-    threads = options[:threads] || 20
-    on_success = options[:on_success] || :on_success
-    on_error = options[:on_error] || :on_error
     all_urls.reject!{|url| is_cached? url} if @use_cache
-    @lock = Mutex.new
+    return unless all_urls.size > 0
+
+    threads    = options[:threads]    || [10, all_urls.size].min
+    on_success = options[:on_success] || :on_success
+    on_error   = options[:on_error]   || :on_error
+    user_agent = options[:user_agent] || "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
+    proxy      = options[:proxy]      || nil
+
+    @lock ||= Mutex.new
+    @http_clients ||= threads.times.map{HTTPClient.new(options[:proxies] ? options[:proxies].rotate!.first : proxy, user_agent).tap{|c| c.ssl_config.verify_mode, c.receive_timeout = OpenSSL::SSL::VERIFY_NONE, 10000}}
+    debug 'starting multi'
+
     all_urls.each_slice(threads) do |urls|
-      next unless urls.size > 0
-      EventMachine.run do
-        multi = EventMachine::MultiRequest.new
-        urls.each_with_index do |url, i|
-          multi.add i, EventMachine::HttpRequest.new(url, options).send(method, request_options)
-        end
-        multi.callback do
-          (0...multi.requests.length).each do |i|				
-            if multi.responses[:callback][i]
-              @lock.synchronize do
-                if defined? on_success
-                  send on_success, urls[i], multi.responses[:callback][i].response, multi.responses[:callback][i].response_header
-                else
-                  raise "#{on_success} not defined!"
-                end
-              end
+      urls.each_with_index.map do |url, i|
+        Thread.new do
+          begin
+            response = @http_clients[i].send method, url, options[:query], options[:headers]
+          rescue Exception => e
+            error = e
+          end
+          @lock.synchronize do
+            if response
+              send on_success, url, response
             else
-              if defined? on_error
-                send on_error, urls[i], multi.requests[i].error
-              else
-                raise "#{on_error} not defined!"
-              end
+              send on_error, url, e
             end
           end
-          EventMachine.stop
         end
-      end
+      end.each{|thread| thread.join}
     end
   end
 
-  def multi_get *args; multi_get_or_post 'get', *args; end
-  def multi_post *args; multi_get_or_post 'post', *args; end
+  def multi_get *args; multi_get_or_post 'get_content', *args; end
+  def multi_post *args; multi_get_or_post 'post_content', *args; end
   def multi_head *args; multi_get_or_post 'head', *args; end
 
 end

data/lib/scrapey/scrapey.rb

@@ -52,7 +52,9 @@ module Scrapey
     end
     case
       when item.is_a?(Array) then @csv << item
-      when item.is_a?(Hash) then @csv << @fields.map{|f| item[f]}
+      when item.is_a?(Hash)
+        raise 'No fields defined!' unless @fields
+        @csv << @fields.map{|f| item[f]}
       else raise "unsupported type: #{item.class}"
     end
   end

data/lib/scrapey/template.rb

@@ -12,7 +12,7 @@ module Scrapey
         FileUtils.mv fn, fn.gsub('template', name)
       end
       buf = File.read "#{name}.iss"
-      buf.gsub! /Template/, "rightmove_rentals".tr('_', ' ').gsub(/\w+/){|x| x.capitalize}
+      buf.gsub! /Template/, name.tr('_', ' ').gsub(/\w+/){|x| x.capitalize}
       buf.gsub! /template/, name
       File.open("#{name}.iss", 'w'){|f| f << buf}
 

data/output.csv

@@ -1,3 +1,5 @@
 url,status
 http://www.bing.com/,200
 http://www.bing.com/404.html,404
+http://locahlost2/foo,getaddrinfo: No such host is known.  (http://locahlost2:80)
+http://www.google.com/,302

data/template/Rakefile

@@ -13,10 +13,11 @@ end
 
 desc "Copy installer to dropbox folder"
 task 'dropbox' do
+  file = 'setup.exe'
   raise 'no dropbox folder!' unless ENV['DROPBOX']
   folder = [ENV['DROPBOX'], name].join('/').squeeze('/')
   FileUtils.mkdir(folder) unless File.exists?(folder)
-  FileUtils.cp "Output/setup.exe", folder
-  url = [ENV['DROPBOX_public_url'], name, 'setup.exe'].join('/').squeeze('/')
+  FileUtils.cp "Output/#{file}", folder
+  url = [ENV['DROPBOX_public_url'], name, file].join('/').squeeze('/')
   puts "uploaded to #{url}"
 end

data/template/output.csv

@@ -0,0 +1,2 @@
+name,address,zip
+,,

data/template/src/template.rb

@@ -1,6 +1,30 @@
 require 'scrapey'
+require 'pry'
+
+# some skeleton code that I like to start with
 # require 'scrapey/multi' #=> requires em-http-request
 
 # sample customizations...
 # @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
 # @output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")
+
+def scrape div
+  a = div.at('a')
+  url = URI.join(@url, a[:href]).to_s
+  return unless visited? url
+  item = {}
+  
+  save item  
+  exit if defined? Ocra
+rescue StandardError => e
+  binding.pry
+end
+
+#use_cache :redis
+
+fields 'name', 'address', 'zip'
+
+@url = "http://www.example.com/"
+
+page = get @url
+scrape page.at('div')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-15 00:00:00.000000000 Z
+date: 2012-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -59,6 +59,7 @@ files:
 - examples/imdb.rb
 - examples/multi.rb
 - examples/multi2.rb
+- examples/output.csv
 - examples/redis.rb
 - examples/status_check.rb
 - lib/scrapey/cache/disk.rb
@@ -71,11 +72,11 @@ files:
 - lib/scrapey/template.rb
 - lib/scrapey.rb
 - output.csv
-- ponsesq
 - scrapey.gemspec
 - template/config/config.yml
 - template/Gemfile
 - template/icon.ico
+- template/output.csv
 - template/Rakefile
 - template/src/schema.rb
 - template/src/template.rb

data/ponsesq

@@ -1,593 +0,0 @@
-=> #<EventMachine::MultiRequest:0x2237178
- @callbacks=[],
- @deferred_args=[#<EventMachine::MultiRequest:0x2237178 ...>],
- @deferred_status=:succeeded,
- @deferred_timeout=nil,
- @errbacks=nil,
- @requests=
-  {0=>
-    #<EventMachine::HttpClient:0x21adc28
-     @callbacks=[],
-     @conn=
-      #<EventMachine::HttpConnection:0x21f3f30
-       @connopts=
-        #<HttpConnectionOptions:0x2231100
-         @connect_timeout=5,
-         @host="locahlost2",
-         @inactivity_timeout=10,
-         @port=80,
-         @proxy=nil,
-         @tls={}>,
-       @deferred=true,
-       @middleware=[],
-       @uri="http://locahlost2/foo">,
-     @content_charset=nil,
-     @content_decoder=nil,
-     @cookiejar=
-      #<EventMachine::HttpClient::CookieJar:0x21adc70
-       @jar=#<CookieJar::Jar:0x21adcb8 @domains={}>>,
-     @cookies=[],
-     @deferred_args=[#<EventMachine::HttpClient:0x21adc28 ...>],
-     @deferred_status=:failed,
-     @deferred_timeout=nil,
-     @errbacks=[],
-     @error="unable to resolve server address",
-     @headers=nil,
-     @req=
-      #<HttpClientOptions:0x21f42f0
-       @body=nil,
-       @decoding=true,
-       @file=nil,
-       @followed=0,
-       @headers=
-        {"User-Agent"=>
-          "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-       @host="locahlost2",
-       @keepalive=false,
-       @method="HEAD",
-       @pass_cookies=true,
-       @path=nil,
-       @port=80,
-       @query=nil,
-       @redirects=10,
-       @uri=#<Addressable::URI:0x10e696c URI:http://locahlost2:80/foo>>,
-     @response="",
-     @response_header={},
-     @state=:response_header,
-     @stream=nil>,
-   1=>
-    #<EventMachine::HttpClient:0x20ed610
-     @callbacks=[],
-     @conn=
-      #<EventMachine::HttpConnection:0x2136d08
-       @clients=[],
-       @conn=
-        #<EventMachine::HttpStubConnection:0x20c2e60
-         @callbacks=[],
-         @deferred_args=[],
-         @deferred_status=:succeeded,
-         @deferred_timeout=nil,
-         @errbacks=nil,
-         @parent=#<EventMachine::HttpConnection:0x2136d08 ...>,
-         @signature=5>,
-       @connopts=
-        #<HttpConnectionOptions:0x215c7d8
-         @connect_timeout=5,
-         @host="www.google.com.",
-         @inactivity_timeout=10,
-         @port=80,
-         @proxy=nil,
-         @tls={}>,
-       @deferred=false,
-       @middleware=[],
-       @p=#<HTTP::Parser:0x20da578>,
-       @peer="\x02\x00\x00PJ}G^\x00\x00\x00\x00\x00\x00\x00\x00",
-       @pending=[],
-       @uri="http://www.google.com.">,
-     @content_charset=nil,
-     @content_decoder=nil,
-     @cookiejar=
-      #<EventMachine::HttpClient::CookieJar:0x20ed670
-       @jar=
-        #<CookieJar::Jar:0x20e5198
-         @domains=
-          {".google.com.ph"=>
-            {"/"=>
-              {"PREF"=>
-                PREF=ID=ec6b270e7fe890fa:FF=0:TM=1344992862:LM=1344992862:S=WlLM9juILblGoi5k,
-               "NID"=>
-                NID=62=5O_DMdySUEeIJXvveuCr1U8UEfYEhurh0X2Is-a5f0xhTw5CxEY9gELcAyCmwqt4MxVLDpvT2anCV79hhXHfd-QPi0zRY8bCiqh7BlH1B3w0wfE3eg0PTR_KbXUJBBFm}}}>>,
-     @cookies=[],
-     @deferred_args=[#<EventMachine::HttpClient:0x20ed610 ...>],
-     @deferred_status=:succeeded,
-     @deferred_timeout=nil,
-     @errbacks=
-      [#<Proc:0x20297e8@C:/Ruby193/lib/ruby/gems/1.9.1/gems/em-http-request-1.0.2/lib/em-http/multi.rb:42>],
-     @error=nil,
-     @headers=nil,
-     @req=
-      #<HttpClientOptions:0x2129ec8
-       @body=nil,
-       @decoding=true,
-       @file=nil,
-       @followed=1,
-       @headers=
-        {"User-Agent"=>
-          "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-       @host="www.google.com.ph",
-       @keepalive=false,
-       @method="HEAD",
-       @pass_cookies=true,
-       @path=nil,
-       @port=80,
-       @query=nil,
-       @redirects=10,
-       @uri=#<Addressable::URI:0xe683ec URI:http://www.google.com.ph:80/>>,
-     @response="",
-     @response_header=
-      {"DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-       "EXPIRES"=>"-1",
-       "CACHE_CONTROL"=>"private, max-age=0",
-       "CONTENT_TYPE"=>"text/html; charset=ISO-8859-1",
-       "SET_COOKIE"=>
-        ["PREF=ID=ec6b270e7fe890fa:FF=0:TM=1344992862:LM=1344992862:S=WlLM9juILblGoi5k; expires=Fri, 15-Aug-2014 01:07:42 GMT; path=/; domain=.google.com.ph",
-         "NID=62=5O_DMdySUEeIJXvveuCr1U8UEfYEhurh0X2Is-a5f0xhTw5CxEY9gELcAyCmwqt4MxVLDpvT2anCV79hhXHfd-QPi0zRY8bCiqh7BlH1B3w0wfE3eg0PTR_KbXUJBBFm; expires=Thu, 14-Feb-2013 01:07:42 GMT; path=/; domain=.google.com.ph; HttpOnly"],
-       "P3P"=>
-        "CP=\"This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info.\"",
-       "SERVER"=>"gws",
-       "X_XSS_PROTECTION"=>"1; mode=block",
-       "X_FRAME_OPTIONS"=>"SAMEORIGIN",
-       "CONNECTION"=>"close"},
-     @state=:finished,
-     @stream=nil>,
-   2=>
-    #<EventMachine::HttpClient:0x08d8490
-     @callbacks=[],
-     @conn=
-      #<EventMachine::HttpConnection:0x08d91b0
-       @clients=[],
-       @conn=
-        #<EventMachine::HttpStubConnection:0x08d8148
-         @callbacks=[],
-         @deferred_args=[],
-         @deferred_status=:succeeded,
-         @deferred_timeout=nil,
-         @errbacks=nil,
-         @parent=#<EventMachine::HttpConnection:0x08d91b0 ...>,
-         @signature=3>,
-       @connopts=
-        #<HttpConnectionOptions:0x1fed3b0
-         @connect_timeout=5,
-         @host="www.bing.com",
-         @inactivity_timeout=10,
-         @port=80,
-         @proxy=nil,
-         @tls={}>,
-       @deferred=false,
-       @middleware=[],
-       @p=#<HTTP::Parser:0x08d8028>,
-       @peer="\x02\x00\x00P|j\xAE\xB2\x00\x00\x00\x00\x00\x00\x00\x00",
-       @pending=[],
-       @uri="http://www.bing.com/">,
-     @content_charset=nil,
-     @content_decoder=nil,
-     @cookiejar=
-      #<EventMachine::HttpClient::CookieJar:0x08d8340
-       @jar=
-        #<CookieJar::Jar:0x08d8328
-         @domains=
-          {".bing.com"=>
-            {"/"=>
-              {"_FS"=>_FS=NU=1,
-               "_SS"=>_SS=SID=847F099F99524E2F97F8236B4B203509,
-               "SRCHD"=>SRCHD=D=2430787&MS=2430787&AF=NOFORM,
-               "SRCHUSR"=>SRCHUSR=AUTOREDIR=0&GEOVAR=&DOB=20120815}},
-           "www.bing.com"=>
-            {"/"=>
-              {"SRCHUID"=>
-                SRCHUID=V=2&GUID=28C754BC00C346D19F70AD5235BC50B4}}}>>,
-     @cookies=[],
-     @deferred_args=[#<EventMachine::HttpClient:0x08d8490 ...>],
-     @deferred_status=:succeeded,
-     @deferred_timeout=nil,
-     @errbacks=[],
-     @error=nil,
-     @headers=nil,
-     @req=
-      #<HttpClientOptions:0x08d9030
-       @body=nil,
-       @decoding=true,
-       @file=nil,
-       @followed=0,
-       @headers=
-        {"User-Agent"=>
-          "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-       @host="www.bing.com",
-       @keepalive=false,
-       @method="HEAD",
-       @pass_cookies=true,
-       @path=nil,
-       @port=80,
-       @query=nil,
-       @redirects=10,
-       @uri=#<Addressable::URI:0x46c50c URI:http://www.bing.com:80/>>,
-     @response="",
-     @response_header=
-      {"CACHE_CONTROL"=>"private, max-age=0",
-       "CONTENT_TYPE"=>"text/html",
-       "P3P"=>"CP=\"NON UNI COM NAV STA LOC CURa DEVa PSAa PSDa OUR IND\"",
-       "DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-       "CONTENT_LENGTH"=>"1",
-       "CONNECTION"=>"close",
-       "SET_COOKIE"=>
-        ["_FS=NU=1; domain=.bing.com; path=/",
-         "_SS=SID=847F099F99524E2F97F8236B4B203509; domain=.bing.com; path=/",
-         "SRCHD=D=2430787&MS=2430787&AF=NOFORM; expires=Fri, 15-Aug-2014 01:07:42 GMT; domain=.bing.com; path=/",
-         "SRCHUID=V=2&GUID=28C754BC00C346D19F70AD5235BC50B4; expires=Fri, 15-Aug-2014 01:07:42 GMT; path=/",
-         "SRCHUSR=AUTOREDIR=0&GEOVAR=&DOB=20120815; expires=Fri, 15-Aug-2014 01:07:42 GMT; domain=.bing.com; path=/"]},
-     @state=:finished,
-     @stream=nil>,
-   3=>
-    #<EventMachine::HttpClient:0x1cc2a98
-     @callbacks=[],
-     @conn=
-      #<EventMachine::HttpConnection:0x08d7698
-       @clients=[],
-       @conn=
-        #<EventMachine::HttpStubConnection:0x1cc22e8
-         @callbacks=[],
-         @deferred_args=[],
-         @deferred_status=:succeeded,
-         @deferred_timeout=nil,
-         @errbacks=nil,
-         @parent=#<EventMachine::HttpConnection:0x08d7698 ...>,
-         @signature=4>,
-       @connopts=
-        #<HttpConnectionOptions:0x08d7d70
-         @connect_timeout=5,
-         @host="www.bing.com",
-         @inactivity_timeout=10,
-         @port=80,
-         @proxy=nil,
-         @tls={}>,
-       @deferred=false,
-       @middleware=[],
-       @p=#<HTTP::Parser:0x1cc2150>,
-       @peer="\x02\x00\x00P|j\xAE\xB2\x00\x00\x00\x00\x00\x00\x00\x00",
-       @pending=[],
-       @uri="http://www.bing.com/404.html">,
-     @content_charset=nil,
-     @content_decoder=nil,
-     @cookiejar=
-      #<EventMachine::HttpClient::CookieJar:0x1cc27c8
-       @jar=#<CookieJar::Jar:0x1cc2660 @domains={}>>,
-     @cookies=[],
-     @deferred_args=[#<EventMachine::HttpClient:0x1cc2a98 ...>],
-     @deferred_status=:succeeded,
-     @deferred_timeout=nil,
-     @errbacks=[],
-     @error=nil,
-     @headers=nil,
-     @req=
-      #<HttpClientOptions:0x08d7650
-       @body=nil,
-       @decoding=true,
-       @file=nil,
-       @followed=0,
-       @headers=
-        {"User-Agent"=>
-          "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-       @host="www.bing.com",
-       @keepalive=false,
-       @method="HEAD",
-       @pass_cookies=true,
-       @path=nil,
-       @port=80,
-       @query=nil,
-       @redirects=10,
-       @uri=#<Addressable::URI:0xe61f18 URI:http://www.bing.com:80/404.html>>,
-     @response="",
-     @response_header=
-      {"CACHE_CONTROL"=>"no-cache",
-       "CONTENT_TYPE"=>"text/html",
-       "P3P"=>"CP=\"NON UNI COM NAV STA LOC CURa DEVa PSAa PSDa OUR IND\"",
-       "DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-       "CONTENT_LENGTH"=>"1",
-       "CONNECTION"=>"close"},
-     @state=:finished,
-     @stream=nil>},
- @responses=
-  {:callback=>
-    {2=>
-      #<EventMachine::HttpClient:0x08d8490
-       @callbacks=[],
-       @conn=
-        #<EventMachine::HttpConnection:0x08d91b0
-         @clients=[],
-         @conn=
-          #<EventMachine::HttpStubConnection:0x08d8148
-           @callbacks=[],
-           @deferred_args=[],
-           @deferred_status=:succeeded,
-           @deferred_timeout=nil,
-           @errbacks=nil,
-           @parent=#<EventMachine::HttpConnection:0x08d91b0 ...>,
-           @signature=3>,
-         @connopts=
-          #<HttpConnectionOptions:0x1fed3b0
-           @connect_timeout=5,
-           @host="www.bing.com",
-           @inactivity_timeout=10,
-           @port=80,
-           @proxy=nil,
-           @tls={}>,
-         @deferred=false,
-         @middleware=[],
-         @p=#<HTTP::Parser:0x08d8028>,
-         @peer="\x02\x00\x00P|j\xAE\xB2\x00\x00\x00\x00\x00\x00\x00\x00",
-         @pending=[],
-         @uri="http://www.bing.com/">,
-       @content_charset=nil,
-       @content_decoder=nil,
-       @cookiejar=
-        #<EventMachine::HttpClient::CookieJar:0x08d8340
-         @jar=
-          #<CookieJar::Jar:0x08d8328
-           @domains=
-            {".bing.com"=>
-              {"/"=>
-                {"_FS"=>_FS=NU=1,
-                 "_SS"=>_SS=SID=847F099F99524E2F97F8236B4B203509,
-                 "SRCHD"=>SRCHD=D=2430787&MS=2430787&AF=NOFORM,
-                 "SRCHUSR"=>SRCHUSR=AUTOREDIR=0&GEOVAR=&DOB=20120815}},
-             "www.bing.com"=>
-              {"/"=>
-                {"SRCHUID"=>
-                  SRCHUID=V=2&GUID=28C754BC00C346D19F70AD5235BC50B4}}}>>,
-       @cookies=[],
-       @deferred_args=[#<EventMachine::HttpClient:0x08d8490 ...>],
-       @deferred_status=:succeeded,
-       @deferred_timeout=nil,
-       @errbacks=[],
-       @error=nil,
-       @headers=nil,
-       @req=
-        #<HttpClientOptions:0x08d9030
-         @body=nil,
-         @decoding=true,
-         @file=nil,
-         @followed=0,
-         @headers=
-          {"User-Agent"=>
-            "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-         @host="www.bing.com",
-         @keepalive=false,
-         @method="HEAD",
-         @pass_cookies=true,
-         @path=nil,
-         @port=80,
-         @query=nil,
-         @redirects=10,
-         @uri=#<Addressable::URI:0x46c50c URI:http://www.bing.com:80/>>,
-       @response="",
-       @response_header=
-        {"CACHE_CONTROL"=>"private, max-age=0",
-         "CONTENT_TYPE"=>"text/html",
-         "P3P"=>"CP=\"NON UNI COM NAV STA LOC CURa DEVa PSAa PSDa OUR IND\"",
-         "DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-         "CONTENT_LENGTH"=>"1",
-         "CONNECTION"=>"close",
-         "SET_COOKIE"=>
-          ["_FS=NU=1; domain=.bing.com; path=/",
-           "_SS=SID=847F099F99524E2F97F8236B4B203509; domain=.bing.com; path=/",
-           "SRCHD=D=2430787&MS=2430787&AF=NOFORM; expires=Fri, 15-Aug-2014 01:07:42 GMT; domain=.bing.com; path=/",
-           "SRCHUID=V=2&GUID=28C754BC00C346D19F70AD5235BC50B4; expires=Fri, 15-Aug-2014 01:07:42 GMT; path=/",
-           "SRCHUSR=AUTOREDIR=0&GEOVAR=&DOB=20120815; expires=Fri, 15-Aug-2014 01:07:42 GMT; domain=.bing.com; path=/"]},
-       @state=:finished,
-       @stream=nil>,
-     3=>
-      #<EventMachine::HttpClient:0x1cc2a98
-       @callbacks=[],
-       @conn=
-        #<EventMachine::HttpConnection:0x08d7698
-         @clients=[],
-         @conn=
-          #<EventMachine::HttpStubConnection:0x1cc22e8
-           @callbacks=[],
-           @deferred_args=[],
-           @deferred_status=:succeeded,
-           @deferred_timeout=nil,
-           @errbacks=nil,
-           @parent=#<EventMachine::HttpConnection:0x08d7698 ...>,
-           @signature=4>,
-         @connopts=
-          #<HttpConnectionOptions:0x08d7d70
-           @connect_timeout=5,
-           @host="www.bing.com",
-           @inactivity_timeout=10,
-           @port=80,
-           @proxy=nil,
-           @tls={}>,
-         @deferred=false,
-         @middleware=[],
-         @p=#<HTTP::Parser:0x1cc2150>,
-         @peer="\x02\x00\x00P|j\xAE\xB2\x00\x00\x00\x00\x00\x00\x00\x00",
-         @pending=[],
-         @uri="http://www.bing.com/404.html">,
-       @content_charset=nil,
-       @content_decoder=nil,
-       @cookiejar=
-        #<EventMachine::HttpClient::CookieJar:0x1cc27c8
-         @jar=#<CookieJar::Jar:0x1cc2660 @domains={}>>,
-       @cookies=[],
-       @deferred_args=[#<EventMachine::HttpClient:0x1cc2a98 ...>],
-       @deferred_status=:succeeded,
-       @deferred_timeout=nil,
-       @errbacks=[],
-       @error=nil,
-       @headers=nil,
-       @req=
-        #<HttpClientOptions:0x08d7650
-         @body=nil,
-         @decoding=true,
-         @file=nil,
-         @followed=0,
-         @headers=
-          {"User-Agent"=>
-            "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-         @host="www.bing.com",
-         @keepalive=false,
-         @method="HEAD",
-         @pass_cookies=true,
-         @path=nil,
-         @port=80,
-         @query=nil,
-         @redirects=10,
-         @uri=
-          #<Addressable::URI:0xe61f18 URI:http://www.bing.com:80/404.html>>,
-       @response="",
-       @response_header=
-        {"CACHE_CONTROL"=>"no-cache",
-         "CONTENT_TYPE"=>"text/html",
-         "P3P"=>"CP=\"NON UNI COM NAV STA LOC CURa DEVa PSAa PSDa OUR IND\"",
-         "DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-         "CONTENT_LENGTH"=>"1",
-         "CONNECTION"=>"close"},
-       @state=:finished,
-       @stream=nil>,
-     1=>
-      #<EventMachine::HttpClient:0x20ed610
-       @callbacks=[],
-       @conn=
-        #<EventMachine::HttpConnection:0x2136d08
-         @clients=[],
-         @conn=
-          #<EventMachine::HttpStubConnection:0x20c2e60
-           @callbacks=[],
-           @deferred_args=[],
-           @deferred_status=:succeeded,
-           @deferred_timeout=nil,
-           @errbacks=nil,
-           @parent=#<EventMachine::HttpConnection:0x2136d08 ...>,
-           @signature=5>,
-         @connopts=
-          #<HttpConnectionOptions:0x215c7d8
-           @connect_timeout=5,
-           @host="www.google.com.",
-           @inactivity_timeout=10,
-           @port=80,
-           @proxy=nil,
-           @tls={}>,
-         @deferred=false,
-         @middleware=[],
-         @p=#<HTTP::Parser:0x20da578>,
-         @peer="\x02\x00\x00PJ}G^\x00\x00\x00\x00\x00\x00\x00\x00",
-         @pending=[],
-         @uri="http://www.google.com.">,
-       @content_charset=nil,
-       @content_decoder=nil,
-       @cookiejar=
-        #<EventMachine::HttpClient::CookieJar:0x20ed670
-         @jar=
-          #<CookieJar::Jar:0x20e5198
-           @domains=
-            {".google.com.ph"=>
-              {"/"=>
-                {"PREF"=>
-                  PREF=ID=ec6b270e7fe890fa:FF=0:TM=1344992862:LM=1344992862:S=WlLM9juILblGoi5k,
-                 "NID"=>
-                  NID=62=5O_DMdySUEeIJXvveuCr1U8UEfYEhurh0X2Is-a5f0xhTw5CxEY9gELcAyCmwqt4MxVLDpvT2anCV79hhXHfd-QPi0zRY8bCiqh7BlH1B3w0wfE3eg0PTR_KbXUJBBFm}}}>>,
-       @cookies=[],
-       @deferred_args=[#<EventMachine::HttpClient:0x20ed610 ...>],
-       @deferred_status=:succeeded,
-       @deferred_timeout=nil,
-       @errbacks=
-        [#<Proc:0x20297e8@C:/Ruby193/lib/ruby/gems/1.9.1/gems/em-http-request-1.0.2/lib/em-http/multi.rb:42>],
-       @error=nil,
-       @headers=nil,
-       @req=
-        #<HttpClientOptions:0x2129ec8
-         @body=nil,
-         @decoding=true,
-         @file=nil,
-         @followed=1,
-         @headers=
-          {"User-Agent"=>
-            "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-         @host="www.google.com.ph",
-         @keepalive=false,
-         @method="HEAD",
-         @pass_cookies=true,
-         @path=nil,
-         @port=80,
-         @query=nil,
-         @redirects=10,
-         @uri=#<Addressable::URI:0xe683ec URI:http://www.google.com.ph:80/>>,
-       @response="",
-       @response_header=
-        {"DATE"=>"Wed, 15 Aug 2012 01:07:42 GMT",
-         "EXPIRES"=>"-1",
-         "CACHE_CONTROL"=>"private, max-age=0",
-         "CONTENT_TYPE"=>"text/html; charset=ISO-8859-1",
-         "SET_COOKIE"=>
-          ["PREF=ID=ec6b270e7fe890fa:FF=0:TM=1344992862:LM=1344992862:S=WlLM9juILblGoi5k; expires=Fri, 15-Aug-2014 01:07:42 GMT; path=/; domain=.google.com.ph",
-           "NID=62=5O_DMdySUEeIJXvveuCr1U8UEfYEhurh0X2Is-a5f0xhTw5CxEY9gELcAyCmwqt4MxVLDpvT2anCV79hhXHfd-QPi0zRY8bCiqh7BlH1B3w0wfE3eg0PTR_KbXUJBBFm; expires=Thu, 14-Feb-2013 01:07:42 GMT; path=/; domain=.google.com.ph; HttpOnly"],
-         "P3P"=>
-          "CP=\"This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info.\"",
-         "SERVER"=>"gws",
-         "X_XSS_PROTECTION"=>"1; mode=block",
-         "X_FRAME_OPTIONS"=>"SAMEORIGIN",
-         "CONNECTION"=>"close"},
-       @state=:finished,
-       @stream=nil>},
-   :errback=>
-    {0=>
-      #<EventMachine::HttpClient:0x21adc28
-       @callbacks=[],
-       @conn=
-        #<EventMachine::HttpConnection:0x21f3f30
-         @connopts=
-          #<HttpConnectionOptions:0x2231100
-           @connect_timeout=5,
-           @host="locahlost2",
-           @inactivity_timeout=10,
-           @port=80,
-           @proxy=nil,
-           @tls={}>,
-         @deferred=true,
-         @middleware=[],
-         @uri="http://locahlost2/foo">,
-       @content_charset=nil,
-       @content_decoder=nil,
-       @cookiejar=
-        #<EventMachine::HttpClient::CookieJar:0x21adc70
-         @jar=#<CookieJar::Jar:0x21adcb8 @domains={}>>,
-       @cookies=[],
-       @deferred_args=[#<EventMachine::HttpClient:0x21adc28 ...>],
-       @deferred_status=:failed,
-       @deferred_timeout=nil,
-       @errbacks=[],
-       @error="unable to resolve server address",
-       @headers=nil,
-       @req=
-        #<HttpClientOptions:0x21f42f0
-         @body=nil,
-         @decoding=true,
-         @file=nil,
-         @followed=0,
-         @headers=
-          {"User-Agent"=>
-            "Scrapey v0.0.5 - https://github.com/monkeysuffrage/scrapey"},
-         @host="locahlost2",
-         @keepalive=false,
-         @method="HEAD",
-         @pass_cookies=true,
-         @path=nil,
-         @port=80,
-         @query=nil,
-         @redirects=10,
-         @uri=#<Addressable::URI:0x10e696c URI:http://locahlost2:80/foo>>,
-       @response="",
-       @response_header={},
-       @state=:response_header,
-       @stream=nil>}}>