pioneer 0.0.4 → 0.0.6

data/CHANGELOG

@@ -13,4 +13,14 @@
 ## v0.0.4
 
 * Fixed stackoverflow on retrying request after failing
-* added `req.retry` method to retry http request in error handlers
+* added `req.retry` method to retry http request in error handlers
+
+## v0.0.5
+
+* Added some minor fixes
+
+## v0.0.6
+
+* Added skip functionality
+* Added some docs
+* Added "response_body" and "response_header" shortcuts

data/lib/pioneer.rb

@@ -4,7 +4,6 @@ require "em-synchrony/em-http"
 require "em-synchrony/fiber_iterator"
 # patch - to remove! maybe pull to em-synchrony?
 require "patch/iterator"
-require "patch/fiber_periodic_timer_iterator"
 # other
 require "logger"
 require 'uri'

data/lib/pioneer/base.rb

@@ -1,13 +1,14 @@
 # encoding: utf-8
 module Pioneer
-  class UndefinedLocations < RuntimeError; end
-  class LocationsNotEnumerable < RuntimeError; end
-  class UndefinedProcessing < RuntimeError; end
-  class LocationsNotEnumerator < RuntimeError; end
-  class HttpRequestError < RuntimeError; end
-  class HttpResponseError < RuntimeError; end
-  class HttpStatusError < RuntimeError; end
-  class HttpRetryRequest < RuntimeError; end
+  class UndefinedLocations < StandardError; end
+  class LocationsNotEnumerable < StandardError; end
+  class UndefinedProcessing < StandardError; end
+  class LocationsNotEnumerator < StandardError; end
+  class HttpRequestError < StandardError; end
+  class HttpResponseError < StandardError; end
+  class HttpStatusError < StandardError; end
+  class HttpRetryRequest < StandardError; end
+  class HttpSkipRequest < StandardError; end
 
   class Base
     attr_reader :name, :concurrency, :sleep, :log_level, :redirect
@@ -32,14 +33,31 @@ module Pioneer
       EM.synchrony do
         # Using FiberPeriodicTimerIterator that implements RPS (request per second feature)
         # In case @sleep is 0 it behaves like standart FiberIterator
-        EM::Synchrony::FiberPeriodicTimerIterator.new(locations, concurrency, sleep).map do |url|
-          result << Request.new(url, self).perform
+        EM::Synchrony::FiberIterator.new(locations, concurrency).map do |url|
+          sleep
+          begin
+            result << Request.new(url, self).perform
+          rescue Pioneer::HttpSkipRequest => e
+            nil # do nothing?
+          end
         end
         EM.stop
       end
       result
     end
 
+    # Sleep if the last request was recently (less then timout period)
+    def sleep
+      @next_start ||= Time.now
+      if @sleep > 0
+        now = Time.now
+        sleep_time = @next_start - Time.now
+        sleep_time = 0 if sleep_time < 0
+        @next_start = Time.now + sleep_time + @sleep
+        EM::Synchrony.sleep(sleep_time) if sleep_time > 0
+      end
+    end
+
     def logger
       @logger ||= begin
         logger = Logger.new(STDOUT)

data/lib/pioneer/http_header.rb

@@ -5,10 +5,11 @@ module Pioneer
     def random
       header = headers.sample
       headers = {
-        'Referer' => 'http://www.google.com/',
+        'Referer' => 'http://www.google.ru/#hl=ru&newwindow=1&sa=X&ei=6oPXTp3OB4Tl4QTdl-zyDQ&ved=0CCAQvwUoAQ&q=kinopoisk&spell=1&bav=on.2,or.r_gc.r_pw.r_cp.,cf.osb&fp=c8a5ca24098b41f5&biw=1127&bih=628',
         'User-Agent' => header,
         'Accept'  => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Connection'  => 'keep-alive'
+        'Connection'  => 'keep-alive',
+        'Cache-Control' => 'no-cache'
       }  
     end
 

data/lib/pioneer/request.rb

@@ -1,59 +1,76 @@
 # encoding: utf-8
 module Pioneer
   class Request
-    attr_reader :pioneer, :url, :result, :response, :error
+    attr_reader :pioneer, :url, :result, :response, :error, :counter
+    
     def initialize(url, pioneer)
-      @url, @pioneer = url, pioneer
-      @url = begin
-        url = "http://" + url unless url =~ /http/
-        URI.escape(url)
-      end
+      @pioneer = pioneer
+      @url     = parse_url(url)
+      @counter = 0
     end
 
+    #
+    # Request processing
+    #
     def perform
       pioneer.logger.info("going to #{url}")
       @result = handle_request_error_or_return_result
     end
 
+    #
     # Handle base fatal request error
+    # If we have got connection error or whatever
+    #   we will fire either Exception or call "if_request_error" if exists
+    #
     def handle_request_error_or_return_result
       begin
         @response = EventMachine::HttpRequest.new(url).get(pioneer.http_opts)
       rescue => e
         @error = "Request totaly failed. Url: #{url}, error: #{e.message}"
-        pioneer.logger.fatal(error)
+        pioneer.logger.fatal(@error)
         if pioneer.respond_to? :if_request_error
           return pioneer.if_request_error(self)
         else
-          raise HttpRequestError, @error
+          raise Pioneer::HttpRequestError, @error
         end
       end
       handle_response_error_or_return_result
-    rescue HttpRetryRequest => e
+    rescue Pioneer::HttpRetryRequest => e
       retry
     end
 
-    # handle http error
+    #
+    # Handle http error
+    # If we can't make proper response we will ether fire Exception
+    #   or call "if_response_error" if exists
+    #
     def handle_response_error_or_return_result
       if response.error
-        error = "Response for #{url} get an error: #{response.error}"
-        pioneer.logger.error(error)
+        @error = "Response for #{url} get an error: #{response.error}"
+        pioneer.logger.error(@error)
         if pioneer.respond_to? :if_response_error
           return pioneer.if_response_error(self)
         else
-          raise HttpResponseError, error
+          raise Pioneer::HttpResponseError, error
         end
       end
       handle_status_or_return_result
     end
 
+    # 
+    # Handle wrong status or run "processing"
+    # If status is not 200 we will either do nothing (?)
+    #   or call "if_status_XXX" if exist
+    #   or "if_status_not_200"
+    #
     def handle_status_or_return_result
       status = response.response_header.status
       case status
       when 200
         pioneer.processing(self)
       else
-        pioneer.logger.error("This #{url} returns this http status: #{status}")
+        @error = "This #{url} returns this http status: #{status}"
+        pioneer.logger.error(@error)
         if pioneer.respond_to? "if_status_#{status}".to_sym
           pioneer.send("if_status_#{status}", self)
         elsif pioneer.respond_to? :if_status_not_200
@@ -64,8 +81,54 @@ module Pioneer
       end
     end
 
-    def retry
-      raise HttpRetryRequest
+    #
+    # We can call retry from crawler like "req.retry"
+    # If count is seted, so it will retry it not more then "count" times
+    #
+    def retry(count=nil)
+      if count
+        @counter += 1
+        skip if @counter > count
+      end
+      raise Pioneer::HttpRetryRequest
+    end
+
+    #
+    # We can skip request from crawler like "req.skip"
+    # I.E. if response_body is blank or 404 error
+    #
+    def skip
+      raise Pioneer::HttpSkipRequest
+    end
+
+    #
+    # We should parse url befor sending request
+    # We use URI.escape for escaping
+    # IMPORTAINT: We should replace ampersand (&) in params with "&" !!!
+    # Pluses (+) weill be replaced with "%2B"
+    #
+    def parse_url(url)
+      url = "http://" + url unless url =~ /http/
+      url = URI.escape(url)
+      # replace "&" ampersands :)
+      url = url.gsub("&", "%26")
+      # replace pluses
+      url = url.gsub("+", "%2B")
+      url
+    end
+
+    #
+    # Shortcut for response.response
+    #
+    def response_body
+      response.response
+    end
+
+    #
+    # Shortcut for response.response_header
+    #
+    def response_header
+      response.response_header
     end
   end
 end

data/lib/pioneer/version.rb

@@ -1,3 +1,3 @@
 module Pioneer
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

data/pioneer.gemspec

@@ -20,5 +20,7 @@ Gem::Specification.new do |s|
 
   # specify any dependencies here; for example:
   s.add_development_dependency "yajl-ruby"
+  s.add_development_dependency "nokogiri"
   s.add_runtime_dependency "em-synchrony"
+  s.add_runtime_dependency "em-http-request"
 end

data/spec/pioneer/request_spec.rb

@@ -18,7 +18,7 @@ describe Pioneer::Request do
   end
 
   it "should redefine methods" do
-    processing = proc{ |req| req.response.response_header.status + 1 }
+    processing = proc{ |req| req.response_header.status + 1 }
     @pioneer2.processing = processing
     @pioneer2.locations = ["www.apple.com", "www.amazon.com"]
     @pioneer2.start.must_equal [201, 201]
@@ -63,4 +63,26 @@ describe Pioneer::Request do
     # and this one will fire up
     (@crawler3.start.first > 10000).must_equal true
   end
+
+  it "should skip url" do
+    @result = []
+    crawler = Pioneer::Crawler.new(redirects: 1)
+    crawler.locations = ["http://not.exist.page.com", "http://amazon.com"]
+    crawler.processing = proc{ |req| @result << req.url }
+    crawler.if_response_error = proc{ |req| req.skip }
+    crawler.start
+    @result.must_equal ["http://amazon.com"]
+  end
+
+  it "should retry 2 times and skip" do
+    @result = []
+    @retries = nil
+    crawler = Pioneer::Crawler.new(redirects: 1)
+    crawler.locations = ["http://not.exist.page.com", "http://amazon.com"]
+    crawler.processing = proc{ |req| @result << req.url }
+    crawler.if_response_error = proc{ |req| @retries = req.counter; req.retry(2); }
+    crawler.start
+    @result.must_equal ["http://amazon.com"]
+    @retries.must_equal 2
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pioneer
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-29 00:00:00.000000000Z
+date: 2012-03-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &73303650 !ruby/object:Gem::Requirement
+  requirement: &76352180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,32 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *73303650
+  version_requirements: *76352180
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &76351940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *76351940
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &73303440 !ruby/object:Gem::Requirement
+  requirement: &76351700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *76351700
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &76351430 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73303440
+  version_requirements: *76351430
 description: Simple async HTTP crawler based on em-synchrony
 email:
 - pedro.yanoviches@gmail.com