pioneer 0.0.6 → 0.0.7

data/CHANGELOG

@@ -23,4 +23,9 @@
 
 * Added skip functionality
 * Added some docs
-* Added "response_body" and "response_header" shortcuts
+* Added "response_body" and "response_header" shortcuts
+
+## v0.0.7
+
+* Rescuing of Retry Exception is removed to Base class, so it will be triggered in context of main loop (it will executed with global sleep timeout)
+* Added `headers` callback support

data/lib/pioneer/base.rb

@@ -16,7 +16,7 @@ module Pioneer
     def initialize(opts = {})
       raise UndefinedLocations, "you should specify `locations` method in your `self.class`" unless self.methods.include? :locations
       raise UndefinedProcessing, "you should specify `processing` method in your `self.class`" unless self.methods.include? :processing
-      raise LocationsNotEnumerator, "you should specify `locations` to return Enumerator" unless self.locations.methods.include? :each
+      # raise LocationsNotEnumerator, "you should specify `locations` to return Enumerator" unless self.locations.methods.include? :each
       @name          = opts[:name]          || "crawler"
       @concurrency   = opts[:concurrency]   || 10
       @sleep         = opts[:sleep]         || 0 # sleep is reversed RPS (1/RPS) - frequency of requests.
@@ -25,18 +25,24 @@ module Pioneer
       @random_header = opts[:random_header] || false
       @header        = opts[:header]        || nil
       @redirects     = opts[:redirects]     || nil
+      @headers       = opts[:headers]      #|| nil
     end
 
+    #
+    # Main method: starting crawling through locations
+    #
     def start
-      raise LocationsNotEnumerable, "location should respond to `each`" unless locations.respond_to? :each
       result = []
       EM.synchrony do
-        # Using FiberPeriodicTimerIterator that implements RPS (request per second feature)
-        # In case @sleep is 0 it behaves like standart FiberIterator
         EM::Synchrony::FiberIterator.new(locations, concurrency).map do |url|
-          sleep
+          counter = 0
           begin
-            result << Request.new(url, self).perform
+            sleep
+            result << Request.new(url, self, counter).perform
+          rescue Pioneer::HttpRetryRequest => e
+            # return to our loop
+            counter += 1
+            retry
           rescue Pioneer::HttpSkipRequest => e
             nil # do nothing?
           end
@@ -46,7 +52,9 @@ module Pioneer
       result
     end
 
+    #
     # Sleep if the last request was recently (less then timout period)
+    #
     def sleep
       @next_start ||= Time.now
       if @sleep > 0
@@ -58,6 +66,9 @@ module Pioneer
       end
     end
 
+    #
+    # Default Pioneer logger
+    #
     def logger
       @logger ||= begin
         logger = Logger.new(STDOUT)
@@ -66,6 +77,9 @@ module Pioneer
       end
     end
 
+    #
+    # Set headers, such as redirects, cookies etc
+    #
     def http_opts
       opts = {}
       opts[:head] = random_header if @random_header
@@ -74,11 +88,23 @@ module Pioneer
       opts
     end
 
+    #
+    # Generate random header for request
+    #
     def random_header
       HttpHeader.random
     end
 
+    #
+    # Headers callback
+    #
+    def headers
+      @headers
+    end
+
+    #
     # we should override only our methods: locations, processing, if_XXX
+    #
     def method_missing(method_name, *args, &block)
       case method_name
       when /locations.*=|processing.*=|if_.+=/
@@ -89,6 +115,9 @@ module Pioneer
       end
     end
 
+    # 
+    # Overriding methods as singeltons so they are availible only for current instance of crawler
+    #
     def override_method(method_name, arg)
       if Proc === arg
         self.define_singleton_method method_name do |req|

data/lib/pioneer/request.rb

@@ -3,10 +3,10 @@ module Pioneer
   class Request
     attr_reader :pioneer, :url, :result, :response, :error, :counter
     
-    def initialize(url, pioneer)
+    def initialize(url, pioneer, counter=0)
       @pioneer = pioneer
       @url     = parse_url(url)
-      @counter = 0
+      @counter = counter
     end
 
     #
@@ -24,7 +24,13 @@ module Pioneer
     #
     def handle_request_error_or_return_result
       begin
-        @response = EventMachine::HttpRequest.new(url).get(pioneer.http_opts)
+        req = EM::HttpRequest.new(url).aget pioneer.http_opts
+        if pioneer.headers
+          req.headers{
+            pioneer.headers.call(req)
+          }
+        end
+        @response = EM::Synchrony.sync req
       rescue => e
         @error = "Request totaly failed. Url: #{url}, error: #{e.message}"
         pioneer.logger.fatal(@error)
@@ -35,8 +41,6 @@ module Pioneer
         end
       end
       handle_response_error_or_return_result
-    rescue Pioneer::HttpRetryRequest => e
-      retry
     end
 
     #
@@ -87,8 +91,7 @@ module Pioneer
     #
     def retry(count=nil)
       if count
-        @counter += 1
-        skip if @counter > count
+        skip if @counter >= count
       end
       raise Pioneer::HttpRetryRequest
     end
@@ -105,7 +108,7 @@ module Pioneer
     # We should parse url befor sending request
     # We use URI.escape for escaping
     # IMPORTAINT: We should replace ampersand (&) in params with "&" !!!
-    # Pluses (+) weill be replaced with "%2B"
+    # Pluses (+) will be replaced with "%2B"
     #
     def parse_url(url)
       url = "http://" + url unless url =~ /http/

data/lib/pioneer/version.rb

@@ -1,3 +1,3 @@
 module Pioneer
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pioneer
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-13 00:00:00.000000000Z
+date: 2012-03-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &76352180 !ruby/object:Gem::Requirement
+  requirement: &77891380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *76352180
+  version_requirements: *77891380
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &76351940 !ruby/object:Gem::Requirement
+  requirement: &77891160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *76351940
+  version_requirements: *77891160
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &76351700 !ruby/object:Gem::Requirement
+  requirement: &77890920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *76351700
+  version_requirements: *77890920
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &76351430 !ruby/object:Gem::Requirement
+  requirement: &77890690 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *76351430
+  version_requirements: *77890690
 description: Simple async HTTP crawler based on em-synchrony
 email:
 - pedro.yanoviches@gmail.com