pioneer 0.0.7 → 0.0.9

data/CHANGELOG

@@ -28,4 +28,12 @@
 ## v0.0.7
 
 * Rescuing of Retry Exception is removed to Base class, so it will be triggered in context of main loop (it will executed with global sleep timeout)
-* Added `headers` callback support
+* Added `headers` callback support
+
+## v0.0.8
+
+* em-http-request options support added
+
+## v0.0.9
+
+* pushing request_opt to init

data/lib/pioneer/base.rb

@@ -11,7 +11,7 @@ module Pioneer
   class HttpSkipRequest < StandardError; end
 
   class Base
-    attr_reader :name, :concurrency, :sleep, :log_level, :redirect
+    attr_reader :name, :concurrency, :sleep, :log_level, :redirect, :request_opts
 
     def initialize(opts = {})
       raise UndefinedLocations, "you should specify `locations` method in your `self.class`" unless self.methods.include? :locations
@@ -26,10 +26,13 @@ module Pioneer
       @header        = opts[:header]        || nil
       @redirects     = opts[:redirects]     || nil
       @headers       = opts[:headers]      #|| nil
+      @request_opts  = opts[:request_opts] #|| nil
     end
 
     #
     # Main method: starting crawling through locations
+    # If we catch Pioneer::HttpRetryRequest then we are retrying request
+    # And if we catch Pioneer::HttpSkipRequest we just return nothing?
     #
     def start
       result = []
@@ -88,6 +91,20 @@ module Pioneer
       opts
     end
 
+    #
+    # EmHttpRequest options
+    #
+    def request_opts
+      opts = {}
+      opts = case @request_opts
+      when Proc
+        @request_opts.call
+      else
+        @request_opts
+      end if @request_opts
+      opts
+    end
+
     #
     # Generate random header for request
     #

data/lib/pioneer/request.rb

@@ -1,12 +1,13 @@
 # encoding: utf-8
 module Pioneer
   class Request
-    attr_reader :pioneer, :url, :result, :response, :error, :counter
+    attr_reader :pioneer, :url, :result, :response, :error, :counter, :request_opts
     
     def initialize(url, pioneer, counter=0)
       @pioneer = pioneer
       @url     = parse_url(url)
       @counter = counter
+      @request_opts = @pioneer.request_opts
     end
 
     #
@@ -24,7 +25,7 @@ module Pioneer
     #
     def handle_request_error_or_return_result
       begin
-        req = EM::HttpRequest.new(url).aget pioneer.http_opts
+        req = EM::HttpRequest.new(url, @request_opts).aget pioneer.http_opts
         if pioneer.headers
           req.headers{
             pioneer.headers.call(req)

data/lib/pioneer/version.rb

@@ -1,3 +1,3 @@
 module Pioneer
-  VERSION = "0.0.7"
+  VERSION = "0.0.9"
 end

data/spec/pioneer/base_spec.rb

@@ -2,4 +2,26 @@
 require 'spec_helper'
 
 describe Pioneer::Base do
+  before do
+  end
+
+  it "should pass request_opts as a proc" do
+    request_opts = proc do
+      {
+        bind: {host: "192.168.1.1", port: '0'}
+      }
+    end
+    @pioneer = Pioneer::Crawler.new(request_opts: request_opts)
+    @pioneer.request_opts[:bind][:host].must_equal "192.168.1.1"
+  end
+
+  it "should pass request_opts as a Hash" do
+    request_opts = begin
+      {
+        bind: {host: "192.168.1.1", port: '0'}
+      }
+    end
+    @pioneer = Pioneer::Crawler.new(request_opts: request_opts)
+    @pioneer.request_opts[:bind][:host].must_equal "192.168.1.1"
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pioneer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-21 00:00:00.000000000Z
+date: 2012-04-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &77891380 !ruby/object:Gem::Requirement
+  requirement: &76605720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *77891380
+  version_requirements: *76605720
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &77891160 !ruby/object:Gem::Requirement
+  requirement: &76605500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *77891160
+  version_requirements: *76605500
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &77890920 !ruby/object:Gem::Requirement
+  requirement: &76605290 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *77890920
+  version_requirements: *76605290
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &77890690 !ruby/object:Gem::Requirement
+  requirement: &76605010 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *77890690
+  version_requirements: *76605010
 description: Simple async HTTP crawler based on em-synchrony
 email:
 - pedro.yanoviches@gmail.com