pioneer 0.0.1 → 0.0.2

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 pkg/*
 tmp/*
 *.log
+test.rb

data/.travis.yml

@@ -0,0 +1,2 @@
+rvm:
+  - 1.9.2

data/CHANGELOG

@@ -1,3 +1,7 @@
 ## v0.0.1
 
-* Initial release
+* Initial release
+
+## v0.0.2
+
+* added some options and logging

data/Gemfile

@@ -1,4 +1,6 @@
 source "http://rubygems.org"
 
+gem "rake"
+
 # Specify your gem's dependencies in pioneer.gemspec
-gemspec
+gemspec

data/README.md

@@ -1,3 +1,115 @@
 # Pioneer
 
-Pioneer is asynchronous crawler over em-synchrony.
+Pioneer is a simple async HTTP crawler based on em-synchrony
+
+# Install
+
+```bash
+gem install pioneer
+```
+
+# Usage
+
+To use `Pioneer` you should specify a class with two methods: `locations` and `processing(req)`.
+
+First one should return enumerable object and second will accept request object.
+
+```ruby
+class Crawler << Pioneer::Base
+  def locations
+    ["http://www.amazon.com", "http://www.apple.com"]
+  end
+
+  def processing(req)
+    File.open(req.url, "w+") do |f|
+      f << req.response.response
+    end
+  end
+end
+
+Crawler.new.start
+```
+
+In this example we are saving two files with html of those two sites.
+
+`start` method will start iterating over urls and return an Array of what `processing` method returns.
+
+# Handling request, response errors and statuses
+
+In case of request or response error `Pioneer` will raise an error. Or we can catch them this way:
+
+```ruby
+class Crawler << Pioneer::Base
+  def locations
+    ["http://www.amazon.com", "http://www.apple.com"]
+  end
+
+  def processing(req)
+    File.open(req.url, "w+") do |f|
+      f << req.response.response
+    end
+  end
+
+  def if_request_error(req)
+    puts "Request error: #{req.error}"
+  end
+
+  def if_response_error(req)
+    puts "Response error: #{req.response.error}"
+  end
+
+  def if_status_203(req)
+    puts "He is trying to redirect me"
+  end
+end
+```
+
+also you can write `if_status_not_200` to handle all statuses not 200, or `if_status_XXX` for any status you want.
+
+# Overriding behavior
+
+You can override all methods on the fly:
+
+```ruby
+crawler = Pioneer::Crawler.new # base simple crawler
+crawler.locations = [url1, url2]
+crawler.processing = proc{ req.response.response_header.status }
+crawler.if_status_404{ |req| "Oups" }
+```
+
+As far as `locations` should return Enumerable you can use nested crawlers to save whole site
+
+```ruby
+require 'pioneer'
+require 'nokogiri'
+class Links
+  include Enumerable
+  def initialize(link)
+    @links = [link]
+  end
+
+  def <<(link)
+    @links << link
+  end
+
+  def each
+    @links.each{ |url| url }
+  end
+end
+
+class LinksCrawler < Pioneer::Base
+  def locations
+    @links = Links.new("http://www.gazeta.ru")
+  end
+
+  def processing(req)
+    doc = Nokogiri::HTML.parse(req.response.response)
+    links = doc.css("a").map{|link| link["href"]} # + some logic to filter links to prevent duplications and another hosts etc
+    @links << links
+    File.new(req.url, "w+"){ |f| f << req.response.response }
+  end
+end
+LinksCrawler.new(concurrency: 20, redirects: 1, sleep: 0.5).start
+```
+
+... to be continued

data/lib/pioneer/request.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 module Pioneer
   class Request
-    attr_reader :pioneer, :url, :result, :response
+    attr_reader :pioneer, :url, :result, :response, :error
     def initialize(url, pioneer)
       @url, @pioneer = url, pioneer
       @url = begin
@@ -20,12 +20,12 @@ module Pioneer
       begin
         @response = EventMachine::HttpRequest.new(url).get(pioneer.http_opts)
       rescue => e
-        error = "Request totaly failed. Url: #{url}, error: #{e.message}"
+        @error = "Request totaly failed. Url: #{url}, error: #{e.message}"
         pioneer.logger.fatal(error)
         if pioneer.respond_to? :if_request_error
           return pioneer.send(:if_request_error, self)
         else
-          raise HttpRequestError, error
+          raise HttpRequestError, @error
         end
       end
       handle_response_error_or_return_result

data/lib/pioneer/version.rb

@@ -1,3 +1,3 @@
 module Pioneer
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/pioneer/request_spec.rb

@@ -51,4 +51,16 @@ describe Pioneer::Request do
     @lastfm_pioneer = LastfmCrawler.new(sleep: 0.25)
     @lastfm_pioneer.start.sort.must_equal LastfmEnum.const_get(:ARTISTS).sort
   end
+
+  it "should use headers" do
+    @crawler1 = KinopoiskCrawler.new(random_header: false)
+    @crawler2 = KinopoiskCrawler.new(random_header: false, redirects: 1)
+    @crawler3 = KinopoiskCrawler.new(random_header: true)
+    # this one will redirect
+    @crawler1.start.must_equal [nil]
+    # this one will return some restrictions (it need real headres)
+    (@crawler2.start.first < 10000).must_equal true
+    # and this one will fire up
+    (@crawler3.start.first > 10000).must_equal true
+  end
 end

data/spec/spec_helper.rb

@@ -22,7 +22,6 @@ class LastfmEnum
 
   def each
     ARTISTS.each do |artist|
-      p artist
       url = "http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist=#{artist}&api_key=b25b959554ed76058ac220b7b2e0a026&format=json"
       yield url
     end
@@ -38,4 +37,15 @@ class LastfmCrawler < Pioneer::Base
     json = Yajl::Parser.parse(req.response.response)
     json["similarartists"]["@attr"]["artist"]
   end
+end
+
+# Kinopoisk
+class KinopoiskCrawler < Pioneer::Base
+  def locations
+    ["http://www.kinopoisk.ru/level/1/film/614667/"]
+  end
+
+  def processing(req)
+    req.response.response.size
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pioneer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-21 00:00:00.000000000Z
+date: 2012-02-22 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &73645260 !ruby/object:Gem::Requirement
+  requirement: &76399260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *73645260
+  version_requirements: *76399260
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &73644990 !ruby/object:Gem::Requirement
+  requirement: &76399050 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73644990
+  version_requirements: *76399050
 description: Simple async HTTP crawler based on em-synchrony
 email:
 - pedro.yanoviches@gmail.com
@@ -41,6 +41,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG
 - Gemfile
 - LICENSE