pioneer 0.0.1.alpha → 0.0.1

data/.gitignore

@@ -4,4 +4,3 @@ Gemfile.lock
 pkg/*
 tmp/*
 *.log
-test.rb

data/CHANGELOG

@@ -1,11 +1,3 @@
 ## v0.0.1
 
-* Initial release
-
-## v0.0.2
-
-* added some options and logging
-
-## v0.0.1alpha
-
-* Heh, but it is still in dev. So let's call it alpha
+* Initial release

data/Gemfile

@@ -1,6 +1,4 @@
 source "http://rubygems.org"
 
-gem "rake"
-
 # Specify your gem's dependencies in pioneer.gemspec
-gemspec
+gemspec

data/README.md

@@ -1,83 +1,3 @@
 # Pioneer
 
-Pioneer is a simple async HTTP crawler based on em-synchrony
-
-And it is very alpha right now. 
-
-# Install
-
-```bash
-gem install pioneer
-```
-
-# Usage
-
-To use `Pioneer` you should specify a class with two methods: `locations` and `processing(req)`.
-
-First one should return enumerable object and second will accept request object.
-
-```ruby
-class Crawler << Pioneer::Base
-  def locations
-    ["http://www.amazon.com", "http://www.apple.com"]
-  end
-
-  def processing(req)
-    File.open(req.url, "w+") do |f|
-      f << req.response.response
-    end
-  end
-end
-
-Crawler.new.start
-```
-
-In this example we are saving two files with html of those two sites.
-
-`start` method will start iterating over urls and return an Array of what `processing` method returns.
-
-# Handling request, response errors and statuses
-
-In case of request or response error `Pioneer` will raise an error. Or we can catch them this way:
-
-```ruby
-class Crawler << Pioneer::Base
-  def locations
-    ["http://www.amazon.com", "http://www.apple.com"]
-  end
-
-  def processing(req)
-    File.open(req.url, "w+") do |f|
-      f << req.response.response
-    end
-  end
-
-  def if_request_error(req)
-    puts "Request error: #{req.error}"
-  end
-
-  def if_response_error(req)
-    puts "Response error: #{req.response.error}"
-  end
-
-  def if_status_203(req)
-    puts "He is trying to redirect me"
-  end
-end
-```
-
-also you can write `if_status_not_200` to handle all statuses not 200, or `if_status_XXX` for any status you want.
-
-# Overriding behavior
-
-You can override all methods on the fly:
-
-```ruby
-crawler = Pioneer::Crawler.new # base simple crawler
-crawler.locations = [url1, url2]
-crawler.processing = proc{ req.response.response_header.status }
-crawler.if_status_404{ |req| "Oups" }
-```
-
-
-... to be continued
+Pioneer is asynchronous crawler over em-synchrony.

data/lib/pioneer/request.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 module Pioneer
   class Request
-    attr_reader :pioneer, :url, :result, :response, :error
+    attr_reader :pioneer, :url, :result, :response
     def initialize(url, pioneer)
       @url, @pioneer = url, pioneer
       @url = begin
@@ -20,12 +20,12 @@ module Pioneer
       begin
         @response = EventMachine::HttpRequest.new(url).get(pioneer.http_opts)
       rescue => e
-        @error = "Request totaly failed. Url: #{url}, error: #{e.message}"
+        error = "Request totaly failed. Url: #{url}, error: #{e.message}"
         pioneer.logger.fatal(error)
         if pioneer.respond_to? :if_request_error
           return pioneer.send(:if_request_error, self)
         else
-          raise HttpRequestError, @error
+          raise HttpRequestError, error
         end
       end
       handle_response_error_or_return_result

data/lib/pioneer/version.rb

@@ -1,3 +1,3 @@
 module Pioneer
-  VERSION = "0.0.1.alpha"
+  VERSION = "0.0.1"
 end

data/spec/pioneer/request_spec.rb

@@ -51,16 +51,4 @@ describe Pioneer::Request do
     @lastfm_pioneer = LastfmCrawler.new(sleep: 0.25)
     @lastfm_pioneer.start.sort.must_equal LastfmEnum.const_get(:ARTISTS).sort
   end
-
-  it "should use headers" do
-    @crawler1 = KinopoiskCrawler.new(random_header: false)
-    @crawler2 = KinopoiskCrawler.new(random_header: false, redirects: 1)
-    @crawler3 = KinopoiskCrawler.new(random_header: true)
-    # this one will redirect
-    @crawler1.start.must_equal [nil]
-    # this one will return some restrictions (it need real headres)
-    (@crawler2.start.first < 10000).must_equal true
-    # and this one will fire up
-    (@crawler3.start.first > 10000).must_equal true
-  end
 end

data/spec/spec_helper.rb

@@ -22,6 +22,7 @@ class LastfmEnum
 
   def each
     ARTISTS.each do |artist|
+      p artist
       url = "http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist=#{artist}&api_key=b25b959554ed76058ac220b7b2e0a026&format=json"
       yield url
     end
@@ -37,15 +38,4 @@ class LastfmCrawler < Pioneer::Base
     json = Yajl::Parser.parse(req.response.response)
     json["similarartists"]["@attr"]["artist"]
   end
-end
-
-# Kinopoisk
-class KinopoiskCrawler < Pioneer::Base
-  def locations
-    ["http://www.kinopoisk.ru/level/1/film/614667/"]
-  end
-
-  def processing(req)
-    req.response.response.size
-  end
 end

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: pioneer
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha
-  prerelease: 6
+  version: 0.0.1
+  prerelease: 
 platform: ruby
 authors:
 - Petr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-22 00:00:00.000000000Z
+date: 2012-02-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &74894120 !ruby/object:Gem::Requirement
+  requirement: &73645260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *74894120
+  version_requirements: *73645260
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &74893910 !ruby/object:Gem::Requirement
+  requirement: &73644990 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *74893910
+  version_requirements: *73644990
 description: Simple async HTTP crawler based on em-synchrony
 email:
 - pedro.yanoviches@gmail.com
@@ -41,7 +41,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .travis.yml
 - CHANGELOG
 - Gemfile
 - LICENSE
@@ -75,9 +74,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: pioneer
 rubygems_version: 1.8.15

data/.travis.yml

@@ -1,2 +0,0 @@
-rvm:
-  - 1.9.2