pioneer 0.0.1.alpha

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+tmp/*
+*.log
+test.rb

data/.travis.yml

@@ -0,0 +1,2 @@
+rvm:
+  - 1.9.2

data/CHANGELOG

@@ -0,0 +1,11 @@
+## v0.0.1
+
+* Initial release
+
+## v0.0.2
+
+* added some options and logging
+
+## v0.0.1alpha
+
+* Heh, but it is still in dev. So let's call it alpha

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gem "rake"
+
+# Specify your gem's dependencies in pioneer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1 @@
+This is the Licence. Isn't it?

data/README.md

@@ -0,0 +1,83 @@
+# Pioneer
+
+Pioneer is a simple async HTTP crawler based on em-synchrony
+
+And it is very alpha right now. 
+
+# Install
+
+```bash
+gem install pioneer
+```
+
+# Usage
+
+To use `Pioneer` you should specify a class with two methods: `locations` and `processing(req)`.
+
+First one should return enumerable object and second will accept request object.
+
+```ruby
+class Crawler << Pioneer::Base
+  def locations
+    ["http://www.amazon.com", "http://www.apple.com"]
+  end
+
+  def processing(req)
+    File.open(req.url, "w+") do |f|
+      f << req.response.response
+    end
+  end
+end
+
+Crawler.new.start
+```
+
+In this example we are saving two files with html of those two sites.
+
+`start` method will start iterating over urls and return an Array of what `processing` method returns.
+
+# Handling request, response errors and statuses
+
+In case of request or response error `Pioneer` will raise an error. Or we can catch them this way:
+
+```ruby
+class Crawler << Pioneer::Base
+  def locations
+    ["http://www.amazon.com", "http://www.apple.com"]
+  end
+
+  def processing(req)
+    File.open(req.url, "w+") do |f|
+      f << req.response.response
+    end
+  end
+
+  def if_request_error(req)
+    puts "Request error: #{req.error}"
+  end
+
+  def if_response_error(req)
+    puts "Response error: #{req.response.error}"
+  end
+
+  def if_status_203(req)
+    puts "He is trying to redirect me"
+  end
+end
+```
+
+also you can write `if_status_not_200` to handle all statuses not 200, or `if_status_XXX` for any status you want.
+
+# Overriding behavior
+
+You can override all methods on the fly:
+
+```ruby
+crawler = Pioneer::Crawler.new # base simple crawler
+crawler.locations = [url1, url2]
+crawler.processing = proc{ req.response.response_header.status }
+crawler.if_status_404{ |req| "Oups" }
+```
+
+
+... to be continued

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+task :default => :spec
+
+desc 'Tests'
+Rake::TestTask.new(:spec) do |t|
+  t.libs << 'spec'
+  t.pattern = 'spec/**/*_spec.rb'
+  t.verbose = false
+end

data/lib/patch/fiber_periodic_timer_iterator.rb

@@ -0,0 +1,39 @@
+module EventMachine
+  module Synchrony
+
+    class FiberPeriodicTimerIterator < EM::Synchrony::Iterator
+
+      # set timeout and start point
+      # each Fiber will be executed not earlier than once per timeout
+      def initialize(list, concurrency=1, timeout=0)
+        @timeout = timeout
+        @next_start = Time.now
+        super list, concurrency
+      end
+
+      # execute each iterator block within its own fiber at particular time offset
+      # and auto-advance the iterator after each call
+      def each(foreach=nil, after=nil, &blk)
+        fe = Proc.new do |obj, iter|
+          Fiber.new do
+            sleep
+            (foreach || blk).call(obj); iter.next
+          end.resume
+        end
+        super(fe, after)
+      end
+
+      # Sleep if the last request was recently (less then timout period)
+      def sleep
+        if @timeout > 0
+          now = Time.now
+          sleep_time = @next_start - Time.now
+          sleep_time = 0 if sleep_time < 0
+          @next_start = Time.now + sleep_time + @timeout
+          EM::Synchrony.sleep(sleep_time) if sleep_time > 0
+        end
+      end
+
+    end
+  end
+end

data/lib/patch/iterator.rb

@@ -0,0 +1,314 @@
+module EventMachine
+  # A simple iterator for concurrent asynchronous work.
+  #
+  # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
+  # instead of happening automatically after the yielded block finishes executing. For example:
+  #
+  #   (0..10).each{ |num| }
+  #
+  # becomes:
+  #
+  #   EM::Iterator.new(0..10).each{ |num,iter| iter.next }
+  #
+  # This is especially useful when doing asynchronous work via reactor libraries and
+  # functions. For example, given a sync and async http api:
+  #
+  #   response = sync_http_get(url); ...
+  #   async_http_get(url){ |response| ... }
+  #
+  # a synchronous iterator such as:
+  #
+  #   responses = urls.map{ |url| sync_http_get(url) }
+  #   ...
+  #   puts 'all done!'
+  #
+  # could be written as:
+  #
+  #   EM::Iterator.new(urls).map(proc{ |url,iter|
+  #     async_http_get(url){ |res|
+  #       iter.return(res)
+  #     }
+  #   }, proc{ |responses|
+  #     ...
+  #     puts 'all done!'
+  #   })
+  #
+  # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
+  # to fetch 10 urls at a time, simply pass in a concurrency of 10:
+  #
+  #   EM::Iterator.new(urls, 10).each do |url,iter|
+  #     async_http_get(url){ iter.next }
+  #   end
+  #
+
+  # Support for Enumerable in Ruby 1.9+
+  module IteratorWithEnumerable
+    def setup_list(list)
+      raise ArgumentError, 'argument must be an Enumerable' unless list.respond_to?(:each)
+      list.to_enum
+    end
+
+    def next_item
+      @next_item
+    end
+
+    # We can't check just next_item as far as it can return nil in two cases: 
+    # when our enumerator is stopped and when it stores nil value
+    def next?
+      begin
+        @next_item = @list.next
+        true
+      rescue StopIteration
+        false
+      rescue => e
+        raise e
+      end
+    end
+  end
+
+  # Ruby 1.8 uses continuations in Enumerable, so we should use Arrays
+  module IteratorWithArray
+    def setup_list(list)
+      raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
+      list.dup.to_a
+    end
+
+    def next_item
+      @list.shift
+    end
+
+    def next?
+      @list.any?
+    end
+  end
+
+  class Iterator
+    include IteratorWithEnumerable if defined? Fiber
+    include IteratorWithArray unless defined? Fiber
+    
+    # Create a new parallel async iterator with specified concurrency.
+    #
+    #   i = EM::Iterator.new(1..100, 10)
+    #
+    # will create an iterator over the range that processes 10 items at a time. Iteration
+    # is started via #each, #map or #inject
+    #
+    def initialize(list, concurrency = 1)
+      @list = setup_list(list)
+      @concurrency = concurrency
+
+      @started = false
+      @ended = false
+    end
+
+    # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
+    # to accomodate the new concurrency level.
+    #
+    def concurrency=(val)
+      old = @concurrency
+      @concurrency = val
+
+      spawn_workers if val > old and @started and !@ended
+    end
+    attr_reader :concurrency
+
+    # Iterate over a set of items using the specified block or proc.
+    #
+    #   EM::Iterator.new(1..100).each do |num, iter|
+    #     puts num
+    #     iter.next
+    #   end
+    #
+    # An optional second proc is invoked after the iteration is complete.
+    #
+    #   EM::Iterator.new(1..100).each(
+    #     proc{ |num,iter| iter.next },
+    #     proc{ puts 'all done' }
+    #   )
+    #
+    def each(foreach=nil, after=nil, &blk)
+      raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
+      raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
+
+      @started = true
+      @pending = 0
+      @workers = 0
+
+      all_done = proc{
+        after.call if after and @ended and @pending == 0
+      }
+
+      @process_next = proc{
+        # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
+        unless @ended or @workers > @concurrency
+          if next?
+            item = next_item
+            @pending += 1
+
+            is_done = false
+            on_done = proc{
+              raise RuntimeError, 'already completed this iteration' if is_done
+              is_done = true
+
+              @pending -= 1
+
+              if @ended
+                all_done.call
+              else
+                EM.next_tick(@process_next)
+              end
+            }
+            class << on_done
+              alias :next :call
+            end
+
+            foreach.call(item, on_done)
+          else
+            @ended = true
+            @workers -= 1
+            all_done.call
+          end
+        else
+          @workers -= 1
+        end
+      }
+
+      spawn_workers
+
+      self
+    end
+
+    # Collect the results of an asynchronous iteration into an array.
+    #
+    #   EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
+    #     EM.system(cmd){ |output,status|
+    #       iter.return(output)
+    #     }
+    #   }, proc{ |results|
+    #     p results
+    #   })
+    #
+    def map(foreach, after)
+      index = 0
+
+      inject([], proc{ |results,item,iter|
+        i = index
+        index += 1
+
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          results[i] = res
+          iter.return(results)
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on a map iterator'
+          end
+        end
+
+        foreach.call(item, on_done)
+      }, proc{ |results|
+        after.call(results)
+      })
+    end
+
+    # Inject the results of an asynchronous iteration onto a given object.
+    #
+    #   EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+    #     EM.system(cmd){ |output,status|
+    #       hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+    #       iter.return(hash)
+    #     }
+    #   }, proc{ |results|
+    #     p results
+    #   })
+    #
+    def inject(obj, foreach, after)
+      each(proc{ |item,iter|
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          obj = res
+          iter.next
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on an inject iterator'
+          end
+        end
+
+        foreach.call(obj, item, on_done)
+      }, proc{
+        after.call(obj)
+      })
+    end
+
+    private
+
+    # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
+    #
+    def spawn_workers
+      EM.next_tick(start_worker = proc{
+        if @workers < @concurrency and !@ended
+          # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
+          @workers += 1
+          @process_next.call
+          EM.next_tick(start_worker)
+        end
+      })
+      nil
+    end
+  end
+end
+
+if __FILE__ == $0
+  $:.unshift File.join(File.dirname(__FILE__), '..')
+  require 'eventmachine'
+
+  # TODO: real tests
+  # TODO: pass in one object instead of two? .each{ |iter| puts iter.current; iter.next }
+  # TODO: support iter.pause/resume/stop/break/continue?
+  # TODO: create some exceptions instead of using RuntimeError
+  # TODO: support proc instead of enumerable? EM::Iterator.new(proc{ return queue.pop })
+
+  EM.run{
+    EM::Iterator.new(1..50).each{ |num,iter| p num; iter.next }
+    EM::Iterator.new([1,2,3], 10).each{ |num,iter| p num; iter.next }
+
+    i = EM::Iterator.new(1..100, 5)
+    i.each(proc{|num,iter|
+      p num.to_s
+      iter.next
+    }, proc{
+      p :done
+    })
+    EM.add_timer(0.03){
+      i.concurrency = 1
+    }
+    EM.add_timer(0.04){
+      i.concurrency = 3
+    }
+
+    EM::Iterator.new(100..150).map(proc{ |num,iter|
+      EM.add_timer(0.01){ iter.return(num) }
+    }, proc{ |results|
+      p results
+    })
+
+    EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+      EM.system(cmd){ |output,status|
+        hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+        iter.return(hash)
+      }
+    }, proc{ |results|
+      p results
+    })
+  }
+end

data/lib/pioneer.rb

@@ -0,0 +1,16 @@
+# Eventmachine
+require "em-synchrony"
+require "em-synchrony/em-http"
+require "em-synchrony/fiber_iterator"
+# patch - to remove! maybe pull to em-synchrony?
+require "patch/iterator"
+require "patch/fiber_periodic_timer_iterator"
+# other
+require "logger"
+require 'uri'
+# Code
+require "pioneer/version"
+require "pioneer/base"
+require "pioneer/request"
+require "pioneer/http_header"
+require "pioneer/crawler"

data/lib/pioneer/base.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+module Pioneer
+  class UndefinedLocations < RuntimeError; end
+  class LocationsNotEnumerable < RuntimeError; end
+  class UndefinedProcessing < RuntimeError; end
+  class LocationsNotEnumerator < RuntimeError; end
+  class HttpRequestError < RuntimeError; end
+  class HttpResponseError < RuntimeError; end
+  class Base
+    attr_reader :name, :concurrency, :sleep, :log_level, :redirect
+
+    def initialize(opts = {})
+      raise UndefinedLocations, "you should specify `locations` method in your `self.class`" unless self.methods.include? :locations
+      raise UndefinedProcessing, "you should specify `processing` method in your `self.class`" unless self.methods.include? :processing
+      raise LocationsNotEnumerator, "you should specify `locations` to return Enumerator" unless self.locations.methods.include? :each
+      @name          = opts[:name]          || "crawler"
+      @concurrency   = opts[:concurrency]   || 10
+      @sleep         = opts[:sleep]         || 0 # sleep is reversed RPS (1/RPS) - frequency of requests.
+      @log_enabled   = opts[:log_enabled]   || true # Logger is enabled by default
+      @log_level     = opts[:log_level]     || Logger::DEBUG
+      @random_header = opts[:random_header] || false
+      @header        = opts[:header]        || nil
+      @redirects     = opts[:redirects]     || nil
+    end
+
+    def start
+      raise LocationsNotEnumerable, "location should respond to `each`" unless locations.respond_to? :each
+      result = []
+      EM.synchrony do
+        # Using FiberPeriodicTimerIterator that implements RPS (request per second feature)
+        # In case @sleep is 0 it behaves like standart FiberIterator
+        EM::Synchrony::FiberPeriodicTimerIterator.new(locations, concurrency, sleep).map do |url|
+          result << Request.new(url, self).perform
+        end
+        EM.stop
+      end
+      result
+    end
+
+    def logger
+      @logger ||= begin
+        logger = @log_enabled ? Logger.new("#{name}.log") : Logger.new("/dev/null")
+        logger.level = log_level
+        logger
+      end
+    end
+
+    def http_opts
+      opts = {}
+      opts[:head] = random_header if @random_header
+      opts[:head] = @header if @header
+      opts[:redirects] = @redirects if @redirects
+      opts
+    end
+
+    def random_header
+      HttpHeader.random
+    end
+
+    # we should override only our methods: locations, processing, if_XXX
+    def method_missing(method_name, *args, &block)
+      case method_name
+      when /locations.*=|processing.*=|if_.+=/
+        method_name = method_name.to_s.gsub("=", "").to_sym
+        override_method(method_name, args.first)
+      else
+        super(method_name, *args, &block)
+      end
+    end
+
+    def override_method(method_name, arg)
+      if Proc === arg
+        self.define_singleton_method method_name do |req|
+          arg.call(req)
+        end
+      else
+        self.define_singleton_method method_name do
+          arg
+        end
+      end
+    end
+  end
+end

data/lib/pioneer/crawler.rb

@@ -0,0 +1,12 @@
+module Pioneer
+  class Crawler < Base
+    # basic crawler :)
+    def locations
+      []
+    end
+
+    def processing(req)
+      nil
+    end
+  end
+end

data/lib/pioneer/http_header.rb

@@ -0,0 +1,273 @@
+module Pioneer
+  module HttpHeader
+    extend self
+
+    def random
+      header = headers.sample
+      headers = {
+        'Referer' => 'http://www.google.com/',
+        'User-Agent' => header,
+        'Accept'  => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Connection'  => 'keep-alive'
+      }  
+    end
+
+    # Get more on http://www.useragentstring.com/pages/useragentstring.php
+    def headers
+      [
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0a2) Gecko/20111101 Firefox/9.0a2',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110613 Firefox/6.0a2',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110612 Firefox/6.0a2',
+        'Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0',
+        'Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0',
+        'Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0 FirePHP/0.6',
+        'Mozilla/5.0 (Windows NT 5.0; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
+        'Mozilla/5.0 (X11; Linux i686 on x86_64; rv:5.0a2) Gecko/20110524 Firefox/5.0a2',
+        'Mozilla/5.0 (Windows NT 6.1; U; ru; rv:5.0.1.6) Gecko/20110501 Firefox/5.0.1 Firefox/5.0.1',
+        'mozilla/3.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/5.0.1',
+        'Mozilla/5.0 (X11; U; Linux i586; de; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (X11; U; Linux amd64; rv:5.0) Gecko/20100101 Firefox/5.0 (Debian)',
+        'Mozilla/5.0 (X11; U; Linux amd64; en-US; rv:5.0) Gecko/20110619 Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux) Gecko Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 FirePHP/0.5',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux x86_64) Gecko Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux ppc; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux AMD64) Gecko Firefox/5.0',
+        'Mozilla/5.0 (X11; FreeBSD amd64; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:5.0) Gecko/20110619 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 6.1.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 5.1; U; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 5.0; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (Windows NT 5.0; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (U; Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:2.2a1pre) Gecko/20110324 Firefox/4.2a1pre',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:2.2a1pre) Gecko/20100101 Firefox/4.2a1pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.2a1pre) Gecko/20110324 Firefox/4.2a1pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.2a1pre) Gecko/20110323 Firefox/4.2a1pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.2a1pre) Gecko/20110208 Firefox/4.2a1pre',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:2.0b9pre) Gecko/20110111 Firefox/4.0b9pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b9pre) Gecko/20101228 Firefox/4.0b9pre',
+        'Mozilla/5.0 (Windows NT 5.1; rv:2.0b9pre) Gecko/20110105 Firefox/4.0b9pre',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b8pre) Gecko/20101114 Firefox/4.0b8pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b8pre) Gecko/20101213 Firefox/4.0b8pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b8pre) Gecko/20101128 Firefox/4.0b8pre',
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b8pre) Gecko/20101114 Firefox/4.0b8pre',
+        'Mozilla/5.0 (Windows NT 5.1; rv:2.0b8pre) Gecko/20101127 Firefox/4.0b8pre',
+        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
+        'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.6 (KHTML, like Gecko) Chrome/16.0.897.0 Safari/535.6',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/11.10 Chromium/15.0.874.120 Chrome/15.0.874.120 Safari/535.2',
+        'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2',
+        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2',
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.860.0 Safari/535.2',
+        'Chrome/15.0.860.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/15.0.860.0',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.04 Chromium/14.0.813.0 Chrome/14.0.813.0 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.814.0 Chrome/14.0.814.0 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.814.0 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.803.0 Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1',
+        'Mozilla/5.0 Slackware/13.37 (X11; U; Linux x86_64; en-US) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41',
+        'Mozilla/5.0 ArchLinux (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/13.0.782.41 Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 5.2; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_3) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1',
+        'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1',
+        'Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00',
+        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52',
+        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52',
+        'Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51',
+        'Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50',
+        'Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50',
+        'Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11',
+        'Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11',
+        'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11',
+        'Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01',
+        'Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01',
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01',
+        'Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01',
+        'Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01',
+        'Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10',
+        'Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10',
+        'Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10',
+        'Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1',
+        'Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00',
+        'Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00',
+        'Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00',
+        'Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00',
+        'Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00',
+        'Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00',
+        'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00',
+        'Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00',
+        'Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.6.31 Version/10.70',
+        'Mozilla/5.0 (Windows NT 5.2; U; ru; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.70',
+        'Mozilla/5.0 (Windows NT 5.1; U; zh-cn; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.70',
+        'Opera/9.80 (X11; Linux i686; U; en-GB) Presto/2.5.24 Version/10.53',
+        'Mozilla/5.0 (Windows NT 5.1; U; zh-cn; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.53',
+        'Mozilla/5.0 (Windows NT 5.1; U; Firefox/5.0; en; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.53',
+        'Mozilla/5.0 (Windows NT 5.1; U; Firefox/4.5; en; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.53',
+        'Mozilla/5.0 (Windows NT 5.1; U; Firefox/3.5; en; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.53',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; ko) Opera 10.53',
+        'Opera/9.80 (X11; U; Linux i686; en-US; rv:1.9.2.3) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (X11; Linux x86_64; U; it) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (Windows NT 6.1; U; de) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (Windows NT 6.0; U; Gecko/20100115; pl) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (Windows NT 6.0; U; en) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (Windows NT 5.1; U; de) Presto/2.2.15 Version/10.10',
+        'Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.2.15 Version/10.10',
+        'Mozilla/5.0 (Windows NT 6.0; U; tr; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 10.10',
+        'Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; de) Opera 10.10',
+        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 6.0; tr) Opera 10.10',
+        'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
+        'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
+        'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)',
+        'Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)',
+        'Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)',
+        'Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))',
+        'Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205',
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)',
+        'Mozilla/5.0 ( ; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
+        'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; FDM; MSIECrawler; Media Center PC 5.0)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)',
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; msn OptimizedIE8;ZHCN)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MS-RTC LM 8; InfoPath.3; .NET4.0C; .NET4.0E) chromeframe/8.0.552.224',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MS-RTC LM 8; .NET4.0C; .NET4.0E; Zune 4.7; InfoPath.3)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MS-RTC LM 8; .NET4.0C; .NET4.0E; Zune 4.7)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MS-RTC LM 8)',
+        'Mozilla/4.0(compatible; MSIE 7.0b; Windows NT 6.0)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; FDM; .NET CLR 1.1.4322)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; Alexa Toolbar; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; Alexa Toolbar)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.1.4322)',
+        'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; .NET CLR 1.0.3705; Media Center PC 3.1; Alexa Toolbar; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)',
+        'Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; el-GR)',
+        'Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 5.2)',
+        'Mozilla/5.0 (MSIE 7.0; Macintosh; U; SunOS; X11; gu; SV1; InfoPath.2; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; WOW64; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; c .NET CLR 3.0.04506; .NET CLR 3.5.30707; InfoPath.1; el-GR)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; c .NET CLR 3.0.04506; .NET CLR 3.5.30707; InfoPath.1; el-GR)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; fr-FR)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; en-US)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.2; WOW64; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (compatible; MSIE 7.0; Windows 98; SpamBlockerUtility 6.3.91; SpamBlockerUtility 6.2.91; .NET CLR 4.1.89;GB)',
+        'Mozilla/4.79 [en] (compatible; MSIE 7.0; Windows NT 5.0; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)',
+        'Mozilla/4.0 (Windows; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30)',
+        'Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1)',
+        'Mozilla/4.0 (compatible;MSIE 7.0;Windows NT 6.0)',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; chromeframe/12.0.742.100)',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)',
+        'Mozilla/5.0 (Windows; U; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)',
+        'Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4325)',
+        'Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/45.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/4.08 (compatible; MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/4.01 (compatible; MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/4.0 (X11; MSIE 6.0; i686; .NET CLR 1.1.4322; .NET CLR 2.0.50727; FDM)',
+        'Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 6.0)',
+        'Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.2)',
+        'Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.0)',
+        'Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/4.0 (MSIE 6.0; Windows NT 5.0)',
+        'Mozilla/4.0 (compatible;MSIE 6.0;Windows 98;Q312461)',
+        'Mozilla/4.0 (Compatible; Windows NT 5.1; MSIE 6.0) (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Mozilla/4.0 (compatible; U; MSIE 6.0; Windows NT 5.1) (Compatible; ; ; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)',
+        'Mozilla/4.0 (compatible; U; MSIE 6.0; Windows NT 5.1)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; Tablet PC 2.0)',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB6.5; QQDownload 534; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729)'
+      ]
+    end
+  end
+end

data/lib/pioneer/request.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+module Pioneer
+  class Request
+    attr_reader :pioneer, :url, :result, :response, :error
+    def initialize(url, pioneer)
+      @url, @pioneer = url, pioneer
+      @url = begin
+        url = "http://" + url unless url =~ /http/
+        URI.escape(url)
+      end
+    end
+
+    def perform
+      pioneer.logger.info("going to #{url}")
+      @result = handle_request_error_or_return_result
+    end
+
+    # Handle base fatal request error
+    def handle_request_error_or_return_result
+      begin
+        @response = EventMachine::HttpRequest.new(url).get(pioneer.http_opts)
+      rescue => e
+        @error = "Request totaly failed. Url: #{url}, error: #{e.message}"
+        pioneer.logger.fatal(error)
+        if pioneer.respond_to? :if_request_error
+          return pioneer.send(:if_request_error, self)
+        else
+          raise HttpRequestError, @error
+        end
+      end
+      handle_response_error_or_return_result
+    end
+
+    # handle http error
+    def handle_response_error_or_return_result
+      if response.error
+        error = "Response for #{url} get an error: #{response.error}"
+        pioneer.logger.error(error)
+        if pioneer.respond_to? :if_response_error
+          return pioneer.send(:if_response_error, self)
+        else
+          raise HttpResponseError, error
+        end
+      end
+      handle_status_or_return_result
+    end
+
+    def handle_status_or_return_result
+      status = response.response_header.status
+      case status
+      when 200
+        pioneer.processing(self)
+      else
+        pioneer.logger.error("This #{url} returns this http status: #{status}")
+        if pioneer.respond_to? "if_status_#{status}".to_sym
+          pioneer.send("if_status_#{status}", self)
+        elsif pioneer.respond_to? :if_status_not_200
+          pioneer.send(:if_status_not_200, self)
+        else
+          nil # nothing?
+        end
+      end
+    end
+  end
+end

data/lib/pioneer/version.rb

@@ -0,0 +1,3 @@
+module Pioneer
+  VERSION = "0.0.1.alpha"
+end

data/pioneer.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "pioneer/version"
+
+Gem::Specification.new do |s|
+  s.name        = "pioneer"
+  s.version     = Pioneer::VERSION
+  s.authors     = ["Petr"]
+  s.email       = ["pedro.yanoviches@gmail.com"]
+  s.homepage    = ""
+  s.summary     = "HTTP crawler"
+  s.description = "Simple async HTTP crawler based on em-synchrony"
+
+  s.rubyforge_project = "pioneer"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "yajl-ruby"
+  s.add_runtime_dependency "em-synchrony"
+end

data/spec/pioneer/base_spec.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Pioneer::Base do
+end

data/spec/pioneer/request_spec.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'yajl'
+# 
+# TODO:
+# Rewrite real live examples with StubServer
+#
+
+describe Pioneer::Request do
+  before do
+    @pioneer1 = CustomCrawler1.new(name: "Custom crawler 1")
+    @pioneer2 = Pioneer::Crawler.new(name: "Base crawler 2")
+    @pioneer3 = Pioneer::Crawler.new(name: "Base crawler 3")
+  end
+
+  it "should return two 200 response statuses" do
+    @pioneer1.start.must_equal [200, 200]
+  end
+
+  it "should redefine methods" do
+    processing = proc{ |req| req.response.response_header.status + 1 }
+    @pioneer2.processing = processing
+    @pioneer2.locations = ["www.apple.com", "www.amazon.com"]
+    @pioneer2.start.must_equal [201, 201]
+    @pioneer2.locations = ["www.ru.erro"]
+    if_response_error = proc{ |req| "fail" }
+    @pioneer2.if_response_error = if_response_error
+    @pioneer2.start.must_equal ["fail"]
+  end
+
+  it "should execute if_status_xxx" do
+    redirector = proc{ |req| "redirected" }
+    error404 = proc{ |req| "notfound" }
+    @pioneer2.locations = ["google.com/redirectmeplease", "http://www.amazon.com/notfoundpage"]
+    @pioneer2.if_status_301 = redirector
+    @pioneer2.if_status_404 = error404
+    @pioneer2.start.must_equal ["redirected", "notfound"]
+  end
+
+  it "should execute if_status_not_200 if another colback is not defined" do
+    not_200 = proc{ "something goes wrong" }
+    redirector = proc{ |req| "redirected" }
+    @pioneer3.locations = ["google.com/redirectmeplease", "http://www.amazon.com/notfoundpage"]
+    @pioneer3.if_status_301 = redirector
+    @pioneer3.if_status_not_200 = not_200
+    @pioneer3.start.must_equal ["redirected", "something goes wrong"]
+  end
+
+  # LAST FM API TEST
+  it "should return similar artists for a number of them" do
+    @lastfm_pioneer = LastfmCrawler.new(sleep: 0.25)
+    @lastfm_pioneer.start.sort.must_equal LastfmEnum.const_get(:ARTISTS).sort
+  end
+
+  it "should use headers" do
+    @crawler1 = KinopoiskCrawler.new(random_header: false)
+    @crawler2 = KinopoiskCrawler.new(random_header: false, redirects: 1)
+    @crawler3 = KinopoiskCrawler.new(random_header: true)
+    # this one will redirect
+    @crawler1.start.must_equal [nil]
+    # this one will return some restrictions (it need real headres)
+    (@crawler2.start.first < 10000).must_equal true
+    # and this one will fire up
+    (@crawler3.start.first > 10000).must_equal true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,51 @@
+require 'pioneer'
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'nokogiri'
+
+# saving two pages
+class CustomCrawler1 < Pioneer::Base
+  def locations
+    ["http://www.ru", "http://www.ru"]
+  end
+
+  def processing(req)
+    req.response.response_header.status
+  end
+end
+
+# LastFM test
+class LastfmEnum
+  include Enumerable
+
+  ARTISTS = ["Cher", "Madonna", "Rolling Stones", "The Beatles", "Muse"]
+
+  def each
+    ARTISTS.each do |artist|
+      url = "http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist=#{artist}&api_key=b25b959554ed76058ac220b7b2e0a026&format=json"
+      yield url
+    end
+  end
+end
+
+class LastfmCrawler < Pioneer::Base
+  def locations
+    LastfmEnum.new
+  end
+
+  def processing(req)
+    json = Yajl::Parser.parse(req.response.response)
+    json["similarartists"]["@attr"]["artist"]
+  end
+end
+
+# Kinopoisk
+class KinopoiskCrawler < Pioneer::Base
+  def locations
+    ["http://www.kinopoisk.ru/level/1/film/614667/"]
+  end
+
+  def processing(req)
+    req.response.response.size
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: pioneer
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+  prerelease: 6
+platform: ruby
+authors:
+- Petr
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-22 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &74894120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *74894120
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: &74893910 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *74893910
+description: Simple async HTTP crawler based on em-synchrony
+email:
+- pedro.yanoviches@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/patch/fiber_periodic_timer_iterator.rb
+- lib/patch/iterator.rb
+- lib/pioneer.rb
+- lib/pioneer/base.rb
+- lib/pioneer/crawler.rb
+- lib/pioneer/http_header.rb
+- lib/pioneer/request.rb
+- lib/pioneer/version.rb
+- pioneer.gemspec
+- spec/pioneer/base_spec.rb
+- spec/pioneer/request_spec.rb
+- spec/spec_helper.rb
+- tmp/just_for_test/railscasts.txt
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: pioneer
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: HTTP crawler
+test_files: []
+has_rdoc: