spider2 0.0.1 → 0.0.9

data/lib/spider.rb

@@ -1,5 +1,21 @@
 # encoding: utf-8
 #require "hpricot"
+#
+
+# copy from active support
+require "httparty"
+Hash.class_eval do
+  def deep_dup
+    duplicate = self.dup
+    duplicate.each_pair do |k,v|
+      tv = duplicate[k]
+      duplicate[k] = tv.is_a?(Hash) && v.is_a?(Hash) ? tv.deep_dup : v
+    end
+    duplicate
+  end
+end unless Hash.new.respond_to? :deep_dup
+
+
 module Spider
   def self.logger
     unless @logger
@@ -52,30 +68,11 @@ Spider::Page.send(:include,Spider::Page::Proxy)
 require "spider/page/label"
 Spider::Page.send(:include,Spider::Page::Label)
 
+require "spider/page/cache"
+Spider::Page.send(:include,Spider::Page::Cache)
 
-spiders_dir = File.join(Rails.root,"spiders")
-$:.push(spiders_dir)
-
-# define constants
-Dir[File.join(spiders_dir,"*")].each do |dir|
-  dir_name = dir.gsub(spiders_dir,"").gsub(/^\//,"")
-  Object.const_set(dir_name.classify,Module.new)
-end
-
-# 先包含初始化文件
-init_file = File.join(spiders_dir,"init.rb")
-require init_file if File.exists? init_file
-
-file_patten = File.join(spiders_dir,"**","*.rb")
-files = Dir[file_patten]
-
-site_files = files.find_all{|i| i =~ /site\.rb/}
-site_files.each{|i| require i}
-
-base_page_files = files.find_all{|i| i =~ /base_page\.rb/}
-base_page_files.each{|i| require i}
 
-files.each{|i| require i }
 
 # 包含 active record methods
 require "spider/active_record_methods"
+require "spider/engine"

data/lib/spider/engine.rb

@@ -0,0 +1,31 @@
+module Spider
+  class Engine < Rails::Engine
+    initializer 'spider' do
+      spiders_dir = File.join(Rails.root,"spiders")
+      $:.push(spiders_dir)
+
+      # define constants
+      Dir[File.join(spiders_dir,"*")].each do |dir|
+        dir_name = dir.gsub(spiders_dir,"").gsub(/^\//,"")
+        Object.const_set(dir_name.classify,Module.new)
+      end
+
+      # 先包含初始化文件
+      init_file = File.join(spiders_dir,"init.rb")
+      require init_file if File.exists? init_file
+
+      file_patten = File.join(spiders_dir,"**","*.rb")
+      files = Dir[file_patten]
+
+      site_files = files.find_all{|i| i =~ /site\.rb/}
+      site_files.each{|i| require i}
+
+      base_page_files = files.find_all{|i| i =~ /base_page\.rb/}
+      base_page_files.each{|i| require i}
+
+      files.each{|i| require i }
+
+    end
+
+  end
+end

data/lib/spider/http.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "spider/httparty_patch"
 module Spider::Http
   include HTTParty
   headers "User-Agent"=>"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.6) Gecko/2009020518 Ubuntu/9.04 (jaunty) Firefox/3.0.6"
@@ -21,8 +22,8 @@ module Spider::Http
     end
   end
 
-  def self.with_proxy(ip,port,&block)
-    http_proxy ip,port
+  def self.with_proxy(proxy,&block)
+    http_proxy proxy.host,proxy.port,proxy.user,proxy.password
     result = yield
     clear_proxy
     result
@@ -31,6 +32,8 @@ module Spider::Http
   def self.clear_proxy
     Spider::Http.default_options.delete :http_proxyaddr
     Spider::Http.default_options.delete :http_proxyport
+    Spider::Http.default_options.delete :http_proxyuser
+    Spider::Http.default_options.delete :http_proxypassword
   end
 
 =begin

data/lib/spider/httparty_patch.rb

@@ -0,0 +1,37 @@
+module HTTParty
+  module ClassMethods
+    def http_proxy(addr=nil, port = nil, user=nil, password=nil)
+      default_options[:http_proxyaddr] = addr
+      default_options[:http_proxyport] = port
+      default_options[:http_proxyuser] = user
+      default_options[:http_proxypassword] = password
+    end
+  end
+
+  class Request
+    def http
+      http = Net::HTTP.new(
+        uri.host,
+        uri.port,
+        options[:http_proxyaddr],
+        options[:http_proxyport],
+        options[:http_proxyuser],
+        options[:http_proxypassword]
+      )
+      http.use_ssl = ssl_implied?
+
+      if options[:timeout] && (options[:timeout].is_a?(Integer) || options[:timeout].is_a?(Float))
+        http.open_timeout = options[:timeout]
+      end   
+
+      attach_ssl_certificates(http)
+
+      if options[:debug_output]
+        http.set_debug_output(options[:debug_output])
+      end 
+
+      http
+    end
+  end
+end
+

data/lib/spider/page.rb

@@ -4,6 +4,8 @@ class Spider::PageExistsAndDoneException < Exception; end
 require "iconv"
 require "digest/md5"
 require "htmlentities"
+require "spider/spider_page"
+require "v8"
 # 从本质上讲，所有的WEB页面都是一个页面(Page)
 # 每个页面拥有一些属性，比如(encoding,title,url)
 # 每个页面有我们感兴趣的信息，我们需要提取出来
@@ -37,16 +39,17 @@ class Spider::Page
     @coder ||= HTMLEntities.new
   end
 
-  def coder
-    self.class.coder
+  def self.parse_query(string)
+    Rack::Utils.parse_query(string)
   end
 
-  def self.class_attribute(*args)
-    # class_attribute跟class_inheritable_accessor对待Array,Hash的方式还存在差异
-    # 现在还是得使用class_inheritable_accessor
-    class_inheritable_accessor *args # 目前还没找到更好的方式
+  def parse_query(string)
+    self.class.parse_query string
   end
 
+  def coder
+    self.class.coder
+  end
 
   extend ActiveModel::Callbacks
 
@@ -61,9 +64,17 @@ class Spider::Page
   self.options = {}
   self.options[:example_url] ||= []
 
-
   SEPARATOR = "<!-- PAGINATE SEPARATOR -->"
 
+  def self.inherited(subclass)
+    subclass.options = options.dup
+    super
+  end
+
+  def options
+    self.class.options
+  end
+
   @@paginate_symbol = "--NEXTPAGE--"
   cattr_accessor :paginate_symbol
 
@@ -115,7 +126,7 @@ class Spider::Page
   end
 
   def self.define_attribute(attribute)
-    self.attribute_names << attribute
+    self.attribute_names += [attribute]
     self.attribute_names.uniq!
     self.attribute_names.compact!
     attribute
@@ -140,6 +151,18 @@ class Spider::Page
     hash
   end
 
+  def v8
+    @v8 ||= V8::Context.new
+  end
+
+  # eval_js "info = {name:'name'}"
+  # v8['info']['name'] => "name" 
+  # v8 is a V8::Context instance
+  # for details about V8 , refer to therubyracer gem
+  def eval_js(js)
+    v8.eval js
+  end
+
   # 对 <base href="xxxx" /> 的标记进行快捷获取
   def base_href
     doc.at("base").try(:attributes).try(:[],"href")
@@ -148,12 +171,11 @@ class Spider::Page
 
   # 从url的query string中分析得到params
   def params
-    Rack::Utils.parse_query(uri.query).tap do |r|
-      # r.symbolize_keys!
-      r.each_pair do |key,value|
-        r[key.to_sym] = value
-      end
+    h = {}
+    Rack::Utils.parse_query(uri.query).each_pair do |key,value|
+      h[key.to_sym] = value
     end
+    h
   end
 
   # 提供对 attributes 的快捷访问
@@ -559,7 +581,7 @@ class Spider::Page
     if exists?
       page = spider_page
     else
-      page = Spider::SpiderPage.new(options.merge(:url=>url,:site=>site.try(:id)))
+      page = ::Spider::SpiderPage.new(options.merge(:url=>url,:site=>site.try(:id)))
     end
     page.content_length = content_length
     #page.labels_hash = Digest::MD5.hexdigest(labels.to_yaml)
@@ -744,16 +766,4 @@ class Spider::Page
   end
 
 
-  def fetch_content_from_url_with_cache(options={})
-    key = Digest::MD5.hexdigest(options.to_json.to_s) + "/" + Digest::MD5.hexdigest(url)
-    @content ||= Rails.cache.fetch key do
-      fetch_content_from_url_without_cache(options)
-    end
-    @content_length = @content.length
-    @content
-  end 
-
-  # alias_method_chain :fetch_content_from_url,:cache
-
-
 end

data/lib/spider/page/cache.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+module Spider::Page::Cache
+  extend ActiveSupport::Concern
+  included do
+    class_attribute :cache_enabled
+    self.cache_enabled = false
+    alias_method_chain :fetch_content_from_url,:cache
+  end
+
+  def fetch_content_from_url_with_cache(*args)
+    fetch_content_from_url_without_cache *args
+    if self.cache_enabled
+      f = cache_file("content")
+      FileUtils.mkdir_p File.dirname(f)
+      File.open f,"w+" do |file|
+        file.write @content
+      end
+    end
+    @content
+  end
+
+  def cached_content
+    if self.cache_enabled
+      f = cache_file("content")
+      File.read f
+    end
+  end
+
+  protected
+  def cache_file(name = '')
+    md5 = Digest::MD5.hexdigest url
+    file = md5 + name
+    file = "#{file[0,3]}/#{file}"
+    if defined? Rails
+      Rails.root.join("tmp",file).to_s
+    else
+      "/tmp/#{file}"
+    end
+  end
+
+  module ClassMethods
+
+    def enable_cache
+      self.cache_enabled = true
+    end
+
+    def disable_cache
+      self.cache_enabled = false
+    end
+
+  end
+end

data/lib/spider/page/filter.rb

@@ -41,6 +41,7 @@ module Spider::Page::Filter
       options.assert_valid_keys :filters,:position
       position = options[:position]
       position = position.to_s + "_" if position
+      send("attributes_#{position}filters=",{})
       filter_attrs = send("attributes_#{position}filters")
       logger.debug "create filter: #{name} : position : #{position},options : #{options.inspect}"
       args.each do |attr_name|

data/lib/spider/page/label.rb

@@ -3,9 +3,6 @@
 module Spider::Page::Label
   extend ActiveSupport::Concern
 
-  module InstanceMethods
-  end
-
   module ClassMethods
     def label(name,options = {},&block)
       name = name.to_sym

data/lib/spider/page/proxy.rb

@@ -1,22 +1,97 @@
 # encoding: utf-8
 module Spider::Page::Proxy
+  class HttpProxy
+    attr_accessor :user,:password,:host,:port
+    # aotianlong:password@192.168.1.1:8000
+    # => :user => "aotianlong",
+    # :password => "password",
+    # :host => "192.168.1.1",
+    # :port => 8000
+    def self.parse(str)
+      {}.tap do |hash|
+        user = password = host = port = nil
+        if str =~ /@/
+          userinfo,addrinfo = str.split("@")
+          addr_hash = parse addrinfo
+          port,host = addr_hash[:port],addr_hash[:host]
+          user_hash = parse userinfo
+          user,password = user_hash[:host],user_hash[:port]
+        else
+          host,port = str.split(":")
+          port = 80 if port.blank?
+        end
+        hash[:host] = host
+        hash[:port] = port
+        hash[:user] = user if user
+        hash[:password] = password if password
+      end
+    end
+
+    def to_s
+      if user && password
+        "#{user}:#{password}@#{host}:#{port}"
+      else
+        "#{host}:#{port}"
+      end
+    end
+
+    def inspect
+      to_s
+    end
+
+    def initialize(host,options = {})
+      hash = self.class.parse host
+      hash.merge! options
+      @host = hash[:host]
+      @port = hash[:port]
+      @user = hash[:user]
+      @password = hash[:password]
+    end
+
+    def valid?(options = {})
+      options[:url] ||= "http://www.google.com"
+      options[:code] ||= 200
+      options[:timeout] ||= 10
+      # options[:match] ||= //
+      Spider::Http.with_proxy self do
+        begin
+          timeout options[:timeout] do
+            response = Spider::Http.get options[:url]
+            r = response.code == options[:code]
+            if options[:match]
+              r && (response.to_s =~ options[:match])
+            else
+              r
+            end
+          end
+        rescue Exception => e
+          false
+        end
+      end
+    end
+
+  end
+
   def self.included(base)
     base.send(:include,InstanceMethods)
     base.send(:extend,ClassMethods)
     base.class_eval do
-      class_attribute :proxies
+      class_attribute :proxy_items
       class_attribute :disabled_proxies
-      self.proxies = []
+      class_attribute :current_proxies
+      class_attribute :proxies_filename
+      self.current_proxies = []
       self.disabled_proxies = []
+      self.proxies_filename = nil
+      self.proxy_items = []
 
       before_fetch do |page|
-        proxies.compact!
-        proxies.uniq!
-        host,port = proxies.shuffle.first
-        port ||= 80
-        if host
-          logger.debug "set proxy: #{host}:#{port}"
-          Spider::Http.http_proxy host,port
+        self.current_proxies = proxies.shuffle.first(5) if self.current_proxies.empty?
+        logger.debug "current_proxies: #{current_proxies.inspect}"
+        proxy = current_proxies.shuffle.first
+        if proxy.try(:host)
+          logger.debug "set proxy: #{proxy.inspect}"
+          Spider::Http.http_proxy proxy.host,proxy.port,proxy.user,proxy.password
         else
           Spider::Http.clear_proxy
         end
@@ -25,15 +100,17 @@ module Spider::Page::Proxy
       after_fetch do |page|
         logger.debug "reset proxy"
         # Spider::Http.http_proxy old_host,old_port
-        if page.content.blank?
+        if page.content.blank? #|| page.code == 502 # bad gateway
           # retry, and set proxy to disabled
           # proxies
-          puts "proxies before:#{self.proxies.inspect}"
-          disabled_proxy = proxies.find{|proxy| proxy.first == Spider::Http.default_options[:http_proxyaddr] }
-          proxies.delete disabled_proxy
-          self.disabled_proxies += [disabled_proxy]
-          puts "proxies after:#{self.proxies.inspect}"
-          unless proxies.empty?
+          puts "proxies before:#{self.current_proxies.inspect}"
+          disabled_proxy = current_proxies.find{|proxy| proxy.host == Spider::Http.default_options[:http_proxyaddr] && proxy.port == Spider::Http.default_options[:http_proxyport] }
+          if disabled_proxy
+            current_proxies.delete disabled_proxy
+            self.disabled_proxies += [disabled_proxy]
+            puts "proxies after:#{self.current_proxies.inspect}"
+          end
+          unless current_proxies.empty?
             puts 'retry'
             page.request
             next
@@ -42,7 +119,7 @@ module Spider::Page::Proxy
             # no proxies available
             # recover proxies
             # 以便下次仍然使用(防止一次意外失败，而永久排除)
-            self.proxies += self.disabled_proxies
+            self.current_proxies  = []
             self.disabled_proxies = []
             # 不用代理服务器使用自身来获取
           end
@@ -57,83 +134,93 @@ module Spider::Page::Proxy
   module ClassMethods
 
     def disable_proxy
-      proxy(nil,nil)
+      proxy(nil)
     end
 
     def validate_proxies
       valid_proxies = proxies.find_all do |proxy|
-        valid_proxy?(*proxy)
+        valid_proxy?(proxy)
       end
       invalid_proxies = proxies - valid_proxies
       {:valid => valid_proxies,:invalid => invalid_proxies}
     end
 
-    # 指定一个 file 作为 proxy 来源
-    # # ip:port
-    def proxy_file(file)
-      config_root = File.join(Rails.root,"config","spiders")
-      if file =~ /^\//
-        # absolute path
-        content = File.read file
+    def proxies
+      if proxies_filename
+        parse_proxy_file proxies_filename
       else
-        content = File.read(File.join(config_root,file))
+        proxy_items
       end
+    end
+
+    def parse_proxies(content)
       proxies = []
       content.each_line do |line|
         line = line.strip
         if line =~ /^\s*#/
           # 注释
         else
-          if line =~ /\d+?\.\d+?\.\d+?\.\d+?/
-            ip,port = line.split(":")
-            port ||= 80
-            proxies += [[ip,port]]
-          end
-        end
-      end
-      self.proxy do |the_proxies|
-        proxies.each do |p|
-          the_proxies += [p]
+          # proxy line,options
+          proxies << Spider::Page::Proxy::HttpProxy.new(line)
         end
       end
+      proxies
     end
 
-    def valid_proxy?(ip,port = 80,options = {})
-      options[:url] ||= "http://www.google.com"
-      options[:code] ||= 200
-      options[:timeout] ||= 10
-      # options[:match] ||= //
-      Spider::Http.with_proxy ip,port do
-        begin
-          timeout options[:timeout] do
-            response = Spider::Http.get options[:url]
-            r = response.code == options[:code]
-            if options[:match]
-              r && (response.to_s =~ options[:match])
-            else
-              r
-            end
-          end
-        rescue Exception => e
-          false
-        end
+    def parse_proxy_file(file)
+      config_root = File.join(Rails.root,"config","spiders")
+      if file =~ /^\//
+        # absolute path
+        content = File.read file
+      else
+        content = File.read(File.join(config_root,file))
       end
+      parse_proxies content
+    end
+
+    # 指定一个 file 作为 proxy 来源
+    # # ip:port
+    def proxy_file(file,options = {})
+      self.proxies_filename = file
+      # parse_proxy_file(file).each do |proxy|
+      #  self.proxy proxy
+      # end
+    end
+
+    def valid_proxy?(proxy)
+      proxy.valid?
     end
 
     # 直接设置 proxies
     def proxies=(arr)
-      proxy do |ps|
-        arr.each do |a|
-          ps += [a]
+      proxy arr
+    end
+
+    def clear_proxies
+      self.proxy_items = []
+      self.proxies_filename = nil
+    end
+
+    def proxy(host = nil,options = {})
+
+
+      if host.is_a? Array
+        host.each do |h|
+          proxy h,options
         end
+        return
       end
-    end
 
-    def proxy(host = nil,port = 80,&block)
-      self.proxies += [[host,port]] if host
+      if host.is_a? Spider::Page::Proxy::HttpProxy
+        http_proxy = host
+      else
+        http_proxy = Spider::Page::Proxy::HttpProxy.new(host,options || {})
+      end
+
+      self.proxy_items += [http_proxy]
 
       if block_given?
-        yield self.proxies
+        yield([])  # for old syntax
       end
 
     end
@@ -143,7 +230,10 @@ module Spider::Page::Proxy
 
 
   module InstanceMethods
-
+    def proxies
+      self.class.proxies
+    end
   end
 
+
 end

data/lib/spider/page/publish.rb

@@ -5,12 +5,12 @@ module Spider::Page::Publish
 
   included do
 
-      define_model_callbacks :publish
+    define_model_callbacks :publish
+
+    cattr_accessor :publishers
+    self.publishers = []
+    after_crawl :publish
 
-      cattr_accessor :publishers
-      self.publishers = []
-      after_crawl :publish
-      
   end
 
   module ClassMethods
@@ -25,54 +25,52 @@ module Spider::Page::Publish
 
   end
 
-  module InstanceMethods
 
-    def publish_to(*publishers)
-      run_callbacks :publish do
-        logger.debug "publish to #{publishers}"
-        results = []
-        [publishers].flatten.each do |publisher|
-          logger.info "send self to #{publisher}"
-          logger.debug "class:#{publisher.class.name}"
-          publisher = case publisher
-                      when String,Symbol
-                        publisher.to_s.classify.constantize
-                      else
-                        # puts "default: #{publisher}"
-                        publisher
-                      end
-          logger.debug "publisher: #{publisher}"
-          result = nil
-          begin
+  def publish_to(*publishers)
+    run_callbacks :publish do
+      logger.debug "publish to #{publishers}"
+      results = []
+      [publishers].flatten.each do |publisher|
+        logger.info "send self to #{publisher}"
+        logger.debug "class:#{publisher.class.name}"
+        publisher = case publisher
+                    when String,Symbol
+                      publisher.to_s.classify.constantize
+                    else
+                      # puts "default: #{publisher}"
+                      publisher
+                    end
+        logger.debug "publisher: #{publisher}"
+        result = nil
+        begin
 
-            if publisher.respond_to?(:receive_spider_page) 
-              logger.debug "#{publisher} receive spider page #{self}"
-              result = publisher.receive_spider_page self
-              logger.debug "#{publisher} return #{result}" 
-            else
-              logger.debug "publisher: #{publisher} not respond to :receive_spider_page"
-            end
-          rescue Exception=>e
-            logger.error e.message
-            logger.error e.backtrace.join("\n")
+          if publisher.respond_to?(:receive_spider_page) 
+            logger.debug "#{publisher} receive spider page #{self}"
+            result = publisher.receive_spider_page self
+            logger.debug "#{publisher} return #{result}" 
+          else
+            logger.debug "publisher: #{publisher} not respond to :receive_spider_page"
           end
-          results << result
+        rescue Exception=>e
+          logger.error e.message
+          logger.error e.backtrace.join("\n")
         end
-        results
+        results << result
       end
+      results
     end
+  end
 
-    def publish
-      publishers = self.publishers.uniq
-      if [:title,:body].all?{|name| attribute_names.include?(name) }
-        logger.debug "[#{self} publish to #{publishers}"
-        publish_to(publishers)
-      else
-        logger.debug "attribute names not include :title, :body,so publish canceled."
-      end
+  def publish
+    publishers = self.publishers.uniq
+    if [:title,:body].all?{|name| attribute_names.include?(name) }
+      logger.debug "[#{self} publish to #{publishers}"
+      publish_to(publishers)
+    else
+      logger.debug "attribute names not include :title, :body,so publish canceled."
     end
+  end
 
 
-  end
 
 end

data/lib/spider/version.rb

@@ -1,3 +1,3 @@
 module Spider
-  VERSION = "0.0.1"
+  VERSION = "0.0.9"
 end

data/lib/spider2.rb

@@ -0,0 +1 @@
+require "spider"

data/lib/tasks/spider_tasks.rake

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # desc "Explaining what the task does"
 # task :spider_fu do
 #   # Task goes here
@@ -8,25 +9,89 @@ desc "开始采集"
 namespace :spider do
 
   namespace :proxy do
-    desc "test proxy PAGE=XXXX::BasePage" 
-    task :test => :environment do
-      klass = ENV['PAGE']
-      begin
-        klass = klass.constantize
-      rescue Exception => e
-        puts "unknow class `#{klass}`, please set a right spider page class to PAGE=XXXX::XxxPage"
-        exit
+
+
+    desc "test proxy" 
+    task :test2 => :environment do
+      times = ENV['TIMES'] && ENV['TIMES'].to_i || 50
+      verbose = ENV['VERBOSE']
+      proxies = Spider::Page.proxies
+      proxies_count = proxies.size
+      good_proxies = []
+
+      result = {}
+
+      times.times do |i| 
+        puts "round #{i + 1}/#{times}" if verbose
+        proxies.each_with_index do |proxy,index|
+          print "(#{index + 1}/#{proxies_count} #{proxy.inspect}): " if verbose
+          result[proxy] ||= []
+          r = {}
+          time = Benchmark.ms do 
+            r[:success ] = proxy.valid?
+          end / 1000
+          if r[:success]
+            print "OK" if verbose
+          else
+            print "FAILED" if verbose
+          end
+          print " " if verbose
+          print "#{time}s" if verbose
+          print "\n" if verbose
+          r[:time] = time
+          result[proxy] << r
+        end
       end
-      result = klass.validate_proxies
-      puts "valid proxies:"
-      result[:valid].each do |proxy|
-        puts proxy.join(":")
+
+      good_proxies = []
+      result.each_pair do |proxy,data|
+        success_times = data.find_all{|d| d[:success] }.size
+        success_ratio = success_times.to_f / times
+        average_time  = data.map{|d| d[:time] }.sum / data.size
+        good_proxies << proxy if success_ratio > 0.95
+        puts "=> #{proxy}" if verbose
+        puts data.collect{|d| d[:success] ? "*" : "!"}.join("") if verbose
+        puts "  => success times: #{success_times}(#{success_ratio * 100}%)" if verbose
+        puts "  => average time per request: #{average_time} seconds." if verbose
+        puts if verbose
       end
-      puts
-      puts "invalid proxies:"
-      result[:valid].each do |proxy|
-        puts proxy.join(":")
+      puts " =============================== " if verbose
+      puts 'following proxies are 95% complete all test request:' if verbose
+      puts good_proxies.map{|proxy| proxy.to_s }.join("\n")
+    end
+
+    desc "test proxy PAGE=XXXX::BasePage" 
+    task :test => :environment do
+      times = ENV['TIMES'] && ENV['TIMES'].to_i || 10
+      verbose = ENV['VERBOSE']
+      proxies = Spider::Page.proxies
+      proxies_count = proxies.size
+      good_proxies = []
+      proxies.each_with_index do |proxy,index|
+        puts "(#{index + 1}/#{proxies_count}) => #{proxy.to_s}" if verbose
+        total_time = 0.0
+        success_times = 0
+        times.times do 
+          total_time += Benchmark.ms do 
+            if proxy.valid?
+              success_times += 1 
+              print "*" if verbose
+            else
+              print "!" if verbose
+            end
+            STDOUT.flush
+          end / 1000
+        end
+        good_proxies << proxy if success_times == times
+        print "\n"
+        puts "  => success times: #{success_times}" if verbose
+        puts "  => total time:#{total_time/60} minutes" if verbose
+        puts "  => average time per request: #{total_time / times} seconds." if verbose
+        puts if verbose
       end
+      puts " =============================== " if verbose
+      puts 'following proxies are 100% complete all test request:' if verbose
+      puts good_proxies.map{|proxy| proxy.to_s }.join("\n")
     end
   end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: spider2
 version: !ruby/object:Gem::Version 
-  hash: 29
-  prerelease: 
+  hash: 13
+  prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 9
+  version: 0.0.9
 platform: ruby
 authors: 
 - aotianlong
@@ -15,54 +15,10 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-04 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: rails
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 3
-        - 2
-        - 0
-        version: 3.2.0
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: htmlentities
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 59
-        segments: 
-        - 4
-        - 1
-        - 0
-        version: 4.1.0
-  type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: sqlite3
-  prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id003
+date: 2012-09-17 00:00:00 +08:00
+default_executable: 
+dependencies: []
+
 description: a framework to crawl web pages
 email: 
 - aotianlong@gmail.com
@@ -91,7 +47,10 @@ files:
 - lib/generators/spider_migration/spider_migration_generator.rb
 - lib/generators/spider_migration/templates/migration.rb
 - lib/spider/active_record_methods.rb
+- lib/spider/engine.rb
 - lib/spider/http.rb
+- lib/spider/httparty_patch.rb
+- lib/spider/page/cache.rb
 - lib/spider/page/filter.rb
 - lib/spider/page/label.rb
 - lib/spider/page/pagination.rb
@@ -104,6 +63,7 @@ files:
 - lib/spider/spider_page_label.rb
 - lib/spider/version.rb
 - lib/spider.rb
+- lib/spider2.rb
 - lib/tasks/spider_tasks.rake
 - MIT-LICENSE
 - Rakefile
@@ -113,6 +73,7 @@ files:
 - uninstall.rb
 - test/spider_fu_test.rb
 - test/test_helper.rb
+has_rdoc: true
 homepage: http://www.powerapple.com
 licenses: []
 
@@ -142,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: spider