spider2 0.0.1

data/lib/spider/page/pagination.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+# 分页处理
+# 分页一般有三种形式
+# 1)
+# 上页 1 2 3 4 5 6 下页
+# 可以使用
+# paginate :scope=>"#paginte-div"
+# 还有一种形式是 一共有多少页
+# 比如 88 页
+
+module Spider::Page::Pagination
+  module ClassMethods
+    # 实现一个简单的可以解决大部分分页的功能
+    # paginate :body,:links=>".paginate",:to=>[:body] do |page|
+    # end
+    # options:
+    #  scope: 指定分页链接范围
+    #  比如：<div class="pagination"><a href="?1">1</a></div>
+    #  可以指定scope为 div.pagination
+    #  那么所有 div.pagination下的链接都认为是分页链接
+    # include_self: boolean
+    #  是否包含当前页面，默认为false
+    # sort: boolean
+    #  是否对链接进行排序,默认为false
+    def paginate(to=:body,options={},&block)
+      options.assert_valid_keys :scope,:include_self,:sort
+      options.reverse_merge! :include_self=>false,:sort=>false,:scope=>".pagination"
+
+      partial_name = "#{to}_partial"
+      paginated_pages_name = "#{to}_paginated_pages"
+
+      # define_method partial_name,&block # 获得页面部分的body
+
+      define_method paginated_pages_name do
+        pages = self.pages(:scope=>options[:scope])
+
+        if options[:include_self]
+          pages.unshift(self)
+        end
+
+        if options[:sort]
+          pages = pages.sort_by{|page| page.url }
+        end
+        pages
+      end
+
+      # 定义获取全文的method
+      define_method to do 
+        send(paginated_pages_name).collect{|page|
+          page.send(partial_name)
+        } * Spider::Page.paginate_symbol
+      end
+    end
+    
+    # 产生分页链接
+    # generate_paginated_urls 10,"http://www.google.com/article_%d.html",:first=>"http://www.google.com/article.html"
+    def generate_paginated_urls(page_count,url,options={})
+      options.assert_valid_keys :first,:until
+      page_count = 1000 if options[:until] # 1000页总够了吧
+      urls = []
+      page_count.times do |i|
+        if options[:first] && i.zero?
+          urls << options[:first]
+        else
+          urls << url % i
+        end
+      end
+      urls
+    end
+
+  end
+  
+  module InstanceMethods
+    # http://www.google.com/article_1.html
+    # page.generate_paginated_urls(10,/article(_\d+?)\.html/)
+    # 一般有几种方式
+    # 1):
+    # xxxxx.html
+    # xxxxx_1.html
+    # xxxxx_2.html
+    # 2):
+    # xxx.asp
+    # xxx.asp?page=2
+    # xxx.asp?page=3
+    #  xxxxx(_%d).html
+    #  xxx.asp(?page=%d)
+    # 比如 http://www.powerapple.com/articles/11111.html
+    #      http://www.powerapple.com/articles/11111_1.html
+    # 如果已知是 5 页
+    # 那么：http://www.powerapple.com/articles/11111.html
+    #       http://www.powerapple.com/articles/11111_2.html
+    #       http://www.powerapple.com/articles/11111_3.html
+    #       http://www.powerapple.com/articles/11111_4.html
+    #       http://www.powerapple.com/articles/11111_5.html
+    # generate_paginated_urls 5,
+    #   :url=>"http://www.powerapple.com/articles/1111_%d.html",
+    #   :start=>1,
+    #   :unshift=>"http://www.powerapple.com/articles/1111.html"
+    def generate_paginated_urls(url,options={})
+      options.assert_valid_keys :start,:prepend,:append,:until_failure,:count
+      number = options[:count].to_i
+      number = 1000 if options[:until_failure] # 根据条件，获取页面
+      new_url = url
+      start   = options[:start]
+      urls = []
+      number.times do |index|
+        index += 1
+        next if start && index < start
+        page_url = (url % index)
+        if options[:until_failure]
+          response = Spider::Http.head(page_url)
+          unless response.code == 200
+            break # 遇到404就退出
+          end
+        end
+        urls << page_url
+      end
+      if options[:prepend]
+        [options[:prepend]].flatten.reverse.each do |url|
+          urls.unshift(url)
+        end
+      end
+      if options[:append]
+        [options[:append]].flatten.reverse.each do |url|
+          urls.push(url)
+        end
+      end
+      urls
+    end
+
+    def generate_paginated_pages(*args)
+      urls = generate_paginated_urls(*args)
+      urls.collect{|url| go(url) }
+    end
+  end
+  
+  def self.included(base)
+    base.send(:include,InstanceMethods)
+    base.send(:extend,ClassMethods)
+  end
+  
+end

data/lib/spider/page/proxy.rb

@@ -0,0 +1,149 @@
+# encoding: utf-8
+module Spider::Page::Proxy
+  def self.included(base)
+    base.send(:include,InstanceMethods)
+    base.send(:extend,ClassMethods)
+    base.class_eval do
+      class_attribute :proxies
+      class_attribute :disabled_proxies
+      self.proxies = []
+      self.disabled_proxies = []
+
+      before_fetch do |page|
+        proxies.compact!
+        proxies.uniq!
+        host,port = proxies.shuffle.first
+        port ||= 80
+        if host
+          logger.debug "set proxy: #{host}:#{port}"
+          Spider::Http.http_proxy host,port
+        else
+          Spider::Http.clear_proxy
+        end
+      end
+
+      after_fetch do |page|
+        logger.debug "reset proxy"
+        # Spider::Http.http_proxy old_host,old_port
+        if page.content.blank?
+          # retry, and set proxy to disabled
+          # proxies
+          puts "proxies before:#{self.proxies.inspect}"
+          disabled_proxy = proxies.find{|proxy| proxy.first == Spider::Http.default_options[:http_proxyaddr] }
+          proxies.delete disabled_proxy
+          self.disabled_proxies += [disabled_proxy]
+          puts "proxies after:#{self.proxies.inspect}"
+          unless proxies.empty?
+            puts 'retry'
+            page.request
+            next
+          else
+            puts 'finished retry.'
+            # no proxies available
+            # recover proxies
+            # 以便下次仍然使用(防止一次意外失败，而永久排除)
+            self.proxies += self.disabled_proxies
+            self.disabled_proxies = []
+            # 不用代理服务器使用自身来获取
+          end
+        end
+        Spider::Http.clear_proxy
+      end
+
+
+    end
+  end
+
+  module ClassMethods
+
+    def disable_proxy
+      proxy(nil,nil)
+    end
+
+    def validate_proxies
+      valid_proxies = proxies.find_all do |proxy|
+        valid_proxy?(*proxy)
+      end
+      invalid_proxies = proxies - valid_proxies
+      {:valid => valid_proxies,:invalid => invalid_proxies}
+    end
+
+    # 指定一个 file 作为 proxy 来源
+    # # ip:port
+    def proxy_file(file)
+      config_root = File.join(Rails.root,"config","spiders")
+      if file =~ /^\//
+        # absolute path
+        content = File.read file
+      else
+        content = File.read(File.join(config_root,file))
+      end
+      proxies = []
+      content.each_line do |line|
+        line = line.strip
+        if line =~ /^\s*#/
+          # 注释
+        else
+          if line =~ /\d+?\.\d+?\.\d+?\.\d+?/
+            ip,port = line.split(":")
+            port ||= 80
+            proxies += [[ip,port]]
+          end
+        end
+      end
+      self.proxy do |the_proxies|
+        proxies.each do |p|
+          the_proxies += [p]
+        end
+      end
+    end
+
+    def valid_proxy?(ip,port = 80,options = {})
+      options[:url] ||= "http://www.google.com"
+      options[:code] ||= 200
+      options[:timeout] ||= 10
+      # options[:match] ||= //
+      Spider::Http.with_proxy ip,port do
+        begin
+          timeout options[:timeout] do
+            response = Spider::Http.get options[:url]
+            r = response.code == options[:code]
+            if options[:match]
+              r && (response.to_s =~ options[:match])
+            else
+              r
+            end
+          end
+        rescue Exception => e
+          false
+        end
+      end
+    end
+
+    # 直接设置 proxies
+    def proxies=(arr)
+      proxy do |ps|
+        arr.each do |a|
+          ps += [a]
+        end
+      end
+    end
+
+    def proxy(host = nil,port = 80,&block)
+      self.proxies += [[host,port]] if host
+
+      if block_given?
+        yield self.proxies
+      end
+
+    end
+
+  end
+
+
+
+  module InstanceMethods
+
+  end
+
+end

data/lib/spider/page/publish.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+module Spider::Page::Publish
+
+  extend ActiveSupport::Concern
+
+  included do
+
+      define_model_callbacks :publish
+
+      cattr_accessor :publishers
+      self.publishers = []
+      after_crawl :publish
+      
+  end
+
+  module ClassMethods
+
+    # publish_to Article
+    # Article will set to publisher
+    # publish_to Article will called when #crawl
+    def publish_to(*publishers) 
+      logger.debug "[#{self}] set publisher: #{publishers}"
+      self.publishers += publishers
+    end
+
+  end
+
+  module InstanceMethods
+
+    def publish_to(*publishers)
+      run_callbacks :publish do
+        logger.debug "publish to #{publishers}"
+        results = []
+        [publishers].flatten.each do |publisher|
+          logger.info "send self to #{publisher}"
+          logger.debug "class:#{publisher.class.name}"
+          publisher = case publisher
+                      when String,Symbol
+                        publisher.to_s.classify.constantize
+                      else
+                        # puts "default: #{publisher}"
+                        publisher
+                      end
+          logger.debug "publisher: #{publisher}"
+          result = nil
+          begin
+
+            if publisher.respond_to?(:receive_spider_page) 
+              logger.debug "#{publisher} receive spider page #{self}"
+              result = publisher.receive_spider_page self
+              logger.debug "#{publisher} return #{result}" 
+            else
+              logger.debug "publisher: #{publisher} not respond to :receive_spider_page"
+            end
+          rescue Exception=>e
+            logger.error e.message
+            logger.error e.backtrace.join("\n")
+          end
+          results << result
+        end
+        results
+      end
+    end
+
+    def publish
+      publishers = self.publishers.uniq
+      if [:title,:body].all?{|name| attribute_names.include?(name) }
+        logger.debug "[#{self} publish to #{publishers}"
+        publish_to(publishers)
+      else
+        logger.debug "attribute names not include :title, :body,so publish canceled."
+      end
+    end
+
+
+  end
+
+end

data/lib/spider/page/validation.rb

@@ -0,0 +1,136 @@
+# encoding: utf-8
+module Spider::Page::Validation
+  
+  class ValidationError < Exception; end
+  
+  module InstanceMethods
+    
+    def valid_url?
+      self.class.valid_url?(url)
+    end
+    
+  end
+  
+  module ClassMethods
+    
+    # 根据 url 来返回合适的 Spider::Page 类
+    def find_by_url(url)
+      Spider::Site.find_pages.find do |page|
+        page.valid_url?(url)
+      end
+    end
+    
+    # 返回一个数组，匹配url的所有page
+    def find_all_by_url(url)
+      pages = Spider::Site.find_pages.find_all do |page|
+        page.valid_url?(url)
+      end
+      if pages.empty?
+        # 获得该 domain 下的所有 pages
+        sites = Spider::Site.all.find_all do |site|
+          site.valid_domain?(url)
+        end
+        pages = sites.collect{|site| site.pages }.flatten
+      end
+      pages
+    end
+
+    # 直接创建 page 的实例
+    def create_all_by_url(url)
+      find_all_by_url(url).collect{|klass| klass.new url }
+    end
+
+    # 直接创建 page 的实例
+    def create_by_url(url)
+      find_by_url(url).try(:new,url)
+    end
+    
+    # 判断 url 是不是该类能处理的
+    def valid_url?(url)
+      valid = false
+      self.validate_url_procs.each do |p|
+
+        if p.call(url)
+          valid = true  
+          break
+        end
+                 
+      end
+      valid
+    end
+    
+    # 例子：
+    # validate_url "china.com",:match=>/suzhou/,:unmatch=>/beijing/
+    # validate_url "google.com",:match=>/baidu/,:unmatch=>/yahoo/
+    def validate_url(options={},&block)
+      options.assert_valid_keys :match,:unmatch,:example,:domain
+      p = lambda do |url|
+        
+        begin 
+          uri = URI(url)
+
+          domain  = [options[:domain]].flatten.compact.uniq
+          unless domain.empty?
+            unless domain.any?{|d| uri.host.end_with? d }
+              raise ValidationError.new(" domain: #{uri.host} not in #{domain.inspect} ")
+            end
+          end
+
+
+          match   = [options[:match]].flatten.compact.uniq
+          unmatch = [options[:unmatch]].flatten.compact.uniq
+
+          unless match.all?{ |regexp| 
+            next regexp.call(url) if regexp.is_a? Proc
+            regexp = Regexp.escape regexp if regexp.is_a? String
+            regexp =~ url
+          }
+            raise ValidationError.new("#{url} not match #{match.inspect}")
+          end
+
+          if unmatch.any?{|regexp| 
+            regexp = Regexp.escape regexp if regexp.is_a? String
+            regexp =~ url
+          }
+            raise ValidationError.new("#{url} match #{unmatch.inspect}")
+          end
+        rescue ValidationError=>e
+          logger.debug e.message
+          next false
+        rescue Exception=>e
+          logger.error e.message
+          logger.error e.backtrace.join("\n")
+          next false
+        end
+
+        true
+      end
+      
+      if options[:example]
+        set_example_url options[:example]
+        # 执行自我检查
+        [options[:example]].flatten.each do |url|
+          unless p.call(url)
+            raise ValidationError.new("#{url} is not a valid url for me.")
+          end
+        end
+      end
+      
+      self.validate_url_procs += [p]
+    end
+
+  end
+  
+  def self.included(base)
+    base.class_eval do
+      class_attribute :validate_url_procs
+      self.validate_url_procs = []
+
+      
+      include InstanceMethods
+      extend  ClassMethods
+
+    end
+  end
+  
+end