spider2 0.0.1

data/lib/spider/page.rb

@@ -0,0 +1,759 @@
+# encoding: utf-8
+class Spider::PageExistsAndDoneException < Exception; end
+
+require "iconv"
+require "digest/md5"
+require "htmlentities"
+# 从本质上讲，所有的WEB页面都是一个页面(Page)
+# 每个页面拥有一些属性，比如(encoding,title,url)
+# 每个页面有我们感兴趣的信息，我们需要提取出来
+# 比如列表页面我们敢兴趣的是文章列表的URLS
+# 而文章显示页面，我们感兴趣的是一些具体的文章属性
+# 每个页面可以有一个 parent ，父页面(parent)
+# 页面可以从父页面继承一些属性，比如(encoding)
+# 按照这个原理，采集其实只需要一个类就可以实现
+# 现在的实现方式，就是使用了以上原理
+# 比如：
+# Spider::Page.new('http://www.google.com') do |page|
+#   page.pages do |page|
+#     # 这里的page 仍然是一个 page 的实例 但是URL是 www.google.com 上分析得到的
+#     # 默认的分析方式是使用 Hpricot 的 search("a") 方法
+#     # 然后分析页面，并提取 labels ，保存在 page 里面
+#     page.label "title",page.title
+#     page.label "body",page.title
+#     page.label "author",page.doc.search("#author").innderHTML
+#     # 如果还需要子页面，那么我们还可以这样
+#     page.pages /^http\:\/\// do |page|
+#       # 得到子页面
+#       # 可以继续处理
+#     end
+#   end
+# end
+# 以上是事例代码
+
+class Spider::Page
+
+  def self.coder
+    @coder ||= HTMLEntities.new
+  end
+
+  def coder
+    self.class.coder
+  end
+
+  def self.class_attribute(*args)
+    # class_attribute跟class_inheritable_accessor对待Array,Hash的方式还存在差异
+    # 现在还是得使用class_inheritable_accessor
+    class_inheritable_accessor *args # 目前还没找到更好的方式
+  end
+
+
+  extend ActiveModel::Callbacks
+
+
+  # define_model_callbacks :fetch,:save,:crawl
+  define_model_callbacks :fetch,:save,:crawl
+
+  attr_reader   :url,:doc
+  attr_accessor :title,:content_length,:response,:histories,:encoding,:parent
+
+  class_attribute :options
+  self.options = {}
+  self.options[:example_url] ||= []
+
+
+  SEPARATOR = "<!-- PAGINATE SEPARATOR -->"
+
+  @@paginate_symbol = "--NEXTPAGE--"
+  cattr_accessor :paginate_symbol
+
+  def self.delay(seconds)
+    before_fetch do |page|
+      logger.debug "#{Time.now.to_i}:delay #{seconds}"
+      sleep seconds
+      logger.debug "#{Time.now.to_i}:after delay"
+      true # continue
+    end
+  end
+  
+  def self.set_example_url(*urls)
+    self.options[:example_url] ||= []
+    self.options[:example_url] += urls
+  end
+  
+  def self.example_url
+    self.options[:example_url]
+  end
+
+
+  def self.encoding(new_encoding)
+    self.options['encoding'] = new_encoding
+  end
+  
+  def self.ignore_existing(ignore=true)
+    self.options[:ignore_existing] = ignore
+  end
+  
+  def ignore_existing?
+    !!options[:ignore_existing]
+  end
+
+  class_attribute :attribute_names
+  self.attribute_names = []
+
+  class_attribute :site
+
+  # 设置属性
+  # 所有设置的属性可以使用
+  # attributes 方法获取
+  # 并且保存页面的时候，会进入SpiderLabel的数据库库表中
+  # 是用来指定这个页面上有哪儿些需要的数据
+  def self.define_attributes(*attributes)
+    attributes.each do |attribute|
+      define_attribute attribute
+    end
+  end
+
+  def self.define_attribute(attribute)
+    self.attribute_names << attribute
+    self.attribute_names.uniq!
+    self.attribute_names.compact!
+    attribute
+  end
+
+  # 返回属性
+  # 如果定义了
+  # 比如
+  # class ShowPage < Spider::Page
+  #   define_attributes :title,:body
+  #   def body
+  #     "body"
+  #   end
+  # end
+  # show_page = ShowPage.new "http://www.powerapple.com"
+  # show_page.attributes # => {:title=>nil,:body=>"body"}
+  def attributes(reload=false)
+    hash = {}
+    attribute_names.each do |name|
+      hash[name] = self[name]
+    end
+    hash
+  end
+
+  # 对 <base href="xxxx" /> 的标记进行快捷获取
+  def base_href
+    doc.at("base").try(:attributes).try(:[],"href")
+  end
+
+
+  # 从url的query string中分析得到params
+  def params
+    Rack::Utils.parse_query(uri.query).tap do |r|
+      # r.symbolize_keys!
+      r.each_pair do |key,value|
+        r[key.to_sym] = value
+      end
+    end
+  end
+
+  # 提供对 attributes 的快捷访问
+  def [](*args)
+    read_attribute *args
+  end
+
+  # 提供对 attributes 的快捷访问
+  def []=(name,value)
+    @attributes[name] = value
+  end
+
+  # 开始爬行
+  def crawl(force=false)
+    
+    unless ignore_existing? || force
+      if Spider::SpiderPage.find_by_url(url)
+        logger.info "url: #{url} already exists. skip."
+        return
+      end
+    end
+    
+    logger.debug "#{self} before crawl"
+    run_callbacks :crawl do
+      save
+    end
+  end
+
+  def self.site
+    if name =~ /^([^:]+)/
+      begin 
+        self.site = "#{$1}::Site".constantize.instance
+      rescue Exception=>e
+        nil
+      end
+    end
+  end
+
+  def site
+    self.class.site
+  end
+
+
+  def self.separator
+    SEPARATOR
+  end
+
+  # 日志器
+  def logger
+    self.class.logger
+  end
+
+  # 日志器
+  def self.logger
+    Spider.logger
+  end
+
+  def request(*args)
+    run_callbacks(:fetch) do
+      fetch_content_from_url(*args)
+    end
+  end
+  
+
+  # url 是必须的，其他的可以作为选项
+  def initialize(url,options={})
+    options.reverse_merge! :debug=>false
+    @options = options
+    @parent  = options[:parent]
+    @encoding = options[:encoding] if options[:encoding]
+    @url = url.strip
+    @response = nil
+    @attributes = {}
+  end
+
+  # 是否调试
+  # 如果打开调试 ， 系统会输出更多的信息
+  def debug?
+    @options[:debug]
+  end
+
+  # 是从哪儿个页上过来的
+  # parent_page.pages '/xxxx/yyy.html' do |page|
+  #   page.parent # => parent_page
+  # end
+  def parent
+    @options[:parent]
+  end
+
+
+  alias_method :referer,:parent
+
+  # 返回一个array
+  # 包含所有的父页面
+  def parents
+    parents = []
+    i = self
+    while(p = i.parent)
+      if p
+        parents << p 
+        i = p
+      else
+        break
+      end
+    end
+    parents.compact.reverse
+  end
+
+  alias_method :histories,:parents
+
+  # 默认的网页编码
+  # 优先级没有在 pages 方法中指定的高
+  # Spider::Site.register 'site' do |site|
+  #   site.encoding = 'big5'
+  #   site.pages 'http://www.google.com' do |page|
+  #     page.encoding # => big5
+  #   end
+  #   site.pages 'http://www.baidu.com',:encoding=>"gbk" do |page|
+  #     page.encoding # => gbk
+  #   end
+  # end
+  def encoding
+    @encoding ||= (self.class.options['encoding'] || "utf-8")
+  end
+
+  def encoding=(encoding)
+    # 如果已经获取了内容了，得重置一下内容
+    if @doc || @content
+      @doc = @content = nil
+    end
+    @encoding = encoding
+  end
+
+
+  # all pictures on this page
+  # 返回页面中的所有图片
+  # page.images #=> ['http://www.google.com/logo.gif']
+  def images
+    imgs = []
+    elem = ndoc
+    elem = ndoc.search(options[:scope]) if options[:scope]
+    elem.search('img[@src]').each do |img|
+      src =n img.attributes['src'].value
+      imgs << src
+    end
+    imgs
+  end
+
+
+  def url=(new_url)
+    @doc = @content = nil
+    @attributes = {}
+    @attributes_loaded = false
+    @url = new_url
+  end
+
+
+  def fix_image_urls
+    doc.search('img[@src]').each do |img|
+      img.set_attribute 'src', fix_urls(img.attributes['src'])
+    end
+  end
+
+  # clone a new page
+  def clone
+    self.class.new url,site
+  end
+
+  # return all links in this page
+  # 返回页面中的所有链接
+  # :scope=>"#pagination"
+  # 如果指定了:scope,则会在document的子元素中查询
+  def links(options={},&block)
+    [].tap do |urls|
+      elem = ndoc
+      elem = ndoc.search(options[:scope].strip) if options[:scope] # strip 很重要，如果末尾有空格会导致错误
+      elem.search('a[@href]').each do |a|
+        url = a.attributes['href'].value.strip
+        urls << url unless url.empty? || url =~ /^\s*javascript:/
+      end
+      fix_urls(urls)
+    end
+  end
+
+  # 过滤
+  def label(method,options={})
+    object = send(method)
+    object = case object
+    when String
+      object
+    when Symbol
+      object
+    when Hpricot::Elements
+      object.inner_html
+    when Nokogiri::XML::Element
+      object.to_html
+    else
+      object
+    end
+    [options[:filter]].flatten.uniq.compact.each do |filter|
+      object = case filter 
+      when :javascript
+        elem = Nokogiri::HTML.fragment(object)
+        elem.search("script").remove
+        elem.to_html
+      when :css
+        elem = Nokogiri::HTML.fragment(object)
+        elem.search("style").remove
+        elem.to_html
+      when :tags
+        object.gsub(/<.+?>/,"")
+      end
+    end
+    object
+  end
+
+  # 当前支持所有过滤
+  def filter(html,filters=[])
+    filters = [filters].flatten
+    filters.each do |f|
+      html = case f
+      when :javascript
+        elem = Nokogiri::HTML.fragment(html)
+        elem.search("script").remove
+        elem.to_html
+      when :css
+        elem = Nokogiri::HTML.fragment(html)
+        elem.search("style").remove
+        elem.to_html
+      when :tags
+        html.gsub(/<.+?>/,"")
+      else
+        html
+      end
+    end
+    html
+  end
+
+
+  # filters will all have a chance to process this page
+  # DEPRECATED
+  # 现在暂时没用了
+  def start
+    return
+    begin
+      # all done pages will not processed
+      raise Spider::PageExistsAndDoneException if Spider::SpiderPage.find_by_url_and_done(url,true)
+
+    rescue Exception=>e
+      logger.info "Exception(#{url}): #{e.message}"
+      logger.debug e.backtrace.join("\n")
+    end
+  end
+
+  # 页面内容
+  # 在没有调用这个方法之前
+  # page 是不会去获取 url 中的内容的
+  # page.url # 不会发送http请求的
+  # page.content # 这个时候才会发送http请求，获取页面内容
+  def content
+    @content ||= request
+  end
+
+  # 设置页面
+  def content=(content)
+    content = content.to_s
+    @content_length = content.size
+    @content = content
+  end
+
+  # 页面的标题
+  def title
+    begin
+      ndoc.search("title").inner_html
+    rescue
+      ''
+    end
+  end
+
+  # 返回一个Hpricot文档对象
+  def doc
+    begin
+      @doc ||= Hpricot(content)
+    rescue
+      @doc ||= Hpricot(content,:xml=>true)
+    end
+  end
+
+  # nokogiri doc
+  def ndoc
+    @ndoc ||= Nokogiri::HTML.fragment(content)
+  end
+
+  # 使用nokogiri的搜索方法
+  def nsearch(*args)
+    ndoc.search(*args)
+  end
+
+  # doc.search 的快捷方式
+  def search(*args)
+    doc.search(*args)
+  end
+
+  # 是否已经在系统中存在?
+  def exists?
+    !spider_page.nil?
+  end
+
+
+  def spider_page
+    Spider::SpiderPage.find_by_url(url)
+  end
+
+
+
+
+  # all Spider::Page instance
+  # if block given 
+  # yield the block with page, title
+  # 这个是最重要的方法之一
+  # page.url = "http://www.google.com/home/index.html"
+  # page.pages 'office.html','http://www.google.com/sport.html','/other.html' do |page|
+  #   # 上面三个被传递的URL
+  #   page.url #=> http://www.google.com/home/office.html,http://www.google.com/sport.html','http://www.google.com/other.html'
+  #   # 会以此传入以上url的page实例
+  # end
+  # 如果没有参数
+  # page.pages do |page|
+  #   # 这里会在 http://www.google.com/home/index.html 上的所有连接进行遍历
+  # end
+  # pages 选项
+  #
+  # *filter*
+  # page.pages :filter=>/google/ do |page|
+  #   # 只有符合 :filter 选项中的正则表达式的 url 才会被通过
+  # end
+  #
+  # *append*
+  # page.pages :append=>"http://www.google.com" do |page|
+  #   # 会在已有的URL列表之后，添加上 :append 选项指定的 url
+  #   # 此 参数 可以是一个 array
+  # end
+  # 
+  # *skip_exists*
+  # 一个布尔值，默认为 true 
+  # page.pages :skip_exists=>false do |page|
+  #   # 任何已经被处理过的 url 不会被排除
+  # end
+  def pages(*args)
+    options = args.extract_options!
+    options.reverse_merge! :uniq=>true,:class=>self.class
+    links = args.empty? ? self.links(:scope=>options[:scope]) : fix_urls(args.flatten)
+    logger.info "links before filter: #{links.inspect}" if debug?
+    filter = options[:filter]
+    # filter
+    case filter
+    when nil
+    when Regexp
+      links = links.find_all{|l| l =~ filter }
+    when String
+    when Array
+      #links = links.find_all{|l| filters.find{|f| l =~ f }
+    end
+    
+    # except
+    except = options[:except]
+    case except
+    when nil
+    when Regexp
+      links.reject!{|link| link =~ except }
+    when String
+    when Array
+    end
+
+    # append
+    #logger.info "links after filter: #{links.inspect}" if debug?
+
+    links += [options[:append]].flatten if options[:append]
+
+    #logger.info "links after append: #{links.inspect}" if debug?
+
+    links.uniq! if options[:uniq]
+
+    links.collect{|i| go(i,self.options.clone.merge(:parent=>self,:class=>options[:class])) }.tap do |pages|
+      if block_given?
+        pages.each do |page|
+          logger.debug "yield page: #{page.inspect}" if debug?
+          yield page
+        end
+      end
+    end
+  end
+
+
+  # save to database
+  # 保存进数据库 (SpiderPage类)
+  # 选项
+  # *save_labels*
+  # 是否同时将获得的 labels 保存进 数据库 (SpiderPageLabel类),默认为 true
+  # 一旦保存，下次调用 pages 方法的时候，会将该URL排除
+  def save(options={})
+    logger.info "saving page #{url}"
+    if exists?
+      page = spider_page
+    else
+      page = Spider::SpiderPage.new(options.merge(:url=>url,:site=>site.try(:id)))
+    end
+    page.content_length = content_length
+    #page.labels_hash = Digest::MD5.hexdigest(labels.to_yaml)
+    run_callbacks :save do
+      page.save unless debug?
+
+      # aotianlong:
+      # label 保存貌似没有多大意义，取消保存
+      #
+      # save labels
+      # attributes.each_pair do |name,value|
+      #  label = page.labels.find_or_initialize_by_name name.to_s
+      #  label.value = value.to_s
+      #  label.save unless debug?
+      # end
+      page
+    end
+  end
+
+
+  # 获得所属的("频道")的名称
+  # site.pages 'http://www.baidu.com',:name=>"baidu" do |page|
+  #   page.name # => "baidu"
+  #   page.pages [] do |page|
+  #     page.parent.name #=> "baidu"
+  #   end
+  # end
+  def name
+    @options[:name]
+  end
+
+
+  def full_name
+    "#{site.name}.#{name}"
+  end
+
+
+  # 模拟通过此页面点击进别的页面
+  # class 参数可以是 :list_page, :ListPage , Module::ListPage
+  # 都可以识别
+  def go(url,options={})
+    url = fix_urls(url)
+    klass = options[:class] || self.class
+    klass = case klass
+    when Symbol,String
+      klass_name = klass.to_s.classify
+      unless klass_name =~ /::/
+        klass_name = "#{self.class.parent.name}::#{klass_name}"
+      end
+      klass_name.constantize
+    else
+      klass
+    end
+    klass.new url,{:parent=>self}.merge(options)
+  end
+
+  # 历史记录
+  def back(step=1)
+  end
+
+  def forward(step=1)
+  end
+
+
+  # download a file from internet
+  # example:
+  # Spider::Page.download "http://powerapple.com/logo.gif",:to=>"/tmp/logo.gif"
+  def self.download(url,options={})
+    options.reverse_merge! :to=>File.basename(url),:method=>:get,:params=>{}
+    options[:method] = :get unless [:get,:post,:delete,:head].include?(options[:method])
+    to = options.delete :to
+    response = Spider::Http.send(options[:method],url,options)
+    dir = File.dirname(to)
+    FileUtils.mkdir_p dir unless File.directory?(dir)
+    File.open(to,"w+"){|f| f.write response }
+    to
+  end
+
+  # copy file
+  # example:
+  # Spider::Page.cp "/tmp/test.gif","/tmp/test/test/test/test/test.gif"
+  # destination directory if not exists , it will create automatically
+  def self.cp(src,dest,options={})
+    if File.exists?(src)
+      dir = File.dirname dest
+      FileUtils.mkdir_p dir unless File.directory?(dir)
+      logger.debug "file cp : #{src} -> #{dest}"
+      FileUtils.cp src,dest
+    end
+  end
+  
+  
+  def write_attribute(name,value)
+    @attributes[name] = value
+  end
+
+  def read_attribute(name,reload = false)
+    begin
+      send(name) if respond_to?(name)
+    rescue Exception=>e
+    end
+  end
+
+  # 返回一个当前url的URI类实例
+  def uri
+    @uri ||= URI.parse(url)
+  end
+
+  def reload
+    self.url = url
+    request
+  end
+
+
+
+  private
+
+  # every "/event/xdfasdf.url" like url will change to
+  # http://host.com/event/xdfasdf.url
+  # url can be array or string
+  def fix_urls(url)
+    port = uri.port == 80 ? "" : ":#{uri.port}"
+    case url
+    when /^\//
+      # 不包含主机信息的绝对路径
+      # 
+      # url.replace "#{uri.scheme}://#{uri.host}#{port}/#{url}"
+      url = URI.escape url
+      url.replace uri.merge(url).to_s
+    when /^http:\/\//i
+      # 完整的URL
+      url
+    when Array
+      url.collect!{|u| fix_urls u }
+    else
+      # 不包含主机信息的相对路径
+      url = URI.escape url 
+      path = uri.merge(url).to_s
+      url.replace path
+    end
+    url
+  end
+
+  # this method to fetch content from url
+  # if you want edit user agent , and other http headers , you should edit this method.
+  def fetch_content_from_url(options={})
+    method = options.delete :method
+    method = :get unless [:get,:head,:delete,:post,:put].include? method
+    body = ''
+    begin
+      logger.info "fetch content from url: #{url},method: #{method},options: #{options.inspect}"
+      logger.debug "httparty options: #{Spider::Http.default_options.inspect}"
+      logger.debug "cookies: #{Spider::Http.cookies.inspect}"
+      response = Spider::Http.send(method,url,options)
+      @response = response
+      if RUBY_VERSION >= "1.9" # ruby 1.9 encoding problem
+        body = response.body.force_encoding("utf-8")
+      else
+        body = response.body
+      end
+    # rescue Interrupt=>e
+    #  exit
+    rescue Exception=>e
+      #puts e.message
+      logger.error e.message
+      logger.error e.backtrace.join("\n")
+    end
+    @content = body.tap do |b|
+      if !encoding.blank? && !(encoding =~ /^utf(\-?)8$/i) && @response.content_type =~ /text/i
+        logger.info "iconv #{encoding} -> utf-8"
+        b.replace Iconv.iconv("utf-8//IGNORE","#{encoding}//IGNORE",b).join.to_s
+      end
+      @content_length = b.size
+      begin
+        code = @response.code
+      rescue Exception=>e
+        code = 0
+      end
+      logger.info "status:#{code},#{b.size} byte fetched."
+    end
+    @content
+  end
+
+
+  def fetch_content_from_url_with_cache(options={})
+    key = Digest::MD5.hexdigest(options.to_json.to_s) + "/" + Digest::MD5.hexdigest(url)
+    @content ||= Rails.cache.fetch key do
+      fetch_content_from_url_without_cache(options)
+    end
+    @content_length = @content.length
+    @content
+  end 
+
+  # alias_method_chain :fetch_content_from_url,:cache
+
+
+end