spider2 0.0.1

data/lib/spider/site.rb

@@ -0,0 +1,225 @@
+# encoding: utf-8
+class Spider::Site
+  # include Singleton
+  cattr_accessor :sites,:publishers
+  @@sites = []
+
+  # 可以在rails的initializer目录下设置发布设置
+  # 比如
+  # Spider::Site.publishers = [Article]
+  # 这样 任何 Page 在 调用 publish 的时候都会 调用 Article.receive_spier_page 方法
+  # 参数是一个 page 自身
+  @@publishers = []
+  attr_accessor  :id,:labels,:blocks,:encoding,:disabled,:name,:index_url,:domains
+
+  class_attribute :attributes
+  self.attributes = {}
+
+  def initialize
+    # @pages = []
+    @domains = []
+  end
+
+  def self.id
+    instance.id
+  end
+  
+  def self.set_example_url(*args)
+    logger.warn "Spider::Site.set_example_url will take no effect, please remove."
+  end
+  
+  def self.set_index_url(url)
+    instance.index_url = url
+  end
+
+  def self.set_name(name)
+    instance.name = name
+  end
+
+  def self.set_domains(*domains)
+    instance.domains = domains
+  end
+
+  def self.set_domain(*args)
+    set_domains(*args)
+  end
+  
+  def self.index_page
+    instance.index_page
+  end
+
+  def self.valid_domain?(url)
+    instance.valid_domain?
+  end
+
+  def add_domain(domain)
+    domains.push(domain)
+    domains.uniq!
+    domains
+  end
+
+  def valid_domain?(url)
+    begin
+      host = URI(url).host
+      domains.any?{|domain| host.end_with?(domain) }
+    rescue Exception=>e
+      false
+    end
+  end
+
+  def pages
+    parent = self.class.parent
+    constants = parent.constants
+    @pages = constants.find_all{|i| i =~ /Page/}.reject!{|i| i =~ /^BasePage$/ }.collect{|i| "#{parent}::#{i}".constantize }
+  end
+  
+  def index_page
+    unless @index_page 
+      page = pages.find{|page| page.name =~ /IndexPage$/}
+      if page
+        @index_page ||= page.new index_url
+      end
+    end
+    @index_page
+  end
+
+  def self.pages
+    instance.pages
+  end
+
+  # 开始爬行单个url
+  # 成功，则返回一个Spider::Page实例
+  # 否则返回nil
+  def crawl
+    puts "#{self.class.name}#crawl not implement"
+  end
+
+  def self.crawl(url,options={})
+    sites.each do |site|
+      if site.crawl(url,options)
+        break
+      end
+    end
+  end
+
+  def self.set_id(id)
+    instance.id = id
+  end
+
+  class << self
+    alias_method :name=,:set_name
+    alias_method :id=,:set_id
+  end
+
+
+  def self.inherited(klass)
+    super
+    klass.send(:include,Singleton)
+    klass.instance.id = klass.name.tableize.split("/").first
+    @@sites << klass.instance
+  end
+
+  def self.channels
+    public_instance_methods - Spider::Site.public_instance_methods
+  end
+
+  def channels
+    self.class.channels
+  end
+
+  # 返回所有被注册了的站点名字
+  # Spider::Site.names #=> ['6park','sina','wenxuecity']
+  def self.names
+    sites.collect &:id
+  end
+
+  def logger
+    self.class.logger
+  end
+
+  def self.logger
+    Spider.logger
+  end
+
+
+  def start
+    crawl
+  end
+
+  def spider_pages(options={})
+    SpiderPage.scoped(:conditions=>{:site=>id})
+  end
+
+  # 当 site.enable = false 的时候
+  # site.start 不会有任何动作
+  def enable?
+    !disabled
+  end
+
+  # 是否禁用
+  # 对 enable? 的取反
+  def disabled?
+    !enable?
+  end
+
+  def self.start(name=nil,page_index=nil)
+    load_rules
+    if name.nil?
+      @@sites.each{ |site| site.start(page_index) }
+    else
+      self[name].start(page_index)
+    end
+  end
+
+
+  # find site by name
+  # eg:
+  # site = Spider::Site['Sina']
+  # site.start if site
+  # 获得已经注册了的站点
+  # Spider::Site['sina'] #=> #<Spider::Site name='sina'>
+  # 如果不存在则返回 nil
+  def self.[](id)
+    site = sites.find{|site| site.id == id.to_s}
+  end
+
+  def self.all
+    sites
+  end
+
+
+  def self.full_page_names
+    Spider::Site.all.collect{|site| site.name }.flatten
+  end
+
+  # 获得站点下的所有示例url
+  def example_url
+    pages.collect{|page| page.example_url }.flatten
+  end
+  
+  def self.example_url
+    instance.example_url
+  end
+
+  def self.find_pages(options={})
+    all_pages = Spider::Site.all.collect{|site| site.pages }.flatten
+    attributes = [options[:attributes]].flatten.compact
+
+    unless attributes.empty?
+      all_pages = all_pages.find_all{|page|
+        pass = true
+        attributes.each do |attribute|
+          unless page.attribute_names.include?(attribute)
+            pass = false
+            break
+          end
+        end
+        pass
+      }
+    end
+
+    all_pages
+
+  end
+
+end

data/lib/spider/spider_page.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+class Spider::SpiderPage < ActiveRecord::Base
+  validates_uniqueness_of :url
+  has_many :labels,:class_name=>"Spider::SpiderPageLabel",:foreign_key=>"page_id"
+
+  def labels_data
+    @labels_data ||= returning({}) do |hash|
+      labels.each{|label| hash[label.name] = label.value }
+    end
+  end
+
+  def has_label?
+    !labels_count.zero?
+  end
+
+  accepts_nested_attributes_for :labels
+
+end

data/lib/spider/spider_page_label.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+class Spider::SpiderPageLabel < ActiveRecord::Base
+  belongs_to :page,:class_name=>"Spider::SpiderPage",:foreign_key=>"page_id",:counter_cache=>"labels_count"
+  validates_uniqueness_of :name,:scope=>:page_id
+end

data/lib/spider/version.rb

@@ -0,0 +1,3 @@
+module Spider
+  VERSION = "0.0.1"
+end

data/lib/spider.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+#require "hpricot"
+module Spider
+  def self.logger
+    unless @logger
+      @logger = Logger.new(File.join(Rails.root,"log","spider_#{Rails.env}.log"))
+      @logger.level = Logger::DEBUG
+    end
+    @logger
+  end
+
+  def self.reload
+    error_occur = false
+    Dir[Rails.root.join("spiders","**","*.rb").to_s].each do |file|
+      begin 
+        load file
+      rescue Exception => e
+        error_occur = true
+      end
+    end
+    !error_occur
+  end
+
+end
+
+module Spider
+  module Sites
+    # 所有站点，都会使用这个namespace
+  end
+end
+
+require "spider/http"
+require "spider/site"
+require "spider/page"
+
+
+require "spider/page/filter"
+Spider::Page.send(:include,Spider::Page::Filter)
+
+require "spider/page/publish"
+Spider::Page.send(:include,Spider::Page::Publish)
+
+require "spider/page/validation"
+Spider::Page.send(:include,Spider::Page::Validation)
+
+require "spider/page/pagination"
+Spider::Page.send(:include,Spider::Page::Pagination)
+
+require "spider/page/proxy"
+Spider::Page.send(:include,Spider::Page::Proxy)
+
+require "spider/page/label"
+Spider::Page.send(:include,Spider::Page::Label)
+
+
+spiders_dir = File.join(Rails.root,"spiders")
+$:.push(spiders_dir)
+
+# define constants
+Dir[File.join(spiders_dir,"*")].each do |dir|
+  dir_name = dir.gsub(spiders_dir,"").gsub(/^\//,"")
+  Object.const_set(dir_name.classify,Module.new)
+end
+
+# 先包含初始化文件
+init_file = File.join(spiders_dir,"init.rb")
+require init_file if File.exists? init_file
+
+file_patten = File.join(spiders_dir,"**","*.rb")
+files = Dir[file_patten]
+
+site_files = files.find_all{|i| i =~ /site\.rb/}
+site_files.each{|i| require i}
+
+base_page_files = files.find_all{|i| i =~ /base_page\.rb/}
+base_page_files.each{|i| require i}
+
+files.each{|i| require i }
+
+# 包含 active record methods
+require "spider/active_record_methods"

data/lib/tasks/spider_tasks.rake

@@ -0,0 +1,86 @@
+# desc "Explaining what the task does"
+# task :spider_fu do
+#   # Task goes here
+# end
+
+
+desc "开始采集"
+namespace :spider do
+
+  namespace :proxy do
+    desc "test proxy PAGE=XXXX::BasePage" 
+    task :test => :environment do
+      klass = ENV['PAGE']
+      begin
+        klass = klass.constantize
+      rescue Exception => e
+        puts "unknow class `#{klass}`, please set a right spider page class to PAGE=XXXX::XxxPage"
+        exit
+      end
+      result = klass.validate_proxies
+      puts "valid proxies:"
+      result[:valid].each do |proxy|
+        puts proxy.join(":")
+      end
+      puts
+      puts "invalid proxies:"
+      result[:valid].each do |proxy|
+        puts proxy.join(":")
+      end
+    end
+  end
+
+  task :start=>:environment do
+puts "spider:start" => __LINE__
+    site = ENV['SITE']
+    if site.blank?
+      Spider::Site.all.each do |site|
+        ENV['SITE'] = site.id
+        Rake::Task['spider:site:start'].execute
+      end
+    else
+      Rake::Task['spider:site:start'].execute
+    end
+
+  end
+
+  namespace :site do
+    desc "开始单个网站"
+    task :start=>:environment do
+      site = ENV['SITE']
+      publishers = ENV['PUBLISHERS']
+      if publishers.blank?
+        puts "WARNING: PUBLISHERS is blank."
+        next
+      else
+        Spider::Page.publishers = publishers.split(",")
+        puts "Set publishers to #{Spider::Page.publishers.inspect}"
+      end
+      site = Spider::Site[site]
+      if site
+        puts site.name
+        puts "正在运行..."
+        site.start
+      else
+        puts "site(#{ENV['SITE']}) not found"
+      end
+    end
+
+  end
+
+
+  Rake::TestTask.new(:test => 'db:test:prepare') do |t| 
+    t.libs << 'test'
+    if ENV['SPIDER']
+      t.pattern = "test/spiders/#{ENV['SPIDER']}/*_test.rb"
+    else
+      t.pattern = 'test/spiders/**/*_test.rb'
+    end
+    t.verbose = true
+  end 
+
+  Rake::Task['spider:test'].comment = "测试spider规则"
+
+  # 按照规则 spider 名字，生成 test
+
+end

data/test/spider_fu_test.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require 'test_helper'
+
+class SpiderFuTest < ActiveSupport::TestCase
+  # Replace this with your real tests.
+  test "the truth" do
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+require 'rubygems'
+require 'active_support'
+require 'active_support/test_case'

data/uninstall.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+# Uninstall hook code here

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification 
+name: spider2
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- aotianlong
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-03-04 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rails
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 3
+        - 2
+        - 0
+        version: 3.2.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 59
+        segments: 
+        - 4
+        - 1
+        - 0
+        version: 4.1.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: a framework to crawl web pages
+email: 
+- aotianlong@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/generators/spider/spider_generator.rb
+- lib/generators/spider/templates/base_page.rb
+- lib/generators/spider/templates/base_page_spec.rb
+- lib/generators/spider/templates/index_page.rb
+- lib/generators/spider/templates/index_page_spec.rb
+- lib/generators/spider/templates/index_page_test.rb
+- lib/generators/spider/templates/list_page.rb
+- lib/generators/spider/templates/list_page_spec.rb
+- lib/generators/spider/templates/list_page_test.rb
+- lib/generators/spider/templates/show_page.rb
+- lib/generators/spider/templates/show_page_spec.rb
+- lib/generators/spider/templates/show_page_test.rb
+- lib/generators/spider/templates/site.rb
+- lib/generators/spider/templates/site_spec.rb
+- lib/generators/spider/templates/test.rb
+- lib/generators/spider_migration/spider_migration_generator.rb
+- lib/generators/spider_migration/templates/migration.rb
+- lib/spider/active_record_methods.rb
+- lib/spider/http.rb
+- lib/spider/page/filter.rb
+- lib/spider/page/label.rb
+- lib/spider/page/pagination.rb
+- lib/spider/page/proxy.rb
+- lib/spider/page/publish.rb
+- lib/spider/page/validation.rb
+- lib/spider/page.rb
+- lib/spider/site.rb
+- lib/spider/spider_page.rb
+- lib/spider/spider_page_label.rb
+- lib/spider/version.rb
+- lib/spider.rb
+- lib/tasks/spider_tasks.rake
+- MIT-LICENSE
+- Rakefile
+- init.rb
+- install.rb
+- README
+- uninstall.rb
+- test/spider_fu_test.rb
+- test/test_helper.rb
+homepage: http://www.powerapple.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: spider
+test_files: 
+- test/spider_fu_test.rb
+- test/test_helper.rb