spider2 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,15 @@
+SpiderFu
+========
+
+Introduction goes here.
+
+
+Example
+=======
+
+LaimaimaSpider.start
+
+Example goes here.
+
+
+Copyright (c) 2009 [name of plugin creator], released under the MIT license

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the spider_fu plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the spider_fu plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'SpiderFu'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/init.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+# Include hook code here
+require "spider"

data/install.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+# Install hook code here

data/lib/generators/spider/spider_generator.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require "rails/generators/active_record"
+class SpiderGenerator < ActiveRecord::Generators::Base
+
+  source_root File.expand_path('../templates', __FILE__)
+
+  def generate_files
+      template "site.rb",
+        File.join("spiders/#{file_name}","site.rb")
+      template "base_page.rb",
+        File.join("spiders/#{file_name}","base_page.rb")
+      template "show_page.rb",
+        File.join("spiders/#{file_name}","show_page.rb")
+      template "list_page.rb",
+        File.join("spiders/#{file_name}","list_page.rb")
+      template "index_page.rb",
+        File.join("spiders/#{file_name}","index_page.rb")
+
+      # test
+
+      template "test.rb",
+        File.join("test/spiders/#{file_name}","site_test.rb")
+      template "show_page_test.rb",
+        File.join("test/spiders/#{file_name}","show_page_test.rb")
+      template "list_page_test.rb",
+        File.join("test/spiders/#{file_name}","list_page_test.rb")
+      template "index_page_test.rb",
+        File.join("test/spiders/#{file_name}","index_page_test.rb")
+
+      # spec
+      template "site_spec.rb",
+        File.join("spec/spiders/#{file_name}","site_spec.rb")
+      template "show_page_spec.rb",
+        File.join("spec/spiders/#{file_name}","show_page_spec.rb")
+      template "list_page_spec.rb",
+        File.join("spec/spiders/#{file_name}","list_page_spec.rb")
+      template "index_page_spec.rb",
+        File.join("spec/spiders/#{file_name}","index_page_spec.rb")
+
+  end
+
+end

data/lib/generators/spider/templates/base_page.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+module <%= class_name %>
+  class BasePage < Spider::Page
+    encoding "utf-8"
+  end
+end

data/lib/generators/spider/templates/base_page_spec.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe <%= class_name %>::BasePage do
+
+  before :each do
+  end
+
+  # it "will correct set encoding" do
+  #   <%= class_name %>::BasePage.encoding.should == "gbk"
+  # end
+
+end

data/lib/generators/spider/templates/index_page.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+module <%= class_name %>
+  class IndexPage < BasePage
+    ignore_existing
+  end
+end

data/lib/generators/spider/templates/index_page_spec.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe <%= class_name %>::IndexPage do
+
+  before :each do
+    @index_page = <%= class_name %>::Site.index_page
+  end
+
+  # it "will correct set encoding" do
+  #   <%= class_name %>::IndexPage.encoding.should == "gbk"
+  # end
+
+end

data/lib/generators/spider/templates/index_page_test.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "test_helper"
+class <%= class_name %>::IndexPageTest < ActiveSupport::TestCase
+
+  test "true" do
+    assert true
+  end
+
+end
+

data/lib/generators/spider/templates/list_page.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+module <%= class_name %>
+  class ListPage < BasePage
+    ignore_existing
+  end
+end

data/lib/generators/spider/templates/list_page_spec.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe <%= class_name %>::ListPage do
+
+  before :each do
+    @url = "" # set list page url here
+    @list_page = <%= class_name %>::ListPage.new @url
+  end
+
+  # it "will correct set encoding" do
+  #   <%= class_name %>::IndexPage.encoding.should == "gbk"
+  # end
+
+  it "will return all show pages" do
+    show_pages = @list_page.show_pages
+    show_pages.each do |page|
+      # page.url.shuold 
+    end
+  end
+
+end

data/lib/generators/spider/templates/list_page_test.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "test_helper"
+class <%= class_name %>::ListPageTest < ActiveSupport::TestCase
+
+  test "true" do
+    assert true
+  end
+
+end
+

data/lib/generators/spider/templates/show_page.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+module <%= class_name %>
+  class ShowPage < BasePage
+    # 一些常用操作:
+    # define_attributes :title
+    # filter :title,:filters=>:default
+    # before_crawl{ do some thing}
+    # before_fetch{ do login }
+    # set_proxy "host",port
+    # validate_url :domain=>"powerapple.com",
+    #   :example=>'http://www.powerapple.com',
+    #   :match=>/powerapple/
+  end
+end

data/lib/generators/spider/templates/show_page_spec.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe <%= class_name %>::ShowPage do
+
+  before :each do
+    @url = "" # set show page url here
+    @show_page = <%= class_name %>::ShowPage.new @url
+  end
+
+  it "will parse title correct" do
+    @show_page.title.should == ""
+  end
+
+  it "will parse body correct" do
+    @show_page.body.should == ""
+  end
+
+end

data/lib/generators/spider/templates/show_page_test.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "test_helper"
+class <%= class_name %>::ShowPageTest < ActiveSupport::TestCase
+
+  test "true" do
+    assert true
+  end
+
+end
+

data/lib/generators/spider/templates/site.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+module <%= class_name %>
+  class Site < Spider::Site
+    set_name '<%= class_name %>'
+    set_id   '<%= file_name %>'
+  end
+end

data/lib/generators/spider/templates/site_spec.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe <%= class_name %>::Site do
+
+  before :each do
+  end
+
+  # it "will correct set encoding" do
+  #   <%= class_name %>::Site.encoding.should == "gbk"
+  # end
+
+end

data/lib/generators/spider/templates/test.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "test_helper"
+class <%= class_name %>::SiteTest < ActiveSupport::TestCase
+
+  test "true" do
+    assert true
+  end
+
+end
+

data/lib/generators/spider_migration/spider_migration_generator.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "rails/generators/active_record"
+class SpiderMigrationGenerator < ActiveRecord::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  def copy_files
+    migration_template 'migration.rb',"db/migrate/create_spider_pages_and_spider_page_labels"
+  end 
+
+end
+

data/lib/generators/spider_migration/templates/migration.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+class CreateSpiderPagesAndSpiderPageLabels < ActiveRecord::Migration
+
+  def self.up
+
+    create_table "spider_pages" do |t| 
+      t.string   "url"
+      t.boolean  "done",                         :default => false
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.integer  "labels_count",                 :default => 0
+      t.string   "labels_hash",    :limit => 32
+      t.boolean  "published",                    :default => false
+      t.boolean  "fragment",                     :default => false
+      t.integer  "content_length"
+      t.string   "site"
+    end 
+
+    add_index :spider_pages,:url
+    add_index :spider_pages,:site
+
+
+    create_table "spider_page_labels" do |t| 
+      t.string   "name"
+      t.integer  "page_id"
+      t.text     "value"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end 
+
+    add_index :spider_page_labels,:name
+    add_index :spider_page_labels,:page_id
+
+
+  end
+
+
+  def self.down
+    drop_table :spider_page_labels
+    drop_table :spider_pages
+  end
+end

data/lib/spider/active_record_methods.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+module Spider::ActiveRecordMethods
+
+  def self.included(base)
+    base.send(:extend,ClassMethods)
+    base.send(:include,InstanceMethods)
+  end
+
+  module ClassMethods
+
+    # 从url中创建
+    # 需要自身实现了 receive_spider_page 方法
+    # 才可以正确调用
+    def create_from_url(url,force=false)
+      if !force && ::SpiderPage.find_by_url(url)
+        raise "此URL已经存在于系统中了。"
+      end
+      # 找到能够处理 title跟body的page类
+      pages = Spider::Page.find_all_by_url(url)
+
+      object = nil
+      raise "没有找到合适的规则" if pages.empty?
+
+      Spider::Site.logger.info "采集单个#{human_name} #{url},找到 适合的规则 #{pages.inspect}"
+      pages.each do |page|
+
+        page.logger.info "使用 #{page} 的规则来尝试采集"
+        begin
+          spider_page = page.new(url)
+          results = spider_page.publish_to(self)
+          object = results.first
+        rescue Exception=>e
+          logger.error e.message
+          logger.error e.backtrace.join("\n")
+          object = nil
+        end
+        if object.try(:valid?)
+          page.logger.info "采集成功"
+          # 保存url
+          spider_page.save
+          break
+        else
+          page.logger.info "采集失败: #{object.try(:errors).try(:full_messages).try(:first)}"
+          object = nil
+        end
+      end
+
+      unless object
+        raise "采集器了找到了规则 #{pages.inspect}, 但是都失败了."
+      end
+      object
+
+    end
+
+  end
+
+  module InstanceMethods
+  end
+
+end

data/lib/spider/http.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+module Spider::Http
+  include HTTParty
+  headers "User-Agent"=>"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.6) Gecko/2009020518 Ubuntu/9.04 (jaunty) Firefox/3.0.6"
+  # debug_output Spider.logger
+
+  def self.with_options(new_options = {})
+    @options_mutex ||= Mutex.new
+    @options_mutex.synchronize do
+      original_default_options = self.default_options.deep_dup # deep_dup ,非常重要由active support提供
+      self.default_options.merge! new_options
+      begin
+        yield
+      rescue Exception => e
+        Spider.logger.error e.message
+        Spider.logger.error e.backtrace.join("\n")
+      ensure 
+        original_default_options[:headers].delete 'referer'
+        self.default_options = original_default_options
+      end
+    end
+  end
+
+  def self.with_proxy(ip,port,&block)
+    http_proxy ip,port
+    result = yield
+    clear_proxy
+    result
+  end
+
+  def self.clear_proxy
+    Spider::Http.default_options.delete :http_proxyaddr
+    Spider::Http.default_options.delete :http_proxyport
+  end
+
+=begin
+  def self.get(url,options = {})
+    open(url).read
+  end
+=end
+
+end
+

data/lib/spider/page/filter.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+# 过滤html用的
+module Spider::Page::Filter
+
+  def self.included(base)
+    base.class_eval do
+      send(:include,InstanceMethods)
+      send(:extend,ClassMethods)
+      alias_method_chain :read_attribute,:filter
+      class_attribute :attributes_filters
+      class_attribute :attributes_before_filters
+      class_attribute :attributes_after_filters
+      self.attributes_filters = {}
+      self.attributes_before_filters = {}
+      self.attributes_after_filters = {}
+    end
+  end
+
+  module ClassMethods
+
+    def default_filter(text)
+      sanitize(text)
+    end
+
+    def sanitizer
+      @sanitizer ||= HTML::WhiteListSanitizer.new
+    end
+
+    def sanitize(text,options={})
+      sanitizer.sanitize(text,options)
+    end
+
+    # class Sina::ShowPage < Sina::BasePage
+    #   filter :title,:filters=>[:default]
+    #   filter :title2,:filters=>:method_to_call_to_filter 
+    #   filter :title3,:filters=>lambda{|i| 'string to return '}
+    # end
+    def filter(*args,&block)
+      options = args.extract_options!
+      options[:filters] = block if block
+      options.assert_valid_keys :filters,:position
+      position = options[:position]
+      position = position.to_s + "_" if position
+      filter_attrs = send("attributes_#{position}filters")
+      logger.debug "create filter: #{name} : position : #{position},options : #{options.inspect}"
+      args.each do |attr_name|
+        filter_attrs[attr_name] ||= {}
+        # current_options[:filters] = options
+        filter_attrs[attr_name] = options #current_options
+      end
+      logger.debug "#{filter_attrs.inspect}"
+    end 
+
+
+    def filters
+      {:before=>attributes_before_filters,:middle=>attributes_filters,:after=>attributes_after_filters}
+    end
+
+  end
+
+  module InstanceMethods
+
+    
+    def read_attribute_with_filter(name,reload = false)
+      # 会经过过滤 
+      unless !reload && @attributes[name]
+        value = read_attribute_without_filter(name) 
+        value = filter_for_attribute(name,value)
+        @attributes[name] = value     
+      end
+      @attributes[name]
+    end
+
+    def default_filter(text)
+      self.class.default_filter text
+    end
+    
+    def sanitize(*args,&block)
+      self.class.sanitize(*args,&block)
+    end
+
+
+    def filter_for_attribute(name,text)
+      [self.attributes_before_filters,self.attributes_filters,self.attributes_after_filters].each do |filter_attrs|
+        filter_attrs.each_pair do |attr_name,filter|
+          logger.debug "#{attr_name} : #{filter.inspect}"
+          if attr_name == name
+            text = filter_text(text,filter[:filters])
+            logger.debug "#{name}(Filtered):#{text}"
+          end
+        end
+      end
+      text
+    end
+
+    # 执行过滤操作
+    def filter_text(text,filters=[])
+      [filters].flatten.each do |filter|
+        # 可以定义多个内建的过滤器
+        text = case filter
+        when :default
+          logger.debug "default filter"
+          default_filter(text)
+        when Symbol
+          logger.debug "symbol filter"
+          if respond_to?(filter)
+            send(filter,text)
+          else
+            text
+          end
+        when Proc
+          logger.debug "proc filter"
+          filter.bind(self).call(text)
+        else 
+          logger.debug "unknow filter"
+          text
+        end
+      end
+      text
+    end
+
+
+
+  end
+
+
+  def filters
+    self.class.filters
+  end
+
+  
+end

data/lib/spider/page/label.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+# 用来存储页面中分析到的有用信息
+module Spider::Page::Label
+  extend ActiveSupport::Concern
+
+  module InstanceMethods
+  end
+
+  module ClassMethods
+    def label(name,options = {},&block)
+      name = name.to_sym
+      define_attribute name
+      define_method(name) do |*args|
+        opts = args.extract_options!
+        instance_variable_name = "@__#{name}__"
+        if opts[:reload]
+          instance_variable_set(instance_variable_name,nil)
+        end
+        value = instance_variable_get(instance_variable_name)
+        unless value
+          value = block.bind(self).call
+          instance_variable_set(instance_variable_name,value)
+        end
+        value
+      end
+    end
+  end
+end