loyal_spider 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+# LoyalSpider
+
+这是一些对网站内容的一些蜘蛛爬虫类. 爬取网页的内容.等
+

data/Rakefile

@@ -0,0 +1,21 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'LoyalSpider'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+

data/lib/loyal_spider.rb

@@ -0,0 +1,13 @@
+# -*- encoding : utf-8 -*-
+require 'rest-client'
+require 'sanitize'
+require "rails_config"
+require "nokogiri"
+
+require "loyal_spider/config"
+require "loyal_spider/ables"
+require "loyal_spider/clients"
+require "loyal_spider/utils"
+
+module LoyalSpider
+end

data/lib/loyal_spider/ables.rb

@@ -0,0 +1,7 @@
+# -*- encoding : utf-8 -*-
+require "#{File.dirname(__FILE__)}/ables/fetch_able"
+require "#{File.dirname(__FILE__)}/ables/entity_able"
+require "#{File.dirname(__FILE__)}/ables/entity_lister_able"
+require "#{File.dirname(__FILE__)}/ables/fetch_options"
+require "#{File.dirname(__FILE__)}/ables/fetch_result"
+

data/lib/loyal_spider/ables/entity_able.rb

@@ -0,0 +1,86 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module EntityAble
+    # 有返回信息的
+    def self.included base
+      base.class_eval do
+        attr_accessor :title           # 标题
+        attr_accessor :url             # 标题
+        attr_accessor :content         # 正文
+        attr_accessor :tags            # 标签
+        attr_accessor :up_rating       # 好评数
+        attr_accessor :down_rating     # 差评数目
+        attr_accessor :comments_count  # 评论数目
+        attr_accessor :authors         # 抓取的作者信息
+        attr_accessor :publish_time    # 发布时间
+        attr_accessor :errors          # 错误
+
+        include InstanceMethods
+      end
+    end
+
+    module InstanceMethods
+      def initialize attrs={}
+        attrs.each do |key, value|
+          self.send(:"#{key}=", value)
+        end
+
+        self.errors = {}
+      end
+
+      def tags
+        @tags ||= []
+      end
+
+      def tags_array
+        @tags_array ||= self.tags.map{|_tag| _tag[:text] }
+      end
+
+      # 图片
+      def images
+        @images ||= Nokogiri::HTML.parse(self.content).css('img').map do |img_doc|
+          {
+            :src => img_doc.attr('src'),
+            :title => img_doc.attr('title'),
+            :alt   => img_doc.attr('alt')
+          }
+        end
+      end
+
+      def images?
+        self.images.any?
+      end
+
+      def authors
+        @authors ||= []
+      end
+
+      def authors_array
+        @authors_array ||= self.authors.map{|_author| _author[:text] }
+      end
+
+      def valid?
+        self.valid!
+        self.errors.empty?
+      end
+
+      def valid!
+        unless self.content.to_s.strip.size > 0
+          add_error :content, '不能为空'
+        end
+
+        unless self.url.to_s.strip.size > 0
+          add_error :url, '不能为空'
+        end
+      end
+
+      private
+
+      def add_error field, message
+        (self.errors[field] ||= []) << message
+      end
+      
+    end
+
+  end
+end

data/lib/loyal_spider/ables/entity_lister_able.rb

@@ -0,0 +1,119 @@
+# -*- encoding : utf-8 -*-
+## -*- encoding : utf-8 -*-
+module LoyalSpider
+  module EntityListerAble
+    # 有返回信息的
+    def self.included base
+      base.class_eval do
+        attr_writer :current_page        # current_page
+
+        include ::LoyalSpider::FetchAble
+        include InstanceMethods
+        extend  ClassMethods
+      end
+    end
+
+    module ClassMethods
+      # options:
+      #   - url_format
+      #   - url_format_first
+      #   - url_format_options
+      def config_loyal_spider_entity_lister options={}
+        @entity_lister_options ||= options
+
+        self.config_loyal_spider_default_fetch_options(
+          self.entity_lister_options.delete(:fetch_options) || {}
+        )
+      end
+
+      def paged_fetch page, options={}, &block
+        lister = self.new
+        lister.current_page = page
+        lister.fetch options, &block
+      end
+
+      def entity_lister_options
+        @entity_lister_options ||= {}
+      end
+
+      # 按页抓取
+      def paged_fetch page, options={}, &block
+        self.new.paged_fetch page, options, &block
+      end
+    end
+
+    module InstanceMethods
+      def paged_fetch page, options={}, &block
+
+        self.url_format_options.merge!(
+          options[:url_format_options] || {}
+        )
+
+        self.current_page = page
+        self.fetch options, &block
+      end
+
+      def url_format
+        @url_format ||= self.class.entity_lister_options[:url_format]
+      end
+
+      def url_format_first
+        @url_format_first ||= self.class.entity_lister_options[:url_format_first]
+      end
+
+      def url_format_options
+        @url_format_options ||= (self.class.entity_lister_options[:url_format_options] || {})
+      end
+
+      def _before_fetch options={}
+        @entities = []
+      end
+
+      def _after_fetch_success result
+        result.entities = self.entities
+      end
+
+      def entities
+        @entities ||= []
+      end
+
+      # TODO
+      def entity_clazz
+        self.class.entity_lister_options[:entity_clazz]
+      end
+
+      def new_entity attrs={}
+        if self.entity_clazz
+          self.entity_clazz.new(attrs) 
+        end
+      end
+
+      def add_entity entity
+        self.entities << entity
+      end
+
+      def current_page
+        @current_page ||= 1
+      end
+
+      def first_page?
+        self.current_page < 2
+      end
+
+      def fetch_url
+        return @fetch_url if defined?(@fetch_url)
+
+        _url_format = self.first_page? ? self.url_format_first : self.url_format
+
+        @fetch_url ||= sprintf(
+          _url_format, (
+            self.url_format_options || {}
+          ).merge(
+            :page => self.current_page
+          )
+        )
+      end
+    end
+
+  end
+end

data/lib/loyal_spider/ables/fetch_able.rb

@@ -0,0 +1,130 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module FetchAble
+    # 有返回信息的
+    def self.included base
+      base.class_eval do
+        attr_accessor :fetch_options
+
+        include InstanceMethods
+        extend  ClassMethods
+      end
+    end
+
+    module ClassMethods
+      # 配置蜘蛛的抓取配置
+      def config_loyal_spider_default_fetch_options options={}
+        @default_fetch_options ||= options
+      end
+
+      def default_fetch_options
+        @default_fetch_options ||= {}
+      end
+
+      def fetch options={}, &block
+        self.new.fetch options, &block
+      end
+    end
+
+    module InstanceMethods
+      def base_url
+        self.fetch_options.base_url
+      end
+
+      def fetch options={}, &block
+        self._before_fetch options if self.respond_to?(:_before_fetch, true)
+        self.before_fetch options if self.respond_to?(:before_fetch, true)
+
+        result = _perform_fetch options, &block
+
+        if result.success?
+          self.after_fetch_success(result) if self.respond_to?(:after_fetch_success, true)
+          self._after_fetch_success(result) if self.respond_to?(:_after_fetch_success, true)
+        else
+          self.after_fetch_fail(result) if self.respond_to?(:after_fetch_fail, true)
+          self._after_fetch_fail(result) if self.respond_to?(:_after_fetch_fail, true)
+        end
+
+        result
+      end
+
+      private
+
+      def _perform_fetch options={}, &block
+        _fetch_url = self.respond_to?(:fetch_url, true) ? self.fetch_url : self.url
+
+        @fetch_options = ::LoyalSpider::FetchOptions.new(
+          ::LoyalSpider::HashUtil.deep_merge(self.class.default_fetch_options, options).merge(
+            :url => _fetch_url
+          )
+        )
+
+        begin
+          ::RestClient::Request.execute @fetch_options.net_options do |response, request, response_result, &_block|
+            response_code = response.code
+
+            response_status = if (200..207).include?(response_code)
+                                :success
+                              elsif (300..307).include?(response_code)
+                                :redirect
+                              elsif (400..450).include?(response_code)
+                                :request_error
+                              else
+                                :server_error
+                              end
+
+            _result = ::LoyalSpider::FetchResult.new(
+              :response_status  => response_status,
+              :response_code    => response_code,
+              :response         => response.force_encoding(self.fetch_options.encoding_type).encode!('UTF-8'),
+              :request          => request,
+              :response_result  => response_result,
+              :fetch_options    => self.fetch_options,
+              :url              => _fetch_url
+            )
+
+            if block_given?
+              block.call _result, &_block
+            end
+
+            _result
+          end
+        rescue Exception => exception
+          _error_result = {
+            :response         => nil,
+            :request          => nil,
+            :response_result  => nil,
+            :fetch_options    => self.fetch_options,
+            :url              => _fetch_url,
+            :exception        => exception
+          }
+
+          case exception
+          when ::SocketError
+            _error_result.merge!(
+              :response_status  => :socket_error,
+              :response_code    => 502
+            )
+          else
+            _error_result.merge!(
+              :response_status  => :request_error,
+              :response_code    => 400
+            )
+          end
+
+          _result = ::LoyalSpider::FetchResult.new(
+            _error_result
+          )
+
+          if block_given?
+            block.call _result, &_block
+          end
+
+          _result
+
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/ables/fetch_options.rb

@@ -0,0 +1,38 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  class FetchOptions
+    attr_accessor :method             # 抓取方法
+    attr_accessor :open_timeout       # 打开超时配置
+    attr_accessor :timeout            # 打开超时配置
+    attr_accessor :encoding_type      # 编码
+    attr_accessor :headers            # 请求头
+    attr_accessor :url                # url
+    attr_accessor :base_url           # base_url
+
+    def initialize attrs={}
+      @url           = attrs[:url]       || ''
+      @base_url      = attrs[:base_url]  || ''
+      @method        = attrs[:method]    || :get
+      @timeout       = attrs[:timeout]   || 60  #  单位秒
+      @open_timeout  = attrs[:open_time] || @timeout
+      @encoding_type = attrs[:encoding_type] || 'UTF-8'
+      @headers = {
+        :accept_charset => 'UTF-8,*;q=0.5',
+        :accept_encoding => 'gzip,deflate,sdch',
+        :user_agent    => 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17'
+      }.merge(attrs[:headers] || {})
+    end
+
+    def net_options
+      {
+        :url            => self.url,
+        :method         => self.method,
+        :timeout        => self.timeout,
+        :open_timeout   => self.open_timeout,
+        :encoding_type  => self.encoding_type,
+        :headers        => self.headers
+      }
+    end
+
+  end
+end

data/lib/loyal_spider/ables/fetch_result.rb

@@ -0,0 +1,34 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  class FetchResult
+
+    attr_accessor :response_status
+    attr_accessor :response_code
+    attr_accessor :response
+    attr_accessor :request
+    attr_accessor :response_result
+    attr_accessor :fetch_options
+    attr_accessor :url
+    attr_accessor :exception
+    attr_accessor :entities
+
+    def initialize attrs={}
+      attrs.each do |key, value|
+        self.send(:"#{key}=", value)
+      end
+    end
+
+    def success?
+      self.response_status == :success
+    end
+
+    def fail?
+      !success?
+    end
+
+    def response_html_doc
+      @response_html_doc ||= Nokogiri::HTML.parse(self.response) if self.response
+    end
+
+  end
+end

data/lib/loyal_spider/clients.rb

@@ -0,0 +1,13 @@
+# -*- encoding : utf-8 -*-
+require "#{File.dirname(__FILE__)}/clients/kuaile_mahua/article_entity"
+require "#{File.dirname(__FILE__)}/clients/kuaile_mahua/article_entity_lister"
+
+require "#{File.dirname(__FILE__)}/clients/haha365/article_entity"
+require "#{File.dirname(__FILE__)}/clients/haha365/article_entity_lister"
+
+require "#{File.dirname(__FILE__)}/clients/lengxiaohua/article_entity"
+require "#{File.dirname(__FILE__)}/clients/lengxiaohua/article_entity_lister"
+
+require "#{File.dirname(__FILE__)}/clients/xiaohuadi/article_entity"
+require "#{File.dirname(__FILE__)}/clients/xiaohuadi/article_entity_lister"
+

data/lib/loyal_spider/clients/haha365/article_entity.rb

@@ -0,0 +1,11 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Haha365
+      class ArticleEntity
+        include ::LoyalSpider::EntityAble
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/haha365/article_entity_lister.rb

@@ -0,0 +1,94 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Haha365
+      class ArticleEntityLister
+        include ::LoyalSpider::EntityListerAble
+
+        self.config_loyal_spider_entity_lister :url_format_first => 'http://www.haha365.com/%{category}/',
+          :url_format => 'http://www.haha365.com/%{category}/index_%{page}.htm',
+          :entity_clazz => ::LoyalSpider::Clients::Haha365::ArticleEntity,
+          :fetch_options => {
+            :encoding_type => 'GBK',
+            :base_url => 'http://www.haha365.com'
+          }
+
+        # TODO:
+        def before_fetch options={}
+          puts "before_fetch: #{options}"
+        end
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          html_doc = result.response_html_doc
+
+          # _basic_doc = html_doc.css('html #main .content .left .r_c .cat_llb')
+          _basic_doc = html_doc.css('html #main .content .left .r_c')
+
+            # :content         # 正文
+            # :tags            # 标签
+            # :tags_text       # 标签
+            # :up_rating       # 好评数
+            # :down_rating     # 差评数目
+            # :comments_count  # 评论数目
+            # :authors         # 抓取的作者信息
+
+          (0...(_basic_doc.css('.cat_llb .fl a').size)).each do |_index|
+
+            _title_doc    = _basic_doc.css('.cat_llb h3 a')[_index]
+            _content_doc  = _basic_doc.css('.cat_llb #endtext')[_index]
+            _category_doc = _basic_doc.css('.cat_llb .fl a')[_index]
+
+            if _title_doc.nil? || _content_doc.nil? || _category_doc.nil?
+              next
+            end
+
+            _entity_attr = {}
+
+            _text_content = _content_doc.try :inner_html
+
+            _content = _text_content.to_s.split("<br>\r\n").map do |_cnt|
+              "<p>#{(Sanitize.clean _cnt).to_s.strip}</p>"
+            end.join('')
+
+            _entity_attr[:content] = _content
+
+            _entity_attr[:url] = "#{self.base_url}#{_title_doc.attr('href')}"
+            _entity_attr[:title] = "#{_title_doc.text}"
+
+            if _category_doc
+              _entity_attr[:tags] = [
+                {
+                  :text => _category_doc.text,
+                  :href => "#{self.base_url}#{_category_doc.attr('href')}"
+                }
+              ]
+            else
+              _entity_attr[:tags] = []
+            end
+
+            _entity_attr[:authors] = []
+
+            _entity_attr[:up_rating]      = -1
+            _entity_attr[:down_rating]    = -1
+            _entity_attr[:comments_count] = -1
+
+            _entity = self.new_entity(_entity_attr)
+
+            if _entity.valid?
+              self.add_entity _entity
+            end
+          end
+
+          # debugger
+        end
+
+        def after_fetch_fail result
+          puts "after_fetch fail: #{result}"
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/kuaile_mahua/article_entity.rb

@@ -0,0 +1,49 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module KuaileMahua
+      class ArticleEntity
+        include ::LoyalSpider::EntityAble
+        include ::LoyalSpider::FetchAble
+
+        self.config_loyal_spider_default_fetch_options(
+          :encoding_type => 'GBK',
+          :base_url => 'http://www.kl688.com'
+        )
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          html_doc = result.response_html_doc
+          entity_doc = html_doc.css('.main .main-left .xiaohua .xiaohua-data')
+
+          self.title = entity_doc.css('h1').first.text.to_s.strip
+          self.content = entity_doc.css('.content').inner_html
+          self.tags = entity_doc.css('.link .tags h4 a').map do |_tag_doc|
+            {
+              :text => _tag_doc.text.to_s.strip,
+              :href => "#{self.base_url}#{_tag_doc.attr('href').to_s.strip}"
+            }
+          end
+
+          self.tags_text = self.tags.map do |_tag|
+            _tag[:text]
+          end
+
+          self.authors = entity_doc.css('.link .tags .pusher a').map do |_author_doc|
+            {
+              :text => _author_doc.text.to_s.strip,
+              :href => "#{_author_doc.attr('href').to_s.strip}"
+            }
+          end
+
+          self.up_rating      = entity_doc.css('.tools li a.good').text.to_i
+          self.down_rating    = entity_doc.css('.tools li a.bad').text.to_i
+          self.comments_count = entity_doc.css('.tools li s').first.text.to_i
+
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/kuaile_mahua/article_entity_lister.rb

@@ -0,0 +1,85 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module KuaileMahua
+      class ArticleEntityLister
+        include ::LoyalSpider::EntityListerAble
+
+        self.config_loyal_spider_entity_lister :url_format_first => 'http://www.kl688.com/',
+          :url_format => 'http://www.kl688.com/newjokes/index_%{page}.htm',
+          :entity_clazz => ::LoyalSpider::Clients::KuaileMahua::ArticleEntity,
+          :fetch_options => {
+            :encoding_type => 'GBK',
+            :base_url => 'http://www.kl688.com'
+          }
+
+        # TODO:
+        def before_fetch options={}
+          puts "before_fetch: #{options}"
+        end
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          html_doc = result.response_html_doc
+
+          html_doc.css('.main .main-left .xiaohua').each do |entity_doc|
+            _entity_attr = {}
+
+            _fetch_options = result.fetch_options
+            _base_url      = _fetch_options.base_url.to_s.strip
+
+            _title_link = entity_doc.css('h3 a').first
+
+            # :content         # 正文
+            # :tags            # 标签
+            # :tags_text       # 标签
+            # :up_rating       # 好评数
+            # :down_rating     # 差评数目
+            # :comments_count  # 评论数目
+            # :authors         # 抓取的作者信息
+
+            _entity_attr[:url]   = "#{_base_url}#{_title_link.attr('href').to_s.strip}"
+
+            _entity_attr[:title] = _title_link.text
+            _entity_attr[:content] = entity_doc.css('.content').inner_html
+            _entity_attr[:tags] = entity_doc.css('.link .tags h4 a').map do |_tag_doc|
+              {
+                :text => _tag_doc.text.to_s.strip,
+                :href => "#{_base_url}#{_tag_doc.attr('href').to_s.strip}"
+              }
+            end
+
+            _entity_attr[:authors] = entity_doc.css('.link .tags .pusher a').map do |_author_doc|
+              {
+                :text => _author_doc.text.to_s.strip,
+                :href => "#{_author_doc.attr('href').to_s.strip}"
+              }
+            end
+
+            _entity_attr[:up_rating]      = entity_doc.css('.tools li a.good').text.to_i
+            _entity_attr[:down_rating]    = entity_doc.css('.tools li a.bad').text.to_i
+            _entity_attr[:comments_count] = entity_doc.css('.tools li s').first.text.to_i
+
+            _entity = self.new_entity(_entity_attr)
+
+            if entity_doc.css('.content .more a').any?
+              _entity.fetch
+            end
+
+            if _entity.valid?
+              self.add_entity _entity
+            end
+          end
+
+          # debugger
+        end
+
+        def after_fetch_fail result
+          puts "after_fetch fail: #{result}"
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/lengxiaohua/article_entity.rb

@@ -0,0 +1,20 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Lengxiaohua
+      class ArticleEntity
+        include ::LoyalSpider::EntityAble
+        # include ::LoyalSpider::FetchAble
+
+        # self.config_loyal_spider_default_fetch_options(
+        #   :encoding_type => 'UTF-8',
+        #   :base_url => 'http://lengxiaohua.com'
+        # )
+
+        def valid?
+          super
+        end
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/lengxiaohua/article_entity_lister.rb

@@ -0,0 +1,107 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Lengxiaohua
+      class ArticleEntityLister
+        include ::LoyalSpider::EntityListerAble
+
+        self.config_loyal_spider_entity_lister :url_format_first => 'http://lengxiaohua.com/',
+          :url_format => 'http://lengxiaohua.com/?page_num=%{page}',
+          :entity_clazz => ::LoyalSpider::Clients::Lengxiaohua::ArticleEntity,
+          :fetch_options => {
+            :encoding_type => 'UTF-8',
+            :base_url => 'http://lengxiaohua.com'
+          }
+
+        # TODO:
+        def before_fetch options={}
+          puts "before_fetch: #{options}"
+        end
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          html_doc = result.response_html_doc
+
+          html_doc.css('.joke_wrap li.joke_li').each do |entity_doc|
+            _entity_attr = {}
+
+            _fetch_options = result.fetch_options
+            _base_url      = _fetch_options.base_url.to_s.strip
+
+            # :content         # 正文
+            # :tags            # 标签
+            # :tags_text       # 标签
+            # :up_rating       # 好评数
+            # :down_rating     # 差评数目
+            # :comments_count  # 评论数目
+            # :authors         # 抓取的作者信息
+
+            _joke_id = entity_doc.css('.para_info .para_tool a').first.attr('jokeid')
+
+            _entity_attr[:url] = "#{_base_url}/joke/#{_joke_id}"
+
+            _entity_attr[:title] = ''
+
+            _text_content = entity_doc.css('.para_can pre').first.inner_html
+
+            _content = Sanitize.clean(_text_content).split(/\n/).map do |_cnt|
+              "<p>#{_cnt}</p>"
+            end.join('')
+
+            if _img_box = entity_doc.css('.default_load_imgbox').first
+
+              _image_content = _img_box.css('img').map do |_img|
+                "<img src='#{_img.attr('data-original').to_s.gsub('!water', '')}'/>"
+              end
+
+              _content = _content + "<p>#{_image_content.join('')}</p>" if _image_content.any?
+            end
+
+            _entity_attr[:content] = _content
+
+            _entity_attr[:tags] = entity_doc.css('.tag_box a').map do |_tag_doc|
+              {
+                :text => _tag_doc.text,
+                :href => "#{self.base_url}#{_tag_doc.attr('href')}"
+              }
+            end
+
+            # debugger
+
+            _author_doc = entity_doc.css('.para_info .user_info a').first
+
+            if _author_doc
+              _entity_attr[:authors] = [
+                {
+                  :text => _author_doc.text.to_s.strip,
+                  :href => "#{self.base_url}#{_author_doc.attr('href').to_s.strip}"
+                }
+              ]
+            else
+              _entity_attr[:authors] = []
+            end
+
+            _tool_doc = entity_doc.css('.para_tool')
+
+            _entity_attr[:up_rating]      = _tool_doc.css('a[report=like_joke] span').last.text.gsub(/\W/, '').to_i
+            _entity_attr[:down_rating]    = _tool_doc.css('a[report=unlike_joke] span').last.text.gsub(/\W/, '').to_i
+            _entity_attr[:comments_count] = _tool_doc.css("#show_comment_count_#{_joke_id}").text.gsub(/\W/, '').to_i
+
+            _entity = self.new_entity(_entity_attr)
+
+            if _entity.valid?
+              self.add_entity _entity
+            end
+          end
+
+        end
+
+        def after_fetch_fail result
+          puts "after_fetch fail: #{result}"
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/xiaohuadi/article_entity.rb

@@ -0,0 +1,66 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Xiaohuadi
+      class ArticleEntity
+        include ::LoyalSpider::EntityAble
+        include ::LoyalSpider::FetchAble
+
+        self.config_loyal_spider_default_fetch_options(
+          :encoding_type => 'GBK',
+          :base_url => 'http://www.xiaohuadi.com'
+        )
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          entity_doc = result.response_html_doc.css('.listx')
+
+          _fetch_options = self.fetch_options
+          _base_url      = _fetch_options.base_url.to_s.strip
+
+          # :content         # 正文
+          # :tags            # 标签
+          # :tags_text       # 标签
+          # :up_rating       # 好评数
+          # :down_rating     # 差评数目
+          # :comments_count  # 评论数目
+          # :authors         # 抓取的作者信息
+
+          _text_doc = entity_doc.css('.sonxltitle h1')
+
+          self.title = "#{_text_doc.text}"
+
+          _text_content = entity_doc.css('.sonxlarticle').first.inner_html
+
+          _content = _text_content.split("<br>\r\n").map do |_cnt|
+            "<p>#{Sanitize.clean _cnt}</p>"
+          end.join('')
+
+          self.content = _content
+
+          _category_doc = entity_doc.css('.sonxlPosition a').last
+
+          if _category_doc
+            self.tags = [
+              {
+                :text => _category_doc.text,
+                :href => "#{self.base_url}#{_category_doc.attr('href')}"
+              }
+            ]
+          else
+            self.tags = []
+          end
+
+          self.authors = []
+
+          self.up_rating      = -1
+          self.down_rating    = -1
+          self.comments_count = -1
+
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/clients/xiaohuadi/article_entity_lister.rb

@@ -0,0 +1,96 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  module Clients
+    module Xiaohuadi
+      class ArticleEntityLister
+        include ::LoyalSpider::EntityListerAble
+
+        self.config_loyal_spider_entity_lister :url_format_first => 'http://www.xiaohuadi.com/%{category}/',
+          :url_format => 'http://www.xiaohuadi.com/%{category}/index_%{page}.html',
+          :entity_clazz => ::LoyalSpider::Clients::Xiaohuadi::ArticleEntity,
+          :fetch_options => {
+            :encoding_type => 'GBK',
+            :base_url => 'http://www.xiaohuadi.com'
+          }
+
+        # TODO:
+        def before_fetch options={}
+          puts "before_fetch: #{options}"
+        end
+
+        # TODO
+        def after_fetch_success result
+          # puts "after_fetch success: #{result}"
+          html_doc = result.response_html_doc
+
+          html_doc.css('.ilistxllist>ul').each do |entity_doc|
+            _entity_attr = {}
+
+            _fetch_options = result.fetch_options
+            _base_url      = _fetch_options.base_url.to_s.strip
+
+            # :content         # 正文
+            # :tags            # 标签
+            # :tags_text       # 标签
+            # :up_rating       # 好评数
+            # :down_rating     # 差评数目
+            # :comments_count  # 评论数目
+            # :authors         # 抓取的作者信息
+
+            _link_doc = entity_doc.css('.ilistxlctlB1 a')
+
+            _entity_attr[:url] = "#{_base_url}#{_link_doc.attr('href')}"
+
+            _entity_attr[:title] = "#{_link_doc.text}"
+
+            _text_content = entity_doc.css('.ilistxlctlB2').first.inner_html
+
+            _content = _text_content.split("<br>\r\n").map do |_cnt|
+              "<p>#{Sanitize.clean _cnt}</p>"
+            end.join('')
+
+            _entity_attr[:content] = _content
+
+            _category_doc = entity_doc.css('.ilistxlctlC table td a').last
+
+            if _category_doc
+              _entity_attr[:tags] = [
+                {
+                  :text => _category_doc.text,
+                  :href => "#{self.base_url}#{_category_doc.attr('href')}"
+                }
+              ]
+            else
+              _entity_attr[:tags] = []
+            end
+
+            _entity_attr[:authors] = []
+
+            _tool_doc = entity_doc.css('.ilistxlctlA ul li')
+
+            _entity_attr[:up_rating]      = _tool_doc[1].text.to_i
+            _entity_attr[:down_rating]    = _tool_doc[2].text.to_i
+            _entity_attr[:comments_count] = _tool_doc[0].text.to_i
+
+            _entity = self.new_entity(_entity_attr)
+
+            if _entity.content.include?('未显示完，查看全文')
+              _entity.fetch
+            end
+
+            if _entity.valid?
+              self.add_entity _entity
+            end
+          end
+
+          # debugger
+        end
+
+        def after_fetch_fail result
+          puts "after_fetch fail: #{result}"
+        end
+
+      end
+    end
+  end
+end

data/lib/loyal_spider/config.rb

@@ -0,0 +1,20 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  class << self
+    attr_writer :config
+
+    def config
+      @config ||= Config.new
+    end
+
+    def configure
+      yield self.config ||= Config.new
+    end
+
+  end
+
+  class Config
+
+  end
+end
+

data/lib/loyal_spider/image.rb

@@ -0,0 +1 @@
+# -*- encoding : utf-8 -*-

data/lib/loyal_spider/utils.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+require "#{File.dirname(__FILE__)}/utils/hash_util"
+require "#{File.dirname(__FILE__)}/utils/array_util"
+

data/lib/loyal_spider/utils/array_util.rb

@@ -0,0 +1,16 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  class ArrayUtil
+    def self.extract_options!(arr)
+      if arr.last.is_a?(Hash)
+        arr.pop
+      else
+        {}
+      end
+    end
+
+    def self.init args
+      args.is_a?(Array) ? args : (args.nil? ? [] : [args])
+    end
+  end
+end

data/lib/loyal_spider/utils/hash_util.rb

@@ -0,0 +1,29 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  class HashUtil
+    def self.deep_merge!(a_hash, b_hash)
+      b_hash.each_pair do |k,v|
+        tv = a_hash[k]
+        a_hash[k] = tv.is_a?(Hash) && v.is_a?(Hash) ? self.deep_merge(a_hash, v) : v
+      end
+
+      a_hash
+    end
+
+    def self.deep_merge a_hash, b_hash
+      self.deep_merge! self.deep_dup(a_hash), b_hash
+    end
+
+    def self.deep_dup hash
+      duplicate = hash.dup
+
+      duplicate.each_pair do |k,v|
+        tv = duplicate[k]
+        duplicate[k] = tv.is_a?(Hash) && v.is_a?(Hash) ? (self.deep_dup(tv)) : v
+      end
+
+      duplicate
+    end
+
+  end
+end

data/lib/loyal_spider/version.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+module LoyalSpider
+  VERSION = "0.0.1"
+end

data/lib/tasks/loyal_spider_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :loyal_spider do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification
+name: loyal_spider
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- happy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails_config
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Description of LoyalSpider.
+email:
+- andywang7259@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/loyal_spider/ables/fetch_options.rb
+- lib/loyal_spider/ables/fetch_able.rb
+- lib/loyal_spider/ables/fetch_result.rb
+- lib/loyal_spider/ables/entity_lister_able.rb
+- lib/loyal_spider/ables/entity_able.rb
+- lib/loyal_spider/utils/hash_util.rb
+- lib/loyal_spider/utils/array_util.rb
+- lib/loyal_spider/config.rb
+- lib/loyal_spider/utils.rb
+- lib/loyal_spider/version.rb
+- lib/loyal_spider/image.rb
+- lib/loyal_spider/clients.rb
+- lib/loyal_spider/clients/kuaile_mahua/article_entity.rb
+- lib/loyal_spider/clients/kuaile_mahua/article_entity_lister.rb
+- lib/loyal_spider/clients/xiaohuadi/article_entity.rb
+- lib/loyal_spider/clients/xiaohuadi/article_entity_lister.rb
+- lib/loyal_spider/clients/haha365/article_entity.rb
+- lib/loyal_spider/clients/haha365/article_entity_lister.rb
+- lib/loyal_spider/clients/lengxiaohua/article_entity.rb
+- lib/loyal_spider/clients/lengxiaohua/article_entity_lister.rb
+- lib/loyal_spider/ables.rb
+- lib/loyal_spider.rb
+- lib/tasks/loyal_spider_tasks.rake
+- MIT-LICENSE
+- Rakefile
+- README.md
+homepage: http://github.com/xiuxian123
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2402893976680931226
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2402893976680931226
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Summary of LoyalSpider.
+test_files: []