rails_spider 0.1.0

data/lib/rails_spider.rb

@@ -0,0 +1,10 @@
+require 'rails_spider/engine'
+
+require 'rails_spider/resource'
+
+require 'rails_spider/parser'
+require 'rails_spider/parser/szlawyers'
+
+module RailsSpider
+  # Your code goes here...
+end

data/lib/rails_spider/engine.rb

@@ -0,0 +1,9 @@
+module RailsSpider
+  class Engine < ::Rails::Engine
+    isolate_namespace RailsSpider
+
+    initializer 'rails_spider.assets.precompile' do |app|
+      app.config.assets.precompile += ['rails_spider_manifest.js']
+    end
+  end
+end

data/lib/rails_spider/fetchers.rb

@@ -0,0 +1,2 @@
+require 'rails_spider/fetchers/base'
+require 'rails_spider/fetchers/mechanize'

data/lib/rails_spider/fetchers/base.rb

@@ -0,0 +1,146 @@
+module RailsSpider
+  class Fetcher
+
+    def initialize
+      @page = ''
+    end
+
+    def event_class
+      @event_class = EventSpider.config.event_class.constantize
+    end
+
+    def page_by_url(url, proxy_hash=nil, header_hash=nil, repeat=5)
+      logger.info "Grab the page #{url}"
+      begin
+        change_another_proxy(proxy_hash, header_hash)
+        logger.info "Changed to a new proxy: #{@mechanize.proxy_addr}:#{@mechanize.proxy_port} for #{url}"
+        page = @mechanize.get(url)
+        logger.info "Has been get the page #{url}"
+        page
+      rescue => e
+        logger.error e.message
+        e.backtrace.each do |msg|
+          error_log.error msg
+        end
+        error_log.error "\n"
+        i ||= 0
+        if i < repeat
+          logger.info "Retry to get page for #{i} times"
+          i += 1
+          retry
+        else
+          if url.include?('douban')
+            source = 'douban'
+          elsif url.include?('weibo')
+            source = 'weibo'
+          elsif url.include?('rockbundartmuseum')
+            source = 'waitan'
+          elsif url.include?('citymoments')
+            source = 'citymoment'
+          else
+            source = 'else'
+          end
+          FailUrl.create(url: url, source: source, flag: "spider")
+          logger.warn "Cann't grab url #{url}"
+          return
+        end
+      end
+    end
+
+    def save_page(page)
+      begin
+        page.save_as("html/#{Date.today.to_s}/#{page.uri.to_s.split('http://').last.chomp('/')}")
+      rescue => e
+        logger.error e.message
+        logger.warn "cann't save page #{page.uri}"
+      end
+    end
+
+    def change_another_proxy(proxy_hash=nil, header_hash=nil)
+      if proxy_hash && proxy_hash[:ip] && proxy_hash[:port]
+        ip = proxy_hash[:ip]
+        port = proxy_hash[:port]
+      else
+        index = rand(@proxy.size)
+        ip = @proxy[index][:ip]
+        port = @proxy[index][:port]
+      end
+      @mechanize.set_proxy ip, port
+
+      @mechanize.request_headers = header_hash unless header_hash.nil?
+    end
+
+    def is_grab?(url)
+      event_class.where(url: url).exists? # 表示没有抓取
+    end
+
+    def run
+      logger.info "Start #{self.class} Spider..."
+
+      @links.each do |link|
+        #@city = link.values.first
+        grab_list_link(link.keys.first)
+      end
+
+      logger.info "End of #{self.class} Spider..."
+    end
+
+    def grab_update
+      logger.info "Start #{self.class} Spider grab_update."
+
+      @newlinks.each do |link|
+        @city = link['city'] unless link['city'].blank?
+        grab_list_link(link['url'])
+      end
+
+      logger.info "End of #{self.class} Spider grab_update."
+    end
+
+    def create_event(event_hash)
+      if event_hash.blank?
+        logger.warn "Cann't create event by blank data"
+        return
+      end
+      if is_existed?(event_hash)
+        logger.warn "Paramter:#{event_hash} has been existed cann't to create"
+        return
+      end
+      event = Event.new(event_hash)
+      if event_hash[:place].blank?
+        event.status = -1
+      end
+      event.kind_id = Kind.find_or_create_by(name: event_hash[:kind]).id unless event_hash[:kind].blank?
+      event.subkind_id = set_subkind_id(event_hash[:subkind]) unless event_hash[:subkind].blank?
+      if event_hash[:tags]
+        event_hash[:tags].each do |t|
+          EventTag.create(event_id: event.id, tag_id: Tag.find_or_create_by(name: t).id)
+        end
+      end
+      event.int_id = Event.max(:int_id).blank? ? 1 : Event.max(:int_id) + 1
+      event.save
+      unless event.errors.blank?
+        logger.info event.errors.full_messages.join(' / ')
+      else
+        logger.info 'Save event success'
+      end
+    end
+
+    def is_existed?(event_hash)
+      #if event_hash[:event_id] && event_class.where(event_id: event_hash[:event_id]).first
+      #  return true
+      #end
+      # TODO title and city are the same
+      #if event_hash[:title] && event_class.where(title: event_hash[:title]).first
+      #  return true
+      #end
+      if event_hash[:url] && event = event_class.where(url: event_hash[:url]).first
+        logger.warn "#{event_hash[:url]} has been exist in #{event.id}"
+        return true
+      end
+      return false
+    end
+
+    def keep_on?; return true end # keep on grab?
+
+  end
+end

data/lib/rails_spider/fetchers/mechanize.rb

@@ -0,0 +1,83 @@
+require 'mechanize'
+require 'rails_spider/fetchers/base'
+
+module RailsSpider
+  class Mechanize < Fetcher
+    attr_accessor :mechanize, :logger
+
+    def initialize
+      super
+      @mechanize = ::Mechanize.new
+      @mechanize.open_timeout = 20
+      @mechanize.pluggable_parser.default = @mechanize.pluggable_parser['text/html']
+      @logger = Logger.new STDOUT
+    end
+
+    def page(url)
+      mechanize.get(url)
+    end
+
+    def body(url)
+      page(url).search('body')
+    end
+
+    def links(url)
+      page(url).links.map do |link|
+        begin
+          link.resolved_uri.to_s
+        rescue ::Mechanize::UnsupportedSchemeError
+          ''
+        end
+      end
+    end
+
+    def change_another_proxy(proxy_hash=nil, header_hash=nil)
+      if proxy_hash && proxy_hash[:ip] && proxy_hash[:port]
+        ip = proxy_hash[:ip]
+        port = proxy_hash[:port]
+      else
+        index = rand(@proxy.size)
+        ip = @proxy[index][:ip]
+        port = @proxy[index][:port]
+      end
+      @mechanize.set_proxy ip, port
+
+      @mechanize.request_headers = header_hash unless header_hash.nil?
+    end
+
+    def is_grab?(url)
+      event_class.where(url: url).exists?
+    end
+
+    def run
+      logger.info "Start #{self.class} Spider..."
+
+      @links.each do |link|
+        #@city = link.values.first
+        grab_list_link(link.keys.first)
+      end
+
+      logger.info "End of #{self.class} Spider..."
+    end
+
+    def grab_update
+      logger.info "Start #{self.class} Spider grab_update."
+
+      @newlinks.each do |link|
+        @city = link['city'] unless link['city'].blank?
+        grab_list_link(link['url'])
+      end
+
+      logger.info "End of #{self.class} Spider grab_update."
+    end
+
+    def is_existed?(event_hash)
+      if event_hash[:url] && event = event_class.where(url: event_hash[:url]).first
+        logger.warn "#{event_hash[:url]} has been exist in #{event.id}"
+        return true
+      end
+      return false
+    end
+
+  end
+end

data/lib/rails_spider/fetchers/witar.rb

@@ -0,0 +1,73 @@
+require 'mechanize'
+
+module RailsSpider
+  class Mechanize < Fetcher
+    attr_accessor :mechanize, :logger
+
+    def initialize
+      super
+      @mechanize = Mechanize.new
+      @mechanize.open_timeout = 20
+      @mechanize.pluggable_parser.default = @mechanize.pluggable_parser['text/html']
+      @logger = Logger.new STDOUT
+    end
+
+    def save_page(page)
+      begin
+        page.save_as("html/#{Date.today.to_s}/#{page.uri.to_s.split('http://').last.chomp('/')}")
+      rescue => e
+        logger.error e.message
+        logger.warn "cann't save page #{page.uri}"
+      end
+    end
+
+    def change_another_proxy(proxy_hash=nil, header_hash=nil)
+      if proxy_hash && proxy_hash[:ip] && proxy_hash[:port]
+        ip = proxy_hash[:ip]
+        port = proxy_hash[:port]
+      else
+        index = rand(@proxy.size)
+        ip = @proxy[index][:ip]
+        port = @proxy[index][:port]
+      end
+      @mechanize.set_proxy ip, port
+
+      @mechanize.request_headers = header_hash unless header_hash.nil?
+    end
+
+    def is_grab?(url)
+      event_class.where(url: url).exists?
+    end
+
+    def run
+      logger.info "Start #{self.class} Spider..."
+
+      @links.each do |link|
+        #@city = link.values.first
+        grab_list_link(link.keys.first)
+      end
+
+      logger.info "End of #{self.class} Spider..."
+    end
+
+    def grab_update
+      logger.info "Start #{self.class} Spider grab_update."
+
+      @newlinks.each do |link|
+        @city = link['city'] unless link['city'].blank?
+        grab_list_link(link['url'])
+      end
+
+      logger.info "End of #{self.class} Spider grab_update."
+    end
+
+    def is_existed?(event_hash)
+      if event_hash[:url] && event = event_class.where(url: event_hash[:url]).first
+        logger.warn "#{event_hash[:url]} has been exist in #{event.id}"
+        return true
+      end
+      return false
+    end
+
+  end
+end

data/lib/rails_spider/parser.rb

@@ -0,0 +1,14 @@
+module RailsSpider
+  class Parser
+    attr_accessor :doc
+
+    def initialize(body)
+      @doc = Nokogiri::HTML(body)
+    end
+
+    def save
+      raise 'Should implement in subclass'
+    end
+
+  end
+end

data/lib/rails_spider/parser/szlawyers.rb

@@ -0,0 +1,26 @@
+module RailsSpider
+  class Szlawyers < Parser
+
+    def name
+      doc.at_css('span#lawlist_LawerName').text
+    end
+
+    def sex
+      doc.at_css('span#lawlist_LawerSex').text
+    end
+
+    def office
+      doc.at_css('span#lawlist_Enterprise').text
+    end
+
+    # 资格证号
+    def identify
+      doc.at_css('span#lawlist_LawerqualNo').text
+    end
+
+    def time
+      doc.at_css('span#lawlist_dtLawerqualNo').text
+    end
+
+  end
+end

data/lib/rails_spider/resource.rb

@@ -0,0 +1,58 @@
+require 'rails_spider/fetchers/mechanize'
+
+module RailsSpider
+  class Resource
+    attr_reader :fetcher, :work, :host, :item_path, :list_path, :page_params
+    attr_accessor :page
+    DEFAULT_EXP = "([^\/.?]+)"
+    SYMBOL_EXP = /:\w+/
+
+    def initialize(work, **options)
+      @work = work
+      @host = work.host
+      @list_path = work.list_path
+      @item_path = work.item_path
+      @page_params = work.page_params
+      @page = 1
+      @fetcher ||= RailsSpider::Mechanize.new
+    end
+
+    def run
+      items = get_items
+
+      while items.size > 0 do
+        items.each do |item|
+          save(item)
+        end
+        self.page += 1
+        items = get_items
+      end
+    end
+
+    def get_items
+      fetcher.links(list_url).select { |link| item_exp.match? link }
+    end
+
+    def save(url)
+      body = fetcher.body(url)
+      local = Local.find_or_initialize_by url: url, work_id: work.id
+      local.body = body
+      local.save
+    end
+
+    def list_url
+      list_url = URI.join host, list_path
+      if page.to_i > 0
+        page_query = URI.encode_www_form page_params => page
+        list_url.query = page_query
+      end
+
+      list_url
+    end
+
+    def item_exp
+      Regexp.new(item_path.gsub SYMBOL_EXP, DEFAULT_EXP)
+    end
+
+  end
+end

data/lib/rails_spider/strategies.rb

@@ -0,0 +1,6 @@
+require 'event_spider/strategies/base'
+require 'event_spider/strategies/douban'
+require 'event_spider/strategies/weibo'
+require 'event_spider/strategies/qunar'
+require 'event_spider/strategies/waitan'
+require 'event_spider/strategies/citymoment'

data/lib/rails_spider/version.rb

@@ -0,0 +1,3 @@
+module RailsSpider
+  VERSION = '0.1.0'
+end