spider_bot 0.0.4

data/lib/spider_bot/http/client.rb

@@ -0,0 +1,166 @@
+#coding: utf-8
+
+module SpiderBot
+  module Http
+    class Client
+
+      # return url for HttpClient
+      attr_reader :url
+
+      # return http user_agent for HttpClient
+      attr_reader :user_agent
+
+      attr_reader :headers
+
+      #
+      attr_accessor :options
+
+      # return connection for HttpClient
+      attr_accessor :connection
+
+      attr_accessor :conn_build
+
+      # Supported User-Agent
+      #
+      # * Linux Firefox (3.6.1)
+      # * Linux Konqueror (3)
+      # * Linux Mozilla
+      # * Linux Chrome
+      # * Mac Firefox
+      # * Mac Mozilla
+      # * Mac Chrome
+      # * Mac Safari
+      # * Mechanize (default)
+      # * Windows IE 6
+      # * Windows IE 7
+      # * Windows IE 8
+      # * Windows IE 9
+      # * Windows Mozilla
+      # * iPhone (3.0)
+      # * iPad
+      # * Android
+     
+      USER_AGENT = {
+        'bot' => "bot/#{SpiderBot::VERSION}",
+        'Linux Firefox' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.1) Gecko/20100122 firefox/3.6.1',
+        'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+        'Linux Chrome' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624  Chrome/26.0.1410.43',
+        'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:35.0) Gecko/20100101 Firefox/35.0',
+        'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18',
+        'Mac Chrome' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36',
+        'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+        'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+        'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
+        'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
+        'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3',
+        'iPad' => 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10',
+        'Android' => 'Mozilla/5.0 (Linux; U; Android 3.0; en-us) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
+      }
+
+      # Initialize a new HttpClient
+      #
+      # @param uri [String] the uri with  
+      # @param options [Hash] the options to create a http with configure
+      # @option options [String] :header set the http request headers
+      # @yield [builder] 
+      #
+      # @example
+      #   http = HttpClient.new
+      # 
+      #   http = HttpClient.new do |http|
+      #     http.user_agent= "Mac Safri"
+      #     http.url= "http://example.com"
+      #   end
+     
+      def initialize(uri = nil, options = nil, &block)
+        @url = uri
+        @options = options
+        @user_agent ||= USER_AGENT['bot']
+        yield self if block_given?
+      end
+
+      def builder(&block)
+        @conn_build = block
+      end
+
+      # Set the url for HttpClient
+      #
+      # @param uri [String] the HttpClient url
+     
+      def url=(uri)
+        @conn = nil
+        @url = uri
+      end
+
+      # Set the headers for HttpClient
+      #
+      # @param headers [String] the HttpClient url
+      # @return [String]
+      
+      def headers=(headers)
+        @headers = headers.merge({"User-Agent" => user_agent})
+      end
+
+      # Set the user agent for HttpClient
+      #
+      # @param name [Symbol] the HttpClient user agent 
+      
+      def user_agent=(name)
+        @user_agent = USER_AGENT[name] || USER_AGENT['bot']
+      end
+
+      # The Faraday connection object
+      # @return [connection] The Faraday connection builder
+      
+      def connection
+        @connection ||= begin
+                          conn = Faraday.new(url: url)
+                          conn.build do |b|
+                            conn_build.call(b)
+                          end if conn_build
+                          conn
+                        end
+      end
+
+
+      # Make request with HttpClient
+      #
+      # @param verb [Symbol] verb one of :get, :post, :put, :delete
+      # @param uri [String] URL path for request
+      # @param query [Hash] additional query parameters for the URL of the request
+      
+      def request(verb, uri, query={})
+        verb == :get ? query_get = query : query_post = query
+        uri = connection.build_url(uri, query_get)
+
+        response = connection.run_request(verb, uri, query_post, headers) do |request|
+          yield request if block_given?
+        end
+        response = Response.new(response)
+        
+        case response.status
+        when 301, 302, 303, 307
+          request(verb, response.headers['location'], query)
+        when 200..299, 300..399
+          response
+        end
+      end
+
+      # Handle get request with HttpClient 
+      #
+      # @param uri [String] URL path for request
+      # @param query [Hash] additional query parameters for the URL of the request
+     
+      def get(uri, query = {}, &block) 
+        request(:get, uri, query, &block)
+      end
+
+      # Handle post request with HttpClient
+      # @param (see #get)
+      def post(uri, query = {}, &block)
+        request(:post, uri, query, &block)
+      end
+    end
+  end
+end

data/lib/spider_bot/http/response.rb

@@ -0,0 +1,83 @@
+module SpiderBot
+  module Http
+    class Response
+      attr_reader :response
+
+      CONTENT_TYPE = {
+        'application/json' => :json,
+        'application/x-www-form-urlencoded' => :html,
+        'text/html' => :html,
+        'text/javascript' => :json,
+        'text/xml' => :xml
+      }
+
+      PARSERS = {
+        :json => lambda{ |body| MultiJson.respond_to?(:adapter) ? MultiJson.load(body) : MultiJson.decode(body) rescue body},
+        :html => lambda{ |body| Nokogiri::HTML(body)},
+        :xml => lambda{ |body| MultiXml.parse(body) }
+      }
+
+      def initialize(response)
+        @response = response
+      end
+
+      def headers
+        response.headers
+      end
+
+      def body(options = {})
+        options = options || {}
+        decode(response.body, options)
+      end
+
+      def decode(body, options = {})
+        return '' if !body 
+        return body if json?
+        charset = body.match(/charset\s*=[\s|\W]*([\w-]+)/)
+        return body if charset[1].downcase == "utf-8"
+        charset_code = charset_covert(charset[1])
+        begin
+          if options[:encode]
+            return body.encode! "utf-8", options[:encode], {:invalid => :replace} 
+          end
+          body.encode! "utf-8", charset_code, {:invalid => :replace} 
+        rescue
+          body
+        end
+      end
+
+      def status
+        response.status
+      end
+
+      # Attempts to determine the content type of the response.
+      def content_type
+        ((response.headers.values_at('content-type', 'Content-Type').compact.first || '').split(';').first || '').strip
+      end
+
+      def json?
+        CONTENT_TYPE[content_type] == :json || !response.body.match(/\<html/)
+      end
+
+      def parser
+        type = CONTENT_TYPE[content_type]
+        type = :json if type == :html && !response.body.match(/\<.*html|/) 
+        type = :html if type.nil?
+        return type
+      end
+
+      def parsed
+        @parsed ||= PARSERS[parser].call(body)
+      end
+
+      def charset_covert(charset)
+        case charset
+        when "gb2312", "GB2312", "GBK"
+          "gbk"
+        else
+          charset
+        end
+      end
+    end
+  end
+end

data/lib/spider_bot/load.rb

@@ -0,0 +1,30 @@
+begin
+  require File.expand_path("./config/application")
+rescue LoadError => e
+  system_boot = File.expand_path("./config/boot.rb")
+  require system_boot if File.exist?(system_boot)
+end
+
+if defined?(Padrino)
+  puts "read padrino environment #{Padrino.env}" 
+  BOTDIR = Dir.glob("#{Padrino.root}/app/bots/**/*_bot.rb")
+  if Padrino.env != :development
+    SpiderBot::Logging.initialize_logger("#{Padrino.root}/log/spider.log")
+  end
+end
+
+if defined?(Rails)
+  class Railtie < Rails::Railtie
+    initializer "disable eager load" do |app|
+      app.config.eager_load = false
+    end
+  end
+  Rails.application.initialize!
+  puts "read rails environment #{Rails.env}"
+  BOTDIR = Dir.glob("#{Rails.root}/app/bots/**/*_bot.rb")
+  Rails.logger.level = Logger::WARN
+  if !Rails.env.development?
+    SpiderBot::Logging.initialize_logger("#{Rails.root}/log/spider.log")
+  end
+end
+

data/lib/spider_bot/logging.rb

@@ -0,0 +1,21 @@
+require 'logger'
+
+module SpiderBot
+  module Logging
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      oldlogger.close if oldlogger && !$TESTING 
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+  end
+end

data/lib/spider_bot/railte.rb

@@ -0,0 +1,6 @@
+module SpiderBot 
+  class Railtie < Rails::Railtie
+    initializer "load bots" do |app|
+    end
+  end
+end

data/lib/spider_bot/string/date.yml

@@ -0,0 +1,29 @@
+
+date:
+  time:
+    year: "年|year|years"
+    month: "月|month|months|mon"
+    week: "周|星期|week|weeks"
+    day: "日|天|day|days"
+    hour: "时|小时|時|小時|hour|hours|hr|hrs|h"
+    min: "分|分钟|minute|minutes|min|mins|m"
+    second: "秒|second|seconds|sec|secs|s"
+  month:
+    jan: "january|jan|一月"
+    feb: "february|feb|二月"
+    mar: "march|mar|三月"
+    apr: "april|apr|四月"
+    may: "may|五月"
+    jun: "june|jun|六月"
+    jul: "july|jul|七月"
+    aug: "august|aug|八月"
+    sep: "september|sep|九月"
+    oct: "october|oct|十月"
+    nov: "november|nov|十一月"
+    dec: "december|dec|十二月"
+  other:
+    ago: "ago|前|以前"
+    today: "today|今天"
+    am: "AM|am|上午"
+    pm: "PM|pm|下午"
+

data/lib/spider_bot/string/time.rb

@@ -0,0 +1,119 @@
+#coding: utf-8
+require "yaml"
+require "active_support/time"
+require 'tzinfo'
+
+DATE_CONFIG = YAML.load_file(File.expand_path("../date.yml", __FILE__))
+
+class String
+  
+  # Parse content to local time
+  #
+  # @param [String] zone time zone with site
+  def parse_time(zone = nil)
+    Time.zone = zone.nil? ? "UTC" : zone
+
+    @time_config = DATE_CONFIG["date"]["time"]
+    @month_config = DATE_CONFIG["date"]["month"]
+    @other_config = DATE_CONFIG["date"]["other"]
+    @time_str = @time_config.values.join("|")
+
+    @time_regex = %r"\d+[\s|\S]*(?:#{@time_str})\s*(?:#{@other_config["ago"]})"
+    @today_regex = %r"#{@other_config["today"]}\s*\d{1,2}:\d{1,2}"
+
+    case self 
+    when @time_regex
+      parse_date_ago
+    when @today_regex 
+      parse_today
+    else
+      parse_date
+    end
+
+  end
+
+  private
+
+  # Parse content if has keyword mean 'ago' 
+  def parse_date_ago
+    now = Time.zone.now 
+    regex_text = self.match(@time_regex)[0]
+    @time = case regex_text
+            when %r"#{@time_config["year"]}"
+              now.years_ago regex_text.match(/\d+/)[0].to_i
+            when %r"#{@time_config["month"]}"
+              now.months_ago regex_text.match(/\d+/)[0].to_i
+            when %r"#{@time_config["week"]}"
+              now.ago regex_text.match(/\d+/)[0].to_i * 60 * 60 * 24 * 7
+            when %r"#{@time_config["day"]}"
+              now.ago regex_text.match(/\d+/)[0].to_i * 60 * 60 * 24
+            when %r"#{@time_config["hour"]}"
+              now.ago regex_text.match(/\d+/)[0].to_i * 60 * 60
+            when %r"#{@time_config["min"]}"
+              now.ago regex_text.match(/\d+/)[0].to_i * 60 
+            when %r"#{@time_config["second"]}"
+              now.ago regex_text.match(/\d+/)[0].to_i
+            else
+              raise "get date errors"
+            end
+  end
+
+  # Parse content if has keyword mean 'today'
+  def parse_today
+    now = Time.zone.now 
+    regex_text = self.match(/\d{1,2}\s*:\s*\d{1,2}:*\d{0,2}/)[0]
+    time_str = now.to_date.to_s + " " + regex_text
+    Time.zone.parse(time_str)
+  end
+
+  def parse_date
+    date_regex1 = %r"(\d{4})[^\d|:]{1,2}(\d{1,2})[^\d|:]{1,2}(\d{1,2})"
+    date_regex2 = %r"(\d{1,2})[^\d|:]{1,2}(\d{1,2})[^\d|:]{1,2}(\d{4})"
+    date_regex3 = %r"([\w|\W]+)[^\d|\w]{1,2}(\d{1,2})[^\d|:]*(\d{4})"
+    time = self.match %r"\d{1,2}\s*:\d{1,2}\s*:*\d{0,2}(?:#{@other_config["am"]}|#{@other_config["pm"]})*"
+      time = time[0].gsub(%r"#{@other_config["am"]}","am").gsub(%r"#{@other_config["pm"]}","pm") if time
+
+    case self
+    when date_regex1
+
+      date_text = self.match date_regex1
+
+      Time.zone.parse "#{date_text[1]}-#{date_text[2]}-#{date_text[3]} #{time}"
+    when date_regex2
+      date_text = self.match date_regex2
+      Time.zone.parse("#{date_text[3]}-#{date_text[1]}-#{date_text[2]} #{time}")
+    when date_regex3
+      date_text = self.match date_regex3
+      month = case date_text[1].downcase
+              when %r"#{@month_config["jan"]}"
+                1
+              when %r"#{@month_config["feb"]}"
+                2
+              when %r"#{@month_config["mar"]}"
+                3
+              when %r"#{@month_config["apr"]}"
+                4
+              when %r"#{@month_config["may"]}"
+                5
+              when %r"#{@month_config["jun"]}"
+                6
+              when %r"#{@month_config["jul"]}"
+                7
+              when %r"#{@month_config["aug"]}"
+                8
+              when %r"#{@month_config["sep"]}"
+                9
+              when %r"#{@month_config["oct"]}"
+                10
+              when %r"#{@month_config["nov"]}"
+                11
+              when %r"#{@month_config["dec"]}"
+                12
+              end
+      Time.zone.parse "#{date_text[3]}-#{month}-#{date_text[2]} #{time}"
+    else
+      Time.zone.parse(self)
+    end
+  end
+end
+

data/lib/spider_bot/version.rb

@@ -0,0 +1,3 @@
+module SpiderBot
+  VERSION = "0.0.4"
+end

data/lib/spider_bot.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "faraday"
+require 'uri'
+require "nokogiri"
+require "multi_json"
+require "multi_xml"
+require 'active_support/core_ext/string/conversions'
+require 'spider_bot/logging'
+require "spider_bot/version"
+
+module SpiderBot
+  class << self
+    def crawl(url, options = {}, &block)
+      crawl_instance = Crawl.new(url, options)
+      return crawl_instance.crawl_data if !block_given?
+      crawl_instance.instance_eval &block
+    end
+
+    def logger
+      SpiderBot::Logging.logger
+    end
+
+    def logger=(log)
+      SpiderBot::Logging.logger = log
+    end
+  end
+
+  autoload :Crawl, 'spider_bot/crawl'
+  autoload :Base, 'spider_bot/base'
+  module Http
+    autoload :Client, 'spider_bot/http/client'
+    autoload :Response, 'spider_bot/http/response'
+  end
+  autoload :Engine, 'spider_bot/engine'
+end
+
+require 'spider_bot/railte' if defined?(Rails) 

data/spider_bot.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spider_bot/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "spider_bot"
+  spec.version       = SpiderBot::VERSION
+  spec.authors       = ["yee.li"]
+  spec.email         = ["yeeli@outlook.com"]
+  spec.summary       = %q{splider bot}
+  spec.description   = %q{splider bot}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "faraday"
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "multi_json"
+  spec.add_dependency "multi_xml"
+  spec.add_dependency "tzinfo"
+  spec.add_dependency "thor"
+  spec.add_dependency 'daemons'
+end