serper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 149badc447dec8ed55714a9ed6768e3f85a9b94e
+  data.tar.gz: 8204acacee0b068b61421c0bc59507ea30717895
+SHA512:
+  metadata.gz: d329ccb1dbf584a4a100bf945740a5ff9ffc391a79f34aa598776cc5db901fdefdf0e04e26920060784c302fd9eb935dcf0a2a623a0acdda8b76bdc61c4235b2
+  data.tar.gz: 36a41f00ff396bde0e3c18587bd58359300591572ff346135f340520b16145272c622f228709d0fe72d770a2174f5bf48cc6c9e98e75fc3b22eef78e8ea14f41

data/README.md

@@ -0,0 +1 @@
+解析百度的搜索结果页面, 并返回结构化数据以进行后续分析.

data/bin/serper

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require 'serper'
+require 'optparse'
+require 'json'
+require 'pp'
+require 'docopt'
+
+cmd = File.basename(__FILE__)
+
+doc = <<DOCOPT
+1. serper -s 'keyword' # search 'keyword' and print parse result
+2. serper -s 'keyword' -o output.json # -o means save result to a file
+3. serper -f 'file path' # parse html source code from file
+4. serper -s 'keyword' -j # search 'keyword' and print parse result in JSON format
+
+Usage:
+  #{cmd} [options]
+
+Options:
+  -h --help               show this help message and exit
+  -v --version            show version and exit
+  -a --analyse Name       analyse as the given name
+  --keywords File         uses with -a, import give keywords File before search
+  -s --search Keyword     search Keyword and show result
+  -f --file File          parse local file or given url
+  -j --json               print JSON output
+  -o --output File         output JSON result to File
+
+DOCOPT
+
+begin
+  options = Docopt::docopt(doc, version: Serper::VERSION)
+  # pp options
+rescue Docopt::Exit => e
+  puts e.message
+end
+
+result = ''
+if options['--analyse']
+  analyse = Serper.analyse(options['--analyse'])
+  analyse.import_keywords(options('--keywords'))
+  analyse.search
+  result = 'Analyse finished!'
+elsif options['--search']
+  result = Serper.search options['--search']
+elsif options['--file']
+  result = Serper.parse_file options['--file']
+else
+  puts "At least given one of -a/-s/-f"
+end
+
+if options['--json']
+  puts result.to_json
+else
+  pp result
+end
+
+open(options['--output'],'w').puts result.to_json if options['--output']

data/lib/serper.rb

@@ -0,0 +1,26 @@
+require "serper/version"
+require "serper/parser"
+require "serper/analyser"
+
+[:baidu].each do |engine_name|
+  %w{crawler parser weight}.each do |part|
+    require File.expand_path("../serper/#{engine_name}/#{part}.rb",__FILE__)
+  end
+end
+
+module Serper
+  ENGINES = {
+      :baidu => Baidu
+  }
+
+  def self.search(engine_name,keyword,page=1)
+    serp = Parser.new(engine_name,keyword,page)
+    serp.search
+    serp
+  end
+
+  def self.analyse(connection)
+    Analyser.new(connection)
+  end
+end
+

data/lib/serper/analyser.rb

@@ -0,0 +1,112 @@
+require 'active_record'
+require 'csv'
+require 'date'
+require 'yaml'
+require 'ruby-progressbar'
+
+module Serper
+  class Analyser
+    def initialize(connection)
+      ActiveRecord::Base.establish_connection(connection)
+    end
+
+    def import_keywords(file)
+      CSV.foreach(file) do |l|
+        Keyword.find_or_create_by(:term => l[0]) do |r|
+          r.pv = l[1]
+          r.category = l[2]
+          r.url_type = l[3]
+          r.url_id = l[4]
+        end
+      end
+    end
+
+    def run(date=Date.today,skip=true)
+      puts "Serper Analyser on #{date}"
+      ENGINES.keys.each do |engine_name|
+        puts engine_name
+        search_engine(engine_name,date,skip)
+      end
+    end
+
+    def search_engine(engine_name,date,skip=true)
+      p = ProgressBar.create(:title => "Searching #{engine_name} - #{date}", :total => Keyword.all.count, :format => '%t (%c/%C) %a %E |%w')
+      Keyword.all.each do |k|
+        check_exists = Weight.where(:engine => engine_name, :date => date, :keyword_id => k.id)
+        if check_exists.any?
+          if skip
+            next
+          else
+            check_exists.destroy_all
+          end
+        end
+
+        serp = Serper.search(engine_name,k.term)
+        serp.weights.each do |w|
+          Weight.create(:date => date,
+                        :keyword_id => k.id,
+                        :engine => engine_name,
+                        :side => w[:side],
+                        :part => w[:part],
+                        :source => w[:type],
+                        :name => w[:name],
+                        :site => w[:site],
+                        :subdomain => w[:subdomain],
+                        :path => w[:path],
+                        :part_rank => w[:part_rank],
+                        :side_rank => w[:side_rank],
+                        :side_weight => w[:side_weight],
+                        :weight => w[:weight]
+          )
+        end
+
+        p.increment
+      end
+    end
+
+    def migrate!
+      ActiveRecord::Schema.define do
+        create_table :serper_keywords do |t|
+          t.string :term
+          t.integer :pv
+          t.string :category
+          t.string :url_type
+          t.integer :url_id
+
+          t.timestamps
+
+          t.index :term
+        end
+
+        create_table :serper_weights do |t|
+          t.date :date
+          t.string :engine
+          t.integer :keyword_id
+          t.string :side # Left Right
+          t.string :part
+          t.string :source # SEO SEM Special
+          t.string :name
+          t.string :site
+          t.string :subdomain
+          t.string :path
+          t.integer :part_rank
+          t.integer :side_rank
+          t.float :side_weight
+          t.float :weight
+
+          t.timestamps
+
+          t.index [:date, :engine, :keyword_id, :side, :side_rank], name: 'weights_pk_index'
+        end
+      end
+    end
+
+    class Keyword < ActiveRecord::Base
+      self.table_name = 'serper_keywords'
+    end
+
+    class Weight < ActiveRecord::Base
+      self.table_name = 'serper_weights'
+    end
+  end
+end

data/lib/serper/baidu/crawler.rb

@@ -0,0 +1,7 @@
+class Serper::Baidu
+  def serp_url(keyword,page)
+    keyword = keyword.gsub(" ","+")
+    page = page.to_i > 1 ? "&pn=#{page.to_i-1}0" : ''
+    URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8&inputT=#{1000+rand(1000)}")
+  end
+end

data/lib/serper/baidu/parser.rb

@@ -0,0 +1,185 @@
+class Serper::Baidu
+  def _parse_ads_right(file)
+    result = []
+    rank = 0
+
+    file[:doc].search('div#ec_im_container span a.c-icon.efc-cert').each do |div|
+      rank += 1
+      url = Addressable::URI.parse(Serper::Helper.parse_data_click(div['data-renzheng'])['identity']['a']['url']).query_values['wd'].to_s.sub('@v','') rescue ''
+      result << {url: url, rank: rank}
+    end
+    result
+  end
+
+  def _parse_ads_top(file)
+    result = []
+    rank = 0
+
+    file[:doc].search('div#content_left').first.children.each do |div|
+      break if div['id'].to_i > 0
+      div.search('span a.c-icon.efc-cert').each do |div|
+        rank += 1
+        url = Addressable::URI.parse(Serper::Helper.parse_data_click(div['data-renzheng'])['identity']['a']['url']).query_values['wd'].to_s.sub('@v', '') rescue ''
+        result << {url: url, rank: rank}
+      end
+    end
+    result
+  end
+
+  def _parse_con_ar(file)
+    result = []
+    divs = file[:doc].search("div#content_right div#con-ar").first
+    return [] if divs.nil?
+    divs.children.each do |div|
+      next unless div['class'].to_s.include?('result-op')
+      result << {:tpl => div['tpl'],
+                 :data_click => Serper::Helper.parse_data_click(div['data-click'])
+      }
+    end
+    result
+  end
+
+  # def _parse_pinpaizhuanqu(file)
+  #   part = file[:doc].search("div[@id='content_left']").first
+  #   return false if part.nil?
+  #
+  #   part.children[2].name == 'script'
+  # end
+
+  def _parse_ranks(file)
+    result = []
+    part = file[:doc].search("div[@id='content_left']").first
+    return result if part.nil?
+
+    part.children.each do |table|
+      next if table.nil?
+      id = table['id'].to_i
+      next unless id > 0 && id < 3000
+
+      r = {:rank => id}
+
+      r[:result_op] = table['class'].to_s.include?('result-op')
+
+      r[:fk] = table['fk']
+
+      r[:srcid] = table['srcid']
+
+      r[:tpl] = table['tpl']
+
+      r[:mu] = table['mu']
+
+      url = table.search('h3/a').first
+      unless url.nil?
+        url = url['href']
+        sleep(rand)
+        url = Serper::Crawler.get_rank_url('http:'+url).headers['location'] if url.include?('//www.baidu.com/link?')
+      end
+      r[:url] = url
+
+      r[:title] = Serper::Helper.get_content_safe(table.search('h3'))
+
+      r[:content] = Serper::Helper.get_content_safe(table.search('div.c-abstract'))
+
+      table.search('a').each do |link|
+        r[:baiduopen] = true if link['href'].to_s.include?('open.baidu.com')
+      end
+      r[:baiduopen] = false if r[:baiduopen].nil?
+
+      result << r
+    end
+    result
+  end
+
+  # def _parse_related_keywords(file)
+  #   result = []
+  #   file[:doc].search('div[@id="rs"]').each do |rs|
+  #     rs.css('a').each do |link|
+  #       result << link.content
+  #     end
+  #   end
+  #   result
+  # end
+
+  # def _parse_result_num(file)
+  #   html = file[:html]
+  #   str = html.scan(/百度为您找到相关结果(.*)个/).join
+  #   str = str.gsub('约','')
+  #   if str.include?('万')
+  #     parts = str.split('万')
+  #     result = parts[0].to_i * 10000 + parts[1].to_i
+  #   else
+  #     result = str.gsub(',', '').to_i
+  #   end
+  #
+  #   result
+  # end
+
+  # def _parse_right_hotel(file)
+  #   rh = file[:doc].search('div[@tpl="right_hotel"]')
+  #   return nil if rh.nil?
+  #
+  #   rh = rh.first
+  #   return nil if rh.nil?
+  #   title = Serper::Helper.get_content_safe(rh.search('div.opr-hotel-title'))
+  #
+  #   {:title => title}
+  # end
+
+  # def _parse_right_personinfo(file)
+  #   rp = file[:doc].search('div[@tpl="right_personinfo"]')
+  #   return nil if rp.nil?
+  #
+  #   title = Serper::Helper.get_content_safe rp.search('span.opr-personinfo-subtitle-large')
+  #   info_summary = Serper::Helper.get_content_safe rp.search('div.opr-personinfo-summary')
+  #   info = Serper::Helper.get_content_safe rp.search('div.opr-personinfo-info')
+  #   source = Serper::Helper.get_content_safe rp.search('div.opr-personinfo-source a')
+  #
+  #   return nil if title.nil? && info.nil? && source.nil?
+  #   {:title => title, :info_summary => info_summary, :info => info, :source => source}
+  # end
+
+  # def _parse_right_relaperson(file)
+  #   relapersons = file[:doc].search('div[@tpl="right_relaperson"]')
+  #   return nil if relapersons.nil?
+  #
+  #   result = []
+  #   relapersons.each do |rr|
+  #     title = rr.search('div.cr-title/span').first
+  #     title = title.content unless title.nil?
+  #     r = []
+  #     rr.search('p.opr-relaperson-name/a').each do |p|
+  #       r << p['title']
+  #     end
+  #     result << {:title => title, :names => r}
+  #   end
+  #   result
+  # end
+
+  # def _parse_right_weather(file)
+  #   rw = file[:doc].search('div[@tpl="right_weather"]')
+  #   return nil if rw.nil?
+  #
+  #   rw = rw.first
+  #   return nil if rw.nil?
+  #
+  #   title = Serper::Helper.get_content_safe(rw.search('div.opr-weather-title'))
+  #   week = rw.search('a.opr-weather-week').first['href']
+  #
+  #   {:title => title, :week => week}
+  # end
+
+  def _parse_zhixin(file)
+    result = []
+    file[:doc].search("div#content_left .result-zxl").each do |zxl|
+      result << {:id => zxl['id'],
+                 :srcid => zxl['srcid'],
+                 :fk => zxl['fk'],
+                 :tpl => zxl['tpl'],
+                 :mu => zxl['mu'],
+                 :data_click => Serper::Helper.parse_data_click(zxl['data-click'])
+      }
+    end
+    result
+  end
+
+end

data/lib/serper/baidu/weight.rb

@@ -0,0 +1,144 @@
+class Serper::Baidu
+  def weight_config
+    {
+        :left_parts => [:ads_top,
+                        :zhixin,
+                        :ranks
+        ],
+
+        :right_parts => [:con_ar,
+                         :ads_right
+        ],
+
+        :left_part_weight => 8,
+
+        :right_part_weight => 2,
+
+        :zhixin_weight => 3.5,
+
+        :baiduopen_weight => 3,
+
+        :rank_special_weight => 2,
+
+        :con_ar_weight => 2
+    }
+  end
+
+  # _weight_of_*** functions
+  # return a hash array
+  # each hash includes: type, name, site, weight
+
+  def _weight_of_ranks(serp_result,side_rank)
+    result = []
+    serp_result[:ranks].each.with_index do |rank,i|
+      side_rank += 1
+
+      url = rank[:url].to_s
+      mu = rank[:mu].to_s
+
+      type = 'SEO'
+      type = 'Special' if rank[:baiduopen]
+
+      unless mu.empty?
+        url = mu
+        type = 'Special'
+      end
+
+      site = Serper::Helper.parse_site(url)
+      subdomain = Serper::Helper.parse_subdomain(url)
+      path = Serper::Helper.parse_path(url)
+
+      name = rank[:tpl].to_s
+
+      weight = 1.0/side_rank.to_f
+      if type == 'Special'
+        if rank[:baiduopen]
+          weight = weight * weight_config[:baiduopen_weight].to_f
+        else
+          weight = weight * weight_config[:rank_special_weight].to_f
+        end
+      end
+
+      part_rank = rank[:rank]
+
+      result << {type: type, name: name, site: site, subdomain: subdomain, path: path, mu: mu, side_rank: side_rank, part_rank: part_rank, side_weight: weight}
+    end
+    [result, side_rank]
+  end
+
+  def _weight_of_ads_top(serp_result,side_rank)
+    result = []
+    serp_result[:ads_top].each.with_index do |ad,i|
+      side_rank += 1
+
+      url = ad[:url].to_s
+      type = 'SEM'
+      name = ''
+      site = Serper::Helper.parse_site(url)
+      subdomain = Serper::Helper.parse_subdomain(url)
+      path = Serper::Helper.parse_path(url)
+
+      part_rank = ad[:rank]
+
+      weight = 1.0/side_rank.to_f
+      result << {type: type, name: name, site: site, subdomain: subdomain, path: path, side_rank: side_rank, part_rank: part_rank, side_weight: weight}
+    end
+    [result, side_rank]
+  end
+
+  def _weight_of_ads_right(serp_result,side_rank)
+    result = []
+    serp_result[:ads_right].each.with_index do |ad,i|
+      side_rank += 1
+
+      url = ad[:url].to_s
+      type = 'SEM'
+      name = ''
+      site = Serper::Helper.parse_site(url)
+      subdomain = Serper::Helper.parse_subdomain(url)
+      path = Serper::Helper.parse_path(url)
+
+      part_rank = ad[:rank]
+
+      weight = 1.0/side_rank.to_f
+      result << {type: type, name: name, site: site, subdomain: subdomain, path: path, side_rank: side_rank, part_rank: part_rank, side_weight: weight}
+    end
+    [result, side_rank]
+  end
+
+  def _weight_of_con_ar(serp_result,side_rank)
+    result = []
+    serp_result[:con_ar].each.with_index do |con,i|
+      side_rank += 1
+
+      url = con[:data_click]['mu'].to_s
+      type = 'Special'
+      name = con[:tpl]
+      site = Serper::Helper.parse_site(url)
+      subdomain = Serper::Helper.parse_subdomain(url)
+      path = Serper::Helper.parse_path(url)
+
+      weight = 1.0 * weight_config[:con_ar_weight]
+      result << {type: type, name: name, site: site, subdomain: subdomain, path: path, side_rank: side_rank, part_rank: i+1, side_weight: weight}
+    end
+    [result, side_rank]
+  end
+
+  def _weight_of_zhixin(serp_result,side_rank)
+    result = []
+    serp_result[:zhixin].each.with_index do |zhixin,i|
+      side_rank += 1
+
+      url = zhixin[:mu].to_s
+      type = 'Special'
+      name = zhixin[:tpl]
+      site = Serper::Helper.parse_site(url)
+      subdomain = Serper::Helper.parse_subdomain(url)
+      weight = 1.0 * weight_config[:zhixin_weight]
+      path = Serper::Helper.parse_path(url)
+
+      result << {type: type, name: name, site: site, subdomain: subdomain, path: path, side_rank: side_rank, part_rank: i+1, side_weight: weight}
+    end
+    [result, side_rank]
+  end
+end

data/lib/serper/crawler.rb

@@ -0,0 +1,84 @@
+require 'httparty'
+
+module Serper
+  class Crawler
+    AllUserAgents = YAML.load(open(File.expand_path('../user_agents.yml',__FILE__)))
+
+    def self.rand_ua
+      AllUserAgents[rand(AllUserAgents.size)]
+    end
+
+    include HTTParty
+    base_uri 'www.baidu.com'
+    follow_redirects false
+    headers "User-Agent" => self.rand_ua, "Referer" => 'http://www.baidu.com/'
+
+    def self.get_serp(url,retries = 3)
+      self.new.get_serp(url,retries)
+    end
+
+    def self.get_rank_url(url)
+      self.new.get_rank_url(url)
+    end
+
+    def get_rank_url(url)
+      begin
+        response = self.class.get(url)
+      rescue StandardError => e
+        puts e.class
+        puts e.message
+        sleep(10)
+        retry
+      end
+      response
+    end
+
+    def get_serp(url, retries = 3)
+      if retries > 0
+        begin
+          response = self.class.get(url)
+        rescue StandardError => e
+          puts e.class
+          puts e.message
+          sleep(10)
+          retry
+        end
+
+        if response.code != 200
+          puts response
+          puts "Retry on URL: #{url}"
+          sleep(rand(60)+1200)
+          response = self.class.get_serp(url,retries - 1)
+        end
+
+        if response.nil?
+          puts "Still error after 3 tries, sleep 3600s now."
+          sleep(3600)
+          response = self.class.get_serp(url)
+        end
+
+        ##Baidu Stopped response Content-Length in headers...
+        #if response.headers['Content-Length'].nil?
+        #  puts "Can't read Content-Length from response, retry."
+        #  response = self.class.get_serp(url,retries-1)
+        #end
+        #
+        #if response.headers['Content-Length'].to_i != response.body.bytesize
+        #  issue_file = "/tmp/serper_crawler_issue_#{Time.now.strftime("%Y%m%d%H%M%S")}.html"
+        #  open(issue_file,'w').puts(response.body)
+        #  puts "Notice:"
+        #  puts "Serper get an error when crawl SERP: response size (#{response.headers['Content-Length']}) not match body size."
+        #  puts "Please see file #{issue_file} for body content."
+        #  puts "Sleep 10s and retry"
+        #  sleep(10)
+        #  response = self.class.get_serp(url)
+        #end
+
+        response
+      else
+        nil
+      end
+    end
+
+  end
+end

data/lib/serper/helper.rb

@@ -0,0 +1,79 @@
+require 'domainatrix'
+
+module Serper
+  module Helper
+    class << self
+      # get content safe from nokogiri search reasult
+      def get_content_safe(noko)
+        return nil if noko.nil?
+        return nil if noko.empty?
+        noko.first.content.strip
+      end
+
+      # parse data click value from baidu div property,
+      # which is a JSON like format
+      def parse_data_click(str)
+        JSON.parse(str
+                     .gsub("'",'"')
+                     .gsub(/({|,)([a-zA-Z0-9_]+):/, '\1"\2":')
+                     #.gsub(/'*([a-zA-Z0-9_]+)'*:/, '"\1":')
+                     #.gsub(/:'([^(',\")]*)'(,|})/,':"\1"\2')
+                   )
+      end
+
+      # normalize weight of given data,
+      # the data must be a hash array structure.
+      # for example : [{a: 1, b: 2}, {a: 2, b: 3}]
+      def normalize(data,weight_col=:weight,normalized_col=:normalized_weight)
+        total_weight = data.reduce(0.0) {|sum,d| sum += d[weight_col].to_f}
+        data.each do|d|
+          d[normalized_col] = d[weight_col].to_f/total_weight
+        end
+        data
+      end
+
+      def parse_site(url)
+        begin
+          url = Domainatrix.parse(url.to_s)
+          site = url.domain + '.' + url.public_suffix
+        rescue Exception => e
+          puts "parse_site from url error:"
+          puts url
+          puts e.class
+          puts e.message
+          site = ''
+        end
+        site
+      end
+
+      def parse_subdomain(url)
+        begin
+          url = Domainatrix.parse(url.to_s)
+          subdomain = url.subdomain
+        rescue Exception => e
+          puts "parse_site from url error:"
+          puts url
+          puts e.class
+          puts e.message
+          subdomain = ''
+        end
+        subdomain
+      end
+
+      def parse_path(url)
+        begin
+          url = Domainatrix.parse(url.to_s)
+          path = url.path
+        rescue Exception => e
+          puts "parse_site from url error:"
+          puts url
+          puts e.class
+          puts e.message
+          path = ''
+        end
+        path
+      end
+
+    end
+  end
+end

data/lib/serper/parser.rb

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+require 'nokogiri'
+require 'uri'
+require 'json'
+require 'serper/crawler'
+require 'serper/helper'
+
+module Serper
+  class Parser
+    attr_reader :engine_name, :keyword, :page, :html, :doc, :result
+
+    def initialize(engine_name,keyword,page=1)
+      @engine_name = engine_name
+      @engine = ENGINES[@engine_name].new
+      @keyword = keyword
+      @page = page
+    end
+
+    def serp_url
+      @engine.serp_url(@keyword,@page)
+    end
+
+    def search
+      html = Crawler.get_serp(serp_url).body
+      parse html
+    end
+
+    def parse(html)
+      html = html.encode!('UTF-8','UTF-8',:invalid => :replace)
+      @file = Hash.new
+      @result = Hash.new
+
+      @file[:html] = html
+      @file[:doc] = Nokogiri::HTML(html)
+
+      @engine.methods.each do |m|
+        next unless m =~ /^_parse_/
+        begin
+          @result[m.to_s.sub('_parse_','').to_sym] = @engine.send m,@file
+        rescue Exception => e
+          issue_file = "/tmp/serper_issue_#{Time.now.strftime("%Y%m%d%H%M%S")}.html"
+          open(issue_file,'w').puts(html)
+          puts "Notice:"
+          puts "Serper gem have a bug, please email to zmingqian@qq.com to report it."
+          puts "Please attach file #{issue_file} in the email and the error information below, thanks!"
+          puts e.message
+          puts e.inspect
+          puts e.backtrace
+          raise "Serper Parser Get An Error!"
+        end
+      end
+
+      @result
+    end
+
+
+    def weights
+      result = []
+      [:left,:right].each do |side|
+        side_rank = 0
+
+        @engine.weight_config["#{side}_parts".to_sym].each do |part|
+          rs,side_rank = @engine.send("_weight_of_#{part}",@result,side_rank)
+
+          rs.each do |r|
+            r[:side] = side.to_s
+            r[:part] = part
+
+            r[:weight] = r[:weight].to_f * @engine.weight_config["#{side}_part_weight".to_sym].to_f
+            result << r
+          end
+        end
+      end
+      Serper::Helper.normalize(result,:side_weight,:weight)
+    end
+  end
+end

data/lib/serper/user_agents.yml

@@ -0,0 +1,183 @@
+---
+- 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/527  (KHTML, like Gecko,
+  Safari/419.3) Arora/0.6 (Change: )'
+- Mozilla/5.0 (Windows; U; ; en-NZ) AppleWebKit/527  (KHTML, like Gecko, Safari/419.3)
+  Arora/0.8.0
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser;
+  .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727; .NET
+  CLR 3.0.04506.30)
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Beamrise/17.2.0.9
+  Chrome/17.0.939.0 Safari/535.8
+- Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0
+  Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0
+- Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1
+  Safari/536.3
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0
+  Safari/536.6
+- Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0
+  Safari/536.6
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1
+  Safari/537.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0
+  Safari/537.36
+- Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0
+  Safari/537.36
+- Mozilla/5.0 (Windows NT 6.1; rv:12.0) Gecko/20120403211507 Firefox/12.0
+- Mozilla/5.0 (Windows NT 6.0; rv:14.0) Gecko/20100101 Firefox/14.0.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20120427 Firefox/15.0a1
+- Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0
+- Mozilla/5.0 (Windows NT 6.2; rv:19.0) Gecko/20121129 Firefox/19.0
+- Mozilla/5.0 (Windows NT 6.2; rv:20.0) Gecko/20121202 Firefox/20.0
+- Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0
+- Mozilla/5.0 (compatible; Konqueror/4.5; Windows) KHTML/4.5.4 (like Gecko)
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR
+  2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Maxthon
+  2.0)
+- Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko)
+  Maxthon/3.0.8.2 Safari/533.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML like Gecko) Maxthon/4.0.0.2000
+  Chrome/22.0.1229.79 Safari/537.1
+- Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)
+- Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727;
+  .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)
+- Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/4.0)
+- Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/5.0)
+- Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)
+- Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; Trident/5.0)
+- Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; WOW64; Trident/5.0)
+- Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media
+  Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)
+- Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)
+- Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/6.0)
+- Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1;
+  .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0
+- Opera/9.25 (Windows NT 6.0; U; en)
+- Opera/9.80 (Windows NT 5.2; U; en) Presto/2.2.15 Version/10.10
+- Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00
+- Opera/9.80 (Windows NT 6.1; U; en) Presto/2.7.62 Version/11.01
+- Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10
+- Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00
+- Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14
+- Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like
+  Gecko) Version/4.0.4 Safari/531.21.10
+- Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like
+  Gecko) Version/5.0.1 Safari/533.17.8
+- Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like
+  Gecko) Version/5.0.2 Safari/533.18.5
+- Mozilla/5.0 (Windows; U; Windows NT 6.2; es-US ) AppleWebKit/540.0 (KHTML like Gecko)
+  Version/6.0 Safari/8900.00
+- Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like
+  Firefox/3.x) SeaMonkey/2.0.12
+- Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20120422 Firefox/12.0 SeaMonkey/2.9
+- Avant Browser/1.2.789rel1 (http://www.avantbrowser.com)
+- Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko)
+  Chrome/4.0.249.0 Safari/532.5
+- Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko)
+  Chrome/5.0.310.0 Safari/532.9
+- Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko)
+  Chrome/7.0.514.0 Safari/534.7
+- Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like
+  Gecko) Chrome/9.0.601.0 Safari/534.14
+- Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like
+  Gecko) Chrome/10.0.601.0 Safari/534.14
+- Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like
+  Gecko) Chrome/11.0.672.2 Safari/534.20
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0
+  Safari/534.27
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24
+  Safari/535.1
+- Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120
+  Safari/535.2
+- Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36
+  Safari/535.7
+- Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421
+  Minefield/3.0.2pre
+- Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10
+- Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11
+  (.NET CLR 3.5.30729)
+- Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6
+  GTB5
+- Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8
+  ( .NET CLR 3.5.30729; .NET4.0E)
+- Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
+- Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
+- Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110622 Firefox/6.0a2
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+- Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre
+- Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0 )
+- Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90)
+- Mozilla/5.0 (Windows; U; Windows XP) Gecko MultiZilla/1.6.1.0a
+- Mozilla/2.02E (Win95; U)
+- Mozilla/3.01Gold (Win95; I)
+- Mozilla/4.8 [en] (Windows NT 5.1; U)
+- Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.4) Gecko Netscape/7.1 (ax)
+- Opera/7.50 (Windows XP; U)
+- Opera/7.50 (Windows ME; U) [en]
+- Opera/7.51 (Windows NT 5.1; U) [en]
+- Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.0
+- Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.2b) Gecko/20021001 Phoenix/0.2
+- Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.23) Gecko/20090825 SeaMonkey/1.1.18
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
+  Camino/2.2.1
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre
+  Camino/2.2a1pre
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko)
+  Chrome/19.0.1063.0 Safari/536.3
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML like Gecko)
+  Chrome/22.0.1229.79 Safari/537.4
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.31 (KHTML like Gecko)
+  Chrome/26.0.1410.63 Safari/537.31
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 1083) AppleWebKit/537.36 (KHTML like Gecko)
+  Chrome/28.0.1469.0 Safari/537.36
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US) AppleWebKit/528.16 (KHTML, like
+  Gecko, Safari/528.16) OmniWeb/v622.8.0.112941
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/528.16 (KHTML,
+  like Gecko, Safari/528.16) OmniWeb/v622.8.0
+- Opera/9.20 (Macintosh; Intel Mac OS X; U; en)
+- Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61
+- Opera/9.80 (Macintosh; Intel Mac OS X 10.4.11; U; en) Presto/2.7.62 Version/11.00
+- Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML,
+  like Gecko) Version/4.0.4 Safari/531.21.10
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15  (KHTML,
+  like Gecko) Version/5.0.3 Safari/533.19.4
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML,
+  like Gecko) Version/5.0.4 Safari/533.20.27
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7; en-us) AppleWebKit/534.20.8 (KHTML,
+  like Gecko) Version/5.1 Safari/534.20.8
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like
+  Gecko) Version/5.1.3 Safari/534.53.10
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/536.26.17 (KHTML like
+  Gecko) Version/6.0.2 Safari/536.26.17
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+  SeaMonkey/2.7.1
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML,
+  like Gecko) Chrome/4.0.302.2 Safari/532.8
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML,
+  like Gecko) Chrome/6.0.464.0 Safari/534.3
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML,
+  like Gecko) Chrome/9.0.597.15 Safari/534.13
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko)
+  Chrome/14.0.835.186 Safari/535.1
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko)
+  Chrome/15.0.874.54 Safari/535.2
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko)
+  Chrome/16.0.912.36 Safari/535.7
+- 'Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 '
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624
+  Firefox/3.5
+- Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218
+  AlexaToolbar/alxf-2.0 Firefox/3.6.14
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0) Gecko/20100101 Firefox/5.0
+- Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0

data/lib/serper/version.rb

@@ -0,0 +1,3 @@
+module Serper
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: serper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- MingQian Zhang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: domainatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Parse SERP result page.
+email:
+- zmingqian@qq.com
+executables:
+- serper
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/serper/analyser.rb
+- lib/serper/baidu/crawler.rb
+- lib/serper/baidu/parser.rb
+- lib/serper/baidu/weight.rb
+- lib/serper/crawler.rb
+- lib/serper/helper.rb
+- lib/serper/parser.rb
+- lib/serper/version.rb
+- lib/serper.rb
+- bin/serper
+- README.md
+- lib/serper/user_agents.yml
+homepage: https://github.com/semseo/serper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: SERP
+test_files: []