baiduserp 0.0.2

data/README.md

@@ -0,0 +1,29 @@
+# Baiduserp
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'baiduserp'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install baiduserp
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/bin/baiduserp

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'baiduserp'
+require 'optparse'
+require 'json'
+require 'pp'
+
+usage = "Usage: 
+1. baiduserp -s 'keyword' # search 'keyword' and print parse result
+2. baiduserp -s 'keyword' -o output.json # -o means save result to a file
+3. baiduserp -f 'file path' # parse html source code from file
+4. baiduserp -s 'keyword' -j # search 'keyword' and print parse result in JSON format
+"
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = usage
+
+  opts.on("-s Keyword", "--search Keyword", "Search Keyword & Parse SERP") do |v|
+    options[:keyword] = v
+  end
+
+  opts.on("-j","--jsonprint","Print result in JSON format") do |v|
+    options[:jsonprint] = v
+  end
+
+  opts.on("-o Output", "--output Output", "Save Result to File in JSON format") do |v|
+    options[:output] = v
+  end
+
+  opts.on("-f File", "--file File", "Parse Local File") do |v|
+    options[:file] = v
+  end
+end.parse!
+
+result = ''
+
+if options[:file].nil?
+  result = Baiduserp.search options[:keyword]
+else
+  result = Baiduserp.parse_file options[:file]
+end
+
+if options[:output].nil?
+  if options[:jsonprint].nil?
+    pp result
+  else
+    puts result.to_json
+  end
+else
+  open(options[:output],'w').puts result.to_json
+end
+

data/lib/baiduserp.rb

@@ -0,0 +1,16 @@
+require "baiduserp/version"
+require 'baiduserp/parser'
+
+module Baiduserp
+  def self.search(keyword)
+    Parser.new.search keyword
+  end
+
+  def self.parse(html)
+    Parser.new.parse html
+  end
+
+  def self.parse_file(file_path)
+    Parser.new.parse_file file_path
+  end
+end

data/lib/baiduserp/parser.rb

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+require 'nokogiri'
+require 'uri'
+require 'open-uri'
+
+module Baiduserp
+  class Parser
+    BAIDU_RESULT = /找到相关结果(.*)个/
+
+    def parse(html)
+      @html = html
+      @doc = Nokogiri::HTML(@html)
+      @results = []
+      @serp = Hash.new
+      parse_serp_results
+      parse_serp_attrs
+      @serp
+    end
+
+    def search(keyword)
+      parse_file("http://www.baidu.com/s?wd=#{keyword}")
+    end
+
+    def parse_file(file_path)
+      if File.exists? file_path
+        html = open(file_path)
+      else
+        html = open(URI.escape(file_path))
+      end
+      html = html.read.encode('UTF-8')
+      parse html
+    end
+
+    private
+    def get_content_safe(noko)
+      return nil if noko.nil?
+      return nil if noko.empty?
+      noko.first.content
+    end
+
+    def parse_serp_results
+      # left side results
+      @doc.search("//table").each do |table|
+        id = table['id'].to_i
+        parse_serp_table(id,table) if id > 0
+      end
+      # right side ads
+      parse_right_side_ads
+      @serp[:serp_results] = @results
+    end
+
+    def parse_right_side_ads
+      @doc.search("//div[@class='EC_fr EC_PP']").each do |table|
+        id = table['id'].to_s.sub('bdfs','').to_i
+        rank = id + 1
+        url = @doc.search("//div[@id='bdfs#{id}' and @class='EC_fr EC_PP']//font[@size='-1' and @color='#008000']").first.content
+        title = get_content_safe(@doc.search("//div[@id='bdfs#{id}' and @class='EC_fr EC_PP']//a"))
+        content = get_content_safe(@doc.search("//div[@id='bdfs#{id}' and @class='EC_fr EC_PP']//font[@size='-1']"))
+        @results << {:paid => 2, :rank => rank, :url => url, :title => title, :content => content}
+      end
+    end
+
+    def get_url_part_from_string(str)
+      str.split(/( |\s)/).each do |s|
+        return s if s.include? '.'
+      end
+      nil
+    end
+
+    def parse_serp_url(table_id)
+      id = table_id
+      url = nil
+      if id > 3000
+        link_types = ["//table[@id='#{id}']//font[@size='-1' and @color='#008000']"] 
+      else
+        link_types = ["//table[@id='#{id}']//span[@class='g']",
+                      "//table[@id='#{id}']//font[@color='#008000']",
+                      "//table[@id='#{id}']//span[@style='color:#008000']",
+                      "//table[@id='#{id}']//span[@style='color:#008000;']",
+                      "//table[@id='#{id}']//span[@color='#008000']",
+                      "//table[@id='#{id}']//p[@class='g']",
+                      "//table[@id='#{id}']//cite[@color='#008000']",
+                      "//table[@id='#{id}']//cite",
+                      "//table[@id='#{id}']//span[@id='ala_img_desc']"
+                     ]
+      end
+      link_types.each do |link_type|
+        link_search = @doc.search(link_type)
+        url2 = nil
+        url2 =  get_url_part_from_string(link_search[0].content) if link_search.size > 0
+        if url.nil? && (not url2.nil?)
+          url = url2
+        end
+      end
+      url
+    end
+
+    def parse_serp_title(id)
+
+    end
+
+    def parse_serp_content(id)
+      get_content_safe(@doc.search("//table[@id='#{id}']//font[@size='-1']"))
+    end
+
+    def parse_serp_table(id,table)
+      result = Hash.new
+      result[:rank] = id
+      result[:url] = parse_serp_url(id)
+      result[:title] = table.css('h3').first.content
+      result[:content] = parse_serp_content(id)
+
+      if id > 3000 # sem ads
+        result[:paid] = 1
+      else # organic results
+        result[:paid] = 0
+
+        # baidu open
+        table.css('a').each do |link|
+          result[:baiduopen] = 1 if link['href'].to_s.include? 'open.baidu.com'
+        end
+
+        # baidu table mu attr (for maps,baike)
+        result[:mu] = table['mu'] unless table['mu'].nil?
+
+      end
+      
+      @results << result
+    end
+
+    # parse baidu serp attrs : result_num, baidubrand, related_keywords
+    def parse_serp_attrs
+      @serp[:result_num] = parse_serp_result_num
+      @serp[:baidubrand] = parse_serp_baidu_brand
+      @serp[:related_keywords] = parse_serp_related_search
+    end
+
+    def parse_serp_related_search
+      result = []
+      @doc.search('div[@id="rs"]').each do |rs|
+        rs.css('a').each do |link|
+          result << link.content
+        end
+      end
+      result
+    end
+
+    def parse_serp_baidu_brand
+      if @html.include? 'bs.baidu.com/adcoup-mat'
+        result = 1
+      else
+        result = 0
+      end
+      result
+    end
+
+    def parse_serp_result_num
+      str = @html.scan(BAIDU_RESULT).join
+      str = str.gsub('约','')
+      if str.include?('万')
+        parts = str.split('万')
+        return parts[0].to_i * 10000 + parts[1].to_i
+      end
+      str.gsub(',', '').to_i
+    end
+
+  end
+end
+
+

data/lib/baiduserp/version.rb

@@ -0,0 +1,3 @@
+module Baiduserp
+  VERSION = "0.0.2"
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: baiduserp
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- MingQian Zhang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Parse Baidu SERP result page.
+email:
+- zmingqian@qq.com
+executables:
+- baiduserp
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/baiduserp/parser.rb
+- lib/baiduserp/version.rb
+- lib/baiduserp.rb
+- bin/baiduserp
+- README.md
+homepage: http://www.baidu.com/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Baidu SERP
+test_files: []
+has_rdoc: