baidu_web 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in baidu_web.gemspec
+gemspec
+
+gem hpricot, '~>0.8.4'

data/README.md

@@ -0,0 +1,36 @@
+# Baidu_web
+
+Baidu_web是一个基于百度的元搜索引擎，输入关键字，返还百度搜索结果和相关的关键词；
+在线Demo: http://www.inruby.com/search
+
+## 用法
+
+    gem 'baidu_web'
+
+## 调用
+
+    # http://www.baidu.com/s?wd=inruby&pn=100&rn=50
+    BaiduWeb.search("inruby", :per_page => 50, :page_index => 2)
+    
+## 返还结果
+
+    result = BaiduWeb.search('key words')
+    result[:record_arr].each do |record|
+      puts record.title
+      puts record.url
+      puts record.summary
+      puts record.updated_date
+      puts record.item_index
+      puts record.cached_url
+    end
+    result[:ext_key_arr].each do |ext_key|
+     puts ext_key.title
+     puts ext_key.url
+    end
+
+## 在irb中测试:
+
+    $:.unshift(File.dirname(__FILE__))
+    require 'baidu_web'
+    require 'cgi'
+    result = BaiduWeb.search(CGI.escape("游戏"), :per_page => 10, :page_index => 1)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/baidu_web.gemspec

@@ -0,0 +1,61 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "baidu_web/version"
+
+Gem::Specification.new do |s|
+  s.name        = "baidu_web"
+  s.version     = BaiduWeb::VERSION
+  s.authors     = ["krongk"]
+  s.email       = ["kenrome@gmail.com"]
+  s.homepage    = "https://github.com/krongk/baidu_web"
+  s.summary     = %q{baidu_web is a meta search engine for 百度(www.baidu.com)}
+  s.description = %q{
+        # Baidu_web
+
+        Baidu_web是一个基于百度的元搜索引擎，输入关键字，返还百度搜索结果和相关的关键词；
+        在线Demo: http://www.inruby.com/search
+        
+        ## 用法
+        
+            gem 'baidu_web'
+        
+        ## 调用
+        
+            # http://www.baidu.com/s?wd=inruby&pn=100&rn=50
+            BaiduWeb.search("inruby", :per_page => 50, :page_index => 2)
+            
+        ## 返还结果
+        
+            result = BaiduWeb.search('key words')
+            result[:record_arr].each do |record|
+              puts record.title
+              puts record.url
+              puts record.summary
+              puts record.updated_date
+              puts record.item_index
+              puts record.cached_url
+            end
+            result[:ext_key_arr].each do |ext_key|
+             puts ext_key.title
+             puts ext_key.url
+            end
+        
+        ## 在irb中测试:
+        
+            $:.unshift(File.dirname(__FILE__))
+            require 'baidu_web'
+            require 'cgi'
+            result = BaiduWeb.search(CGI.escape("游戏"), :per_page => 10, :page_index => 1)
+  }
+
+  s.rubyforge_project = "baidu_web"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/baidu_web/extension_key.rb

@@ -0,0 +1,9 @@
+module BaiduWeb
+  class ExtensionKey
+  	attr_accessor :title, :parent_key, :source
+  end
+end
+
+
+# Store the extension key of reference key word
+# see www.baidu.com '相关搜索'

data/lib/baidu_web/record.rb

@@ -0,0 +1,24 @@
+module BaiduWeb
+  class Record
+  	attr_accessor :title, :url, :summary, :updated_date, :item_index, :size, :cached_url
+  end
+end
+
+# Store the item of search result.
+# :title
+# 	Returns the matched web page or document's page title.
+# :url
+# 	Returns the absolute URL for the matched web document.
+# :summary
+# 	A brief summary for the matched web document from Baidu.com
+# :date
+# 	Return the searched date.
+# :page_index
+# 	Return the record of the pagination, started with 1.
+# :item_index
+# 	Return the record index in the search result page, started with 1.
+# :size
+# 	Size info for the matched document. Note that it's not an number, instead it's in the form of '32K' or something like that.
+# :cached_url
+# 	Returns the url pointing to the cached version of the matched document on Baidu.com. Note that, for documents with types like DOC, PPT and XSL, there won't be a cached version. So this property always returns undef in these cases.
+# 	

data/lib/baidu_web/string_extension.rb

@@ -0,0 +1,8 @@
+module StringExtension
+  def blank?
+    self.nil? || self.strip.length == 0
+  end
+end
+
+String.send :include, StringExtension
+NilClass.send :include, StringExtension

data/lib/baidu_web/strip.rb

@@ -0,0 +1,5 @@
+class Object
+  def strip
+    self.class == String && !self.nil? ? super : self
+  end
+end

data/lib/baidu_web/version.rb

@@ -0,0 +1,3 @@
+module BaiduWeb
+  VERSION = "0.0.1"
+end

data/lib/baidu_web.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+$:.unshift(File.dirname(__FILE__))
+
+require 'hpricot'
+require 'open-uri'
+#require 'iconv'
+#require 'cgi'
+require "baidu_web/version"
+require "baidu_web/record"
+require "baidu_web/extension_key"
+require "baidu_web/string_extension"
+require "baidu_web/strip"
+
+module BaiduWeb
+  class << self
+	  def search(key_word, options)
+	  	result = {:record_arr => [], :ext_key_arr => [], :source => 'web'}
+
+	  	#@ic = Iconv.new("UTF-8//IGNORE", "GBK//IGNORE")
+
+	  	@key_word = key_word
+	  	return result if @key_word.blank?
+	  	#uri parser key word
+	  	# @key_word = CGI.escape(@key_word)
+
+	  	#determine how many records display on one page. (same as www.baidu.com/?<some params>&rn=50)
+	  	@per_page = options[:per_page]
+	  	@per_page ||= 50
+
+	  	#get which page of result. (same as www.baidu.com/?<some params>&pn=0)
+	  	@page_index = options[:page_index]
+	  	@page_index ||= 1
+
+	  	#get the start item index.
+	  	item_index = (@page_index - 1 ) * @per_page
+
+	  	agent = Mechanize.new
+
+  		url = "http://www.baidu.com/s?wd=#{@key_word}&rn=#{@per_page}&pn=#{item_index}"
+  		#debug: url
+  		spage = agent.get(url)
+  		#debug
+  		# File.open(File.join(File.dirname(__FILE__), 'baidu_result.html'), "w"){|f| f.write(@ic.iconv(spage.body))}
+
+  		#doc = Hpricot(@ic.iconv(spage.body))
+  		doc = Hpricot(spage.body)
+
+	  	#- this is hack on linux:
+	  	#case1:
+	  	# result_page = @ic.iconv(open("http://www.baidu.com/s?wd=#{@key_word}&rn=#{@per_page}&pn=#{item_index}").read)
+	  	#case2:
+	  	# result_page = ""
+	  	# open("http://www.baidu.com/s?wd=#{@key_word}&rn=#{@per_page}&pn=#{item_index}", "r:utf-8") {|f|
+  		#   f.each_line do |line|
+  		#   	 result_page += @ic.iconv(line)
+  		#   end
+	  	# }
+
+  		return result if doc.blank?
+  
+  		result[:record_arr] = extract_item(doc, item_index)
+  		result[:ext_key_arr] = extract_extension_key(doc)
+  		#debug
+  		puts result[:record_arr].size
+  
+  		return result
+	  end
+  
+	  private
+	  def extract_item(content, item_index)
+	  	record_arr = []
+	  	#remove op recors, e.g. search by 'mysql', see the second record.
+	  	content.search("table[@class='result-op']").remove
+
+			content.search("table[@class='result']").each do |res|
+				next if res.at("h3").nil?
+
+				record = Record.new
+
+				title = res.at("h3").inner_text
+				record.title = title
+				record.url = res.at("h3").at("a").attributes['href'].to_s
+
+				summary = []
+				res.at("td[@class='f']").children.each do |elem|
+			  	if elem.respond_to?(:attributes) && elem.attributes['href'] =~ /http:\/\/cache.baidu.com/
+			  	  record.cached_url = elem.attributes['href'] 
+			  	  next
+			  	elsif elem.respond_to?(:attributes) && elem.attributes['class'] == 'g' && elem.to_s =~ /(\d{4}-\d{1,2}-\d{1,2})/
+			  	  record.updated_date = $1
+			  	  next
+			    end
+			    next if elem.respond_to?(:attributes) && elem.attributes['class'] == 't'
+			    summary << elem.inner_text
+				end
+				record.summary = summary.join(' ').gsub(/百度|百度快照|快照/, '')
+
+				item_index += 1
+				record.item_index = item_index
+				record_arr << record
+			end
+			return record_arr
+	  end
+
+	  def extract_extension_key(doc)
+	  	rs = doc.at("//div#rs")
+	  	return [] if rs.nil?
+	  	ext_key_arr = []
+	  	rs.get_elements_by_tag_name("a").each do |link|
+	  	   ext_key = ExtensionKey.new
+	  	   ext_key.title = link.inner_text
+	  	   ext_key.parent_key = @key_word
+	  	   ext_key.source = 'web'
+	  	   ext_key_arr << ext_key
+	  	end
+	  	ext_key_arr
+	  end
+
+	end
+end

data/lib/test.rb

@@ -0,0 +1,60 @@
+# encoding = utf-8
+$:.unshift(File.dirname(__FILE__))
+
+require 'baidu_web'
+require 'cgi'
+result = BaiduWeb.search(CGI.escape("游戏"), :per_page => 10, :page_index => 1)
+
+# class Test
+#   def self.search(key, ha)
+#     puts key
+#     puts ha
+#     puts ha.class
+#     puts ha.size
+#     puts ha[:key].to_s
+#     puts ha[:b].to_s
+#     puts ha[:c].to_s
+
+#   end
+# end
+
+#  Test.search('a', :a=>'a', :key => '444', :b=>44)
+
+
+#  0.times do |i|
+#  	puts i
+#  end
+
+# http://www.baidu.com/s?wd=%C3%C0%C5%AE&rsv_bp=0&rsv_spt=3&inputT=930
+# http://www.baidu.com/s?wd=%B3%C9%B6%BC&pn=0&rn=5&usm=4
+# http://www.baidu.com/s?wd=%C3%C0%C5%AE&pn=50&rn=50&usm=4
+# http://www.baidu.com/s?wd=%C3%C0%C5%AE&pn=100&rn=50&usm=4
+
+#  rn -> per page record
+#  pn -> start record
+
+
+# html = Hpricot(doc);nil
+
+
+# ic = Iconv.new("utf-8//IGNORE", "GB2312//IGNORE")
+# doc = ""
+# open("http://www.baidu.com/s?rn=50&bs=%C3%C0%C5%AE&f=8&rsv_bp=1&wd=mysql&inputT=2955", "r:gb2312:utf-8") {|f|
+#   f.each_line do |line|
+#   	doc += ic.iconv(line)
+#   end
+# };nil
+
+# doc = Hpricot(doc);nil
+
+# results = doc.at("div[@id='container']");nil
+
+# results.search("table[@class='result']");nil
+
+# doc.search("table[@class='result-op']").remove
+
+
+# <table class="result-op" 
+# cellpadding="0" cellspacing="0" srcid="6669" id="2" 
+# mu="http://soft.baidu.com/softwaresearch/s?tn=software&amp;rn=10&amp;wd=mysql" 
+# data-op="{'y':'BF0FFF7F'}"><style>.op_mini_table01_content table{margin-top:4px;}.op_mini_table01_content th{text-align:left;white-space:nowrap;background:url("http://www.baidu.com/aladdin/img/table/bg.gif") repeat-x 0 -37px;font-weight:normal;height:26px;line-height:26px;font-size:13px;padding:0 10px 0 8px;}.op_mini_table01_content td{white-space:nowrap;font-size:14px;border-bottom:#eee 1px solid;}.OP_TABLE_COMMON{ width:100%;}.OP_TABLE_COMMON td{ padding:7px 10px 7px 8px; font-size:14px;}.OP_TABLE_COMMON a,.OP_TABLE_COMMON a em{ text-decoration:none;}.OP_TABLE_COMMON a:hover,.OP_TABLE_COMMON a:hover em{ text-decoration:underline;}</style><script>function jI(D){var C=D;var B=0;while(C=C.parentNode){B=parseInt(C.getAttribute("id"));if(B>0){break}}var A=C.getElementsByTagName("a");for(var B=0;B<A.length;B++){if(D==A[B]){return B}}return A.length-1}function _aMC(C){var B=C,A=-1;while(B=B.parentNode){A=parseInt(B.getAttribute("id"));if(A>0){return A}}};</script><tbody><tr><td class="f"><h3 class="t"><a onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this)})" href="http://soft.baidu.com/softwaresearch/s?tn=software&amp;rn=10&amp;wd=mysql" target="_blank"><font size="3"><em>mysql</em>_相关下载信息363条_百度软件搜索</font></a><span class="tsuf tsuf-op" data="{title : 'mysql_相关下载信息363条_百度软件搜索', link : 'http:\/\/soft.baidu.com\/softwaresearch\/s?tn=software&amp;rn=10&amp;wd=mysql'}"></span></h3><div class="op_mini_table01_content op_software"> <table cellspacing="0" class="OP_TABLE_COMMON"><tbody><tr><th style="border-left:0;">软件名称</th><th>软件大小</th><th>来源</th></tr>  <tr><td> <a target="_blank" onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" href="http://www.skycn.com/soft/30406.html">Apache+Php+<em>Mysql</em> V1.3 绿色自动安装版</a></td><td width="120px;"> 13.44 M </td><td> 天空软件站 </td></tr>   <tr><td> <a target="_blank" onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" href="http://www.newhua.com/soft/3573.htm"><em>MYSQL</em> 5.5.15</a></td><td width="120px;"> 27.76 M </td><td> 华军软件园 </td></tr>   <tr><td> <a target="_blank" onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" href="http://www.duote.com/soft/3169.html"><em>MYSQL</em> For Windows V5.0.67(无毒无插件)</a></td><td width="120px;"> 23.27 M </td><td> 多特软件站 </td></tr>   <tr><td> <a target="_blank" onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" href="http://dl.pconline.com.cn/html_2/1/79/id=465&amp;pn=0.html"><em>MYSQL</em> 5.1.59</a></td><td width="120px;"> 26.7 M </td><td> 太平洋下载 </td></tr>   <tr><td> <a target="_blank" onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" href="http://xiazai.zol.com.cn/detail/9/89874.shtml"><em>MySQL</em> 5.5.15官方下载</a></td><td width="120px;"> 28.43 M </td><td> ZOL软件下载 </td></tr>  </tbody></table> <div style="padding:4px 0 2px;"><a onmousedown="return c({'fm':'alop','title':this.innerHTML,'url':this.href,'p1':_aMC(this),'p2':jI(this)})" style="color:#7777CC;font-size:12px;" href="http://soft.baidu.com/softwaresearch/s?tn=software&amp;rn=10&amp;wd=mysql" target="_blank">查看全部363条结果<span style="font-family:simsun">&gt;&gt;</span></a></div><font size="-1" color="#008000">soft.baidu.com/softwaresearch/s?tn=software&amp;r... 2011-10-1</font></div> </td></tr></tbody></table>

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: baidu_web
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- krongk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-29 00:00:00.000000000 Z
+dependencies: []
+description: ! "\n        # Baidu_web\n\n        Baidu_web是一个基于百度的元搜索引擎，输入关键字，返还百度搜索结果和相关的关键词；\n
+  \       在线Demo: http://www.inruby.com/search\n        \n        ## 用法\n        \n
+  \           gem 'baidu_web'\n        \n        ## 调用\n        \n            # http://www.baidu.com/s?wd=inruby&pn=100&rn=50\n
+  \           BaiduWeb.search(\"inruby\", :per_page => 50, :page_index => 2)\n            \n
+  \       ## 返还结果\n        \n            result = BaiduWeb.search('key words')\n            result[:record_arr].each
+  do |record|\n              puts record.title\n              puts record.url\n              puts
+  record.summary\n              puts record.updated_date\n              puts record.item_index\n
+  \             puts record.cached_url\n            end\n            result[:ext_key_arr].each
+  do |ext_key|\n             puts ext_key.title\n             puts ext_key.url\n            end\n
+  \       \n        ## 在irb中测试:\n        \n            $:.unshift(File.dirname(__FILE__))\n
+  \           require 'baidu_web'\n            require 'cgi'\n            result =
+  BaiduWeb.search(CGI.escape(\"游戏\"), :per_page => 10, :page_index => 1)\n  "
+email:
+- kenrome@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- baidu_web.gemspec
+- lib/baidu_web.rb
+- lib/baidu_web/extension_key.rb
+- lib/baidu_web/record.rb
+- lib/baidu_web/string_extension.rb
+- lib/baidu_web/strip.rb
+- lib/baidu_web/version.rb
+- lib/test.rb
+homepage: https://github.com/krongk/baidu_web
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: baidu_web
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: baidu_web is a meta search engine for 百度(www.baidu.com)
+test_files: []