query 0.0.1 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e414f7561d351c15835fb94956919de3d9d6ac62
-  data.tar.gz: 1e75cc2eb2d552b779d7cc33865248cb73d00d64
+  metadata.gz: e5043b0180a473ab3d213136dfe9ab55ccb4a6d9
+  data.tar.gz: dc0b8b1ee15dc3f4437439712904de92b838520e
 SHA512:
-  metadata.gz: e07b5ab0f23e5775945fa66cd55ae0e822888da4d5aa65968ee5a0cf91d219a3126b2af7836e23cde660b43fccca4570cc33be142d08b83092388de975a84339
-  data.tar.gz: baa4d81223b5911ca159735bdff34ba6b4646ebeefe83112463c733373697ee0823b3de5e57513e97105c55f663bce58d87e8530f48c5d2a8718e50f80785974
+  metadata.gz: baa8ca09bc7bfd19f1eb3e9dffa24cf0cc28aba704d8671769bc26a79c68f10c01d57df4f76d4e6558ea638fd6c211111ca21d1883675450971256fc8369bc2a
+  data.tar.gz: 804e6685b6d7d49e563318a9150eae1a655c52cb6f6ca9084c7fbb0c908fde92a44025761668abe2b7aa4b21ba31fbb0c9ebcbc24a6d3c61fd2415e5344b9fd4

data/.gitignore

@@ -20,4 +20,7 @@ tmp
 # YARD artifacts
 .yardoc
 _yardoc
-doc/
+doc/
+lib/query/.DS_Store
+
+lib/query/.DS_Store

data/Gemfile

@@ -1,4 +1,6 @@
-source 'https://rubygems.org'
 
+source 'http://ruby.taobao.org'
+gem 'addressable'
+gem 'httparty'
 # Specify your gem's dependencies in query.gemspec
 gemspec

data/README.md

@@ -3,22 +3,27 @@ Query
 
 
 #to get the result list by querying "abc"
+
 Query::Engine::Baidu.new.query("abc").ranks().each do |id,value|
     puts id,value
 end
 
 #to get the result list with host "www.abc.com.cn" by querying "abc"
+
 Query::Engine::Baidu.new.query("abc").ranks("www.abc.com.cn").each do |id,value|
     puts id,value
 end
 
 #to get the result list with host which fit the regex /com.cn/ by querying "abc"
+
 Query::Engine::Baidu.new.query("abc").ranks(/com.cn/).each do |id,value|
     puts id,value
 end
 
 # to get the top rank of host "www.abc.com.cn" by querying "abc"
+
 Query::Engine::Baidu.new.query("abc").rank("www.abc.com.cn")
 
 TODO:
-查询结果不多,翻页不存在时的处理,及rspec
+查询结果不多,翻页不存在时的处理,及rspec
+增加其他搜索引擎

data/lib/query/engine/baidu.rb

@@ -1,7 +1,11 @@
 module Query
     module Engine
-        class Baidu < Base
+        class Baidu
+            include Query::Engine
             BaseUri = 'http://www.baidu.com/s?'
+            Options = {
+                :headers => {"User-Agent" => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11'}
+            }
             def self.suggestions(wd)
                 require 'json'
                 json = HTTParty.get("http://suggestion.baidu.com/su?wd=#{URI.encode(wd)}&cb=callback").force_encoding('GBK').encode("UTF-8")
@@ -37,25 +41,25 @@ module Query
                 return HTTParty.get("http://index.baidu.com/main/word.php?word=#{URI.encode(wd.encode("GBK"))}").include?"boxFlash"
             end
 
-            def query(wd)
+            def self.query(wd)
                 q = Array.new
                 q << "wd=#{wd}"
                 q << "rn=#{@perpage.to_i}" if @perpage
                 queryStr = q.join("&")
                 #uri = URI.encode((BaseUri + queryStr).encode('GBK'))
                 uri = URI.encode((BaseUri + queryStr))
-                begin
+                # begin
                     # @page = @a.get uri
-                    @page = HTTParty.get uri
+                    @page = HTTParty.get(uri,Options)
                     r = Query::Result::Baidu.new(@page)
                     r.baseuri = uri
                     r.pagenumber = 1
                     r.perpage = @perpage
                     r
-                rescue Exception => e
-                    warn e.to_s
-                    return false
-                end
+                # rescue Exception => e
+                #     warn e.to_s
+                #     return false
+                # end
 =begin
                 query = "#{query}"
                 @uri = BaseUri+URI.encode(query.encode('GBK'))

data/lib/query/engine/baidu_mobile.rb

@@ -1,11 +1,11 @@
 module Query
     module Engine
-        class BaiduMobile < Base
+        class BaiduMobile
+            include Query::Engine
             BaseUri = 'http://m.baidu.com/s?'
-            headers = {
-                "User-Agent" => 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5'
+            Options = {
+                :headers => {"User-Agent" => 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5'}
             }
-            Options = {:headers => headers}
 
             #基本查询,相当于从搜索框直接输入关键词查询
             def query(wd)

data/lib/query/engine/{qihoo.rb → qihu.rb}

@@ -1,15 +1,20 @@
 module Query
     module Engine
-        class Qihoo < Base
+        class Qihu
+            include Query::Engine
             Host = 'www.so.com'
+            headers = {
+                "User-Agent" => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11'
+            }
+            Options = {:headers => headers}
             #基本查询, 相当于在搜索框直接数据关键词查询
             def query(wd)
                 #用原始路径请求
                 uri = URI.join("http://#{Host}/",URI.encode('s?q='+wd)).to_s
-                page = HTTParty.get(uri)
+                page = HTTParty.get(uri,Options)
                 #如果请求地址被跳转,重新获取当前页的URI,可避免翻页错误
                 uri = URI.join("http://#{Host}/",page.request.path).to_s
-                r = Query::Result::Qihoo.new(page)
+                r = Query::Result::Qihu.new(page)
                 r.baseuri = uri
                 r
             end

data/lib/query/engine/sogou.rb

@@ -0,0 +1,45 @@
+module Query
+	module Engine
+		class Sogou
+			include Query::Engine
+			BaseUri = 'http://www.sogou.com/web?'
+      Options = {
+          :headers => {"User-Agent" => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11'}
+      }
+			class << self
+				def query(wd)
+					q = []
+					q << "query=#{wd}"
+					uri = URI.encode BaseUri+q.join('&')
+					page = HTTParty.get(uri,Options)
+					r = Query::Result::Sogou.new(page)
+					r.baseuri = uri
+					r.perpage = @perpage
+					r.pagenumber = 1
+					r
+				end
+
+				def suggestions(word)
+					suggestions = HTTParty.get "http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key=#{URI.encode(word)}"
+					suggestions = suggestions.encode('utf-8').scan /#{word}[^"]+/
+					suggestions
+				end
+
+        #site:xxx.yyy.com
+				def pages(host)
+					query("site:#{host}")
+				end
+
+        #domain:xxx.yyy.com/path/file.html
+				def links(uri)
+					query("domain:\"#{uri}\"")
+				end
+
+        #site:xxx.yyy.com inurl:zzz
+				# def pages_with(host,string)
+    #       query("site:#{host} inurl:#{string}")
+				# end
+			end
+		end
+	end
+end

data/lib/query/engine/sogou_mobile.rb

@@ -0,0 +1,21 @@
+module Query
+	module Engine
+		class SogouMobile
+			include Query::Engine
+			BaseUri = 'http://wap.sogou.com/web/searchList.jsp'
+			Options = {
+				:headers => {"User-Agent" => 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5'}
+			}
+			class << self
+				def query(wd)
+				queryStr = "keyword=#{wd}"
+				uri = URI.encode(BaseUri + "?" + queryStr)
+				res = HTTParty.get(uri,Options)
+				r = Query::Result::SogouMobile.new(res)
+				r.baseuri = uri
+				r
+				end
+			end
+		end
+	end
+end

data/lib/query/engine.rb

@@ -1,10 +1,17 @@
 module Query
   module Engine
+    attr_accessor :perpage
+    def self.indexed?(url)
+      URI(url)
+      result = query(url)
+      return result.has_result?
+    end
   end
 end
-
-require 'query/engine/base'
+require 'httparty'
 require 'query/engine/baidu'
 require 'query/engine/baidu_mobile'
-require 'query/engine/qihoo'
-require 'query/engine/qihoo_mobile'
+require 'query/engine/qihu'
+require 'query/engine/qihu_mobile'
+require 'query/engine/sogou'
+require 'query/engine/sogou_mobile'

data/lib/query/result/baidu.rb

@@ -1,94 +1,40 @@
 module Query
     module Result
-        class Baidu < Base
-            def ranks
+        class Baidu
+            include Query::Result
+            def seo_ranks
                 return @ranks unless @ranks.nil?
-                @ranks = Hash.new
-                @page.search("//table[@class=\"result\"]|//table[@class=\"result-op\"]").each do |table|
-                    id = table['id']
-                    # if @perpage == 10
-                    #     id = table['id'][-1,1]
-                    #     id = '10' if id == '0'
-                    # end
-
-                    @ranks[id] = Hash.new
-                    url = table.search("[@class=\"g\"]").first
-                    url = url.text unless url.nil?
-                    a = table.search("h3").first
-                    next if a.nil?
-                    @ranks[id]['text'] = a.text
-                    @ranks[id]['href'] = url #a.first['href'].sub('http://www.baidu.com/link?url=','').strip
-                    unless url.nil?
-                        url = url.strip
-                        @ranks[id]['host'] = Addressable::URI.parse(URI.encode("http://#{url}")).host
-                    else
-                        @ranks[id]['host'] = nil
-                    end
+                @page.search("//*[@class='result']|//*[@class='result-op']|//*[@class='result-op c-container']").map.with_index do |table,index|
+                    parse_seo(table).merge({:rank => index + 1})
                 end
-                #@page.search("//table[@class=\"result\"]").map{|table|@page.search("//table[@id=\"#{table['id']}\"]//span[@class=\"g\"]").first}.map{|rank|URI(URI.encode('http://'+rank.text.strip)).host unless rank.nil?}
-                @ranks
             end
 
-            def ads_bottom
-                return {} if @page.search("//table[@bgcolor='f5f5f5']").empty?
-                return ads_top
-                # p @page.search("//table[@bgcolor='f5f5f5']").empty?
-            end
             def ads_top
-                #灰色底推广,上下都有
-                ads = Hash.new
-                @page.search("//table[@bgcolor='#f5f5f5']").each do |table|
-                    id = table['id']
-                    next if id.nil?
-                    id = id[2,3].to_i.to_s
-                    ads[id]= parse_ad(table)
+                @page.search("//*[@class='result']/preceding-sibling::*[contains(@class,'EC_result')]").map.with_index do |div, index|
+                    parse_ad(div).merge(:rank => index + 1)
                 end
-                #白色底推广,只有上部分
-                if ads.empty?
-                    @page.search("//table").each do |table|
-                        id = table['id']
-                        next if id.nil? or id.to_i<3000
-                        id = id[2,3].to_i.to_s
-                        ads[id]= parse_ad(table)
-                    end
-                end
-                ads
             end
-            def parse_ad(table)
-                href = table.search("font[@color='#008000']").text.split(/\s/).first.strip
-                title = table.search("a").first.text.strip
-                {'title'=>title,'href' => href,'host'=>href}
+
+            def ads_bottom
+                @page.search("//*[@class='result']/following-sibling::*[contains(@class,'EC_result')]").map.with_index do |div,index|
+                    parse_ad(div).merge(:rank => index + 1)
+                end
             end
+
             def ads_right
-                ads = {}
-                @page.search("//div[@id='ec_im_container']").each do |table|
-                    table.search("div[@id]").each do |div|
-                        id = div['id'][-1,1].to_i+1
-                        title = div.search("a").first
-                        next if title.nil?
-                        title = title.text
-                        url = div.search("font[@color='#008000']").first
-                        next if url.nil?
-                        url = url.text
-                        ads[id.to_s] = {'title'=>title,'href'=>url,'host'=>url}
-                    end
+                @page.search("//div[@id='ec_im_container']/div[@id]").map.with_index do |div,index|
+                    a = div.search('a').first
+                    url = div.search("*[@class='EC_url']").first.text
+                    url = "http://#{url}"
+                    {
+                        :rank => index + 1,
+                        :text => a.text.strip,
+                        :href => a['href'].strip,
+                        :host => Addressable::URI.parse(URI.encode(url)).host
+                    }
                 end
-                ads
             end
 
-            #return the top rank number from @ranks with the input host
-            # def rank(host)#on base of ranks
-            #     ranks.each do |id,line|
-            #         id = id.to_i
-            #         if host.class == Regexp
-            #             return id if line['host'] =~ host
-            #         elsif host.class == String
-            #             return id if line['host'] == host
-            #         end
-            #     end
-            #     return nil
-            # end
-
             def count
                 @count ||= @page.search("//span[@class='nums']").map{|num|num.content.gsub(/\D/,'').to_i unless num.nil?}.first
             end
@@ -97,25 +43,45 @@ module Query
                 @related_keywords ||= @page.search("//div[@id=\"rs\"]//tr//a").map{|keyword| keyword.text}
             end
 
-            def next
-                url = @page.xpath('//a[text()="下一页>"]').first
-                return if url.nil?
-                url = url['href']
-                url = URI.join(@baseuri,url).to_s
-                page = HTTParty.get(url)
-                r = Query::Result::Baidu.new(page)
-                r.baseuri = url
-                r.pagenumber=@pagenumber+1
-                r.perpage=@perpage
-                r
-
-                # @page = BaiduResult.new(Mechanize.new.click(@page.link_with(:text=>/下一页/))) unless @page.link_with(:text=>/下一页/).nil?
-            end
             def has_result?
                 submit = @page.search('//a[text()="提交网址"]').first
                 return false if submit and submit['href'].include?'sitesubmit'
                 return true
             end
+
+            def next_url
+                @page.search("//a[text()='下一页>']").first['href']
+            end
+
+            private
+            def parse_ad(div)
+                #@todo  should be :
+                #title = div.xpath("*[contains(@class,'ec_title')]",MyFilter.new).first
+                title = div.xpath("//*[contains(@class,'ec_title')]",MyFilter.new).first
+                url = %w( span[@class='ec_url']  a[@class='EC_url'] ).map do |xpath|
+                    node = div.search(xpath).first
+                    node.text if node
+                end.compact.first
+                url = "http://" + url
+                {
+                    :text => title.text,
+                    :href => title['href'],
+                    :host => Addressable::URI.parse(URI.encode(url)).host
+                }
+            end
+
+            def parse_seo(table)
+                url = %w( span[@class="g"]  span[@class="c-showurl"] div[@class="op_zhidao_showurl"]).map do |xpath|
+                    span = table.search(xpath).first
+                    span.text.sub(/\d{4}-\d{1,2}-\d{1,2}/,'').strip if span
+                end.compact.first
+                host = Addressable::URI.parse(URI.encode("http://#{url}")).host
+                {
+                    :text => table.search("h3").first.text.strip,
+                    :href => table.search('a').first['href'].strip,
+                    :host => host
+                }
+            end
         end
     end
 end

data/lib/query/result/baidu_mobile.rb

@@ -1,113 +1,69 @@
 module Query
     module Result
-        class BaiduMobile < Base
-            #返回当前页所有查询结果
-            def ranks
-                #如果已经赋值说明解析过,不需要重新解析,直接返回结果
-                return @ranks unless @ranks.nil?
-                @ranks = Hash.new
-                @page.xpath('//div[@class="result"]').each do |result|
-                    href,text,host,is_mobile = '','','',false
-                    a = result.search("a").first
-                    is_mobile = true unless a.search("img").empty?
-                    host = result.search('[@class="site"]').first
-                    next if host.nil?
-                    host = host.text
-                    href = a['href']
-                    text = a.text
-                    id = href.scan(/&order=(\d+)&/)
-                    if id.empty?
-                        id = nil
-                    else
-                        id = id.first.first.to_i
-                        # id = (@pagenumber-1)*10+id
-                    end
-=begin
-                    result.children.each do |elem|
-                        if elem.name == 'a'
-                            href = elem['href']
-                            id = elem.text.match(/^\d+/).to_s.to_i
-                            text = elem.text.sub(/^\d+/,'')
-                            text.sub!(/^\u00A0/,'')
-                        elsif elem['class'] == 'abs'
-                            elem.children.each do |elem2|
-                                if elem2['class'] == 'site'
-                                    host = elem2.text
-                                    break
-                                end
-                            end
-                        elsif elem['class'] == 'site'
-                            host == elem['href']
-                        end
-                    end
-=end
+        class BaiduMobile
+            include Query::Result
 
-                    @ranks[id.to_s] = {'href'=>href,'text'=>text,'is_mobile'=>is_mobile,'host'=>host.sub(/\u00A0/,'')}
+            def seo_ranks
+                @seo_ranks ||= @page.search("//*[@class='result']|//*[@class='card-result wa-ue-card-result']|//*[@class='result card-result wma-card-box']").map.with_index do |div,index|
+                    parse_seo(div).merge({:rank => index + 1})
                 end
-                @ranks
             end
+
             def ads_top
-                id = 0
-                result = []
-                @page.search("div[@class='ec_wise_ad']/div").each do |div|
-                    id += 1
-                    href = div.search("span[@class='ec_site']").first.text
-                    href = "http://#{href}"
-                    title = div.search("a/text()").text.strip
-                    host = Addressable::URI.parse(URI.encode(href)).host
-                    result[id] = {'title'=>title,'href'=>href,'host'=>host}
+                @ads_top ||= @page.search("//*[@class='result']/preceding-sibling::div[@class='ec_wise_ad']/div").map.with_index do |div,index|
+                    puts index
+                    parse_ad(div).merge({:rank => index + 1})
                 end
-                result
             end
+
             def ads_right
                 []
             end
+
             def ads_bottom
-                []
+                @ads_bottom ||= @page.search("//*[@class='result']/following-sibling::div[@class='ec_wise_ad']/div/div").map.with_index do |div,index|
+                    parse_ad(div).merge({:rank => index + 1})
+                end
             end
+
+            #酒店预订 酒店英文 酒店团购 酒店管理 酒店招聘 快捷酒店 如家快捷酒店 五星级酒店
             def related_keywords
-                @related_keywords ||= @page.search("div[@class='relativewords_info']/a").map{|a|a.text}
+                @related_keywords ||= @page.search("div[@class='rw-list']/a").map{|a|a.text}
             end
-=begin
-            #返回当前页中,符合host条件的结果
-            def ranks_for(specific_host)
-                host_ranks = Hash.new
-                ranks.each do |id,line|
-                    if specific_host.class == Regexp
-                        host_ranks[id] = line if line['host'] =~ specific_host
-                    elsif specific_host.class == String
-                        host_ranks[id] = line if line['host'] == specific_host
-                    end
-                end
-                host_ranks
+
+            def next_url
+               @next_url ||= @page.xpath('//a[contains(text(),"下一页")]').first['href']
             end
-            #return the top rank number from @ranks with the input host
-            def rank(host)#on base of ranks
-                ranks.each do |id,line|
-                    id = id.to_i
-                    if host.class == Regexp
-                        return id if line['host'] =~ host
-                    elsif host.class == String
-                        return id if line['host'] == host
-                    end
-                end
-                return nil
+
+            def count
+
             end
-=end
-            #下一页
-            def next
-                nextbutton = @page.xpath('//a[text()="下一页"]').first
-                return nil if nextbutton.nil?
-                url = URI.encode nextbutton['href']
-                # puts url
-                # p @baseuri
-                # exit
-                url = URI.join(@baseuri,url).to_s
-                page = HTTParty.get(url)
-                r = Query::Result::BaiduMobile.new(page)
-                r.baseuri=url
-                r.pagenumber=@pagenumber+1
-                r
+
+            private
+            def parse_ad(div)
+                url = div.search("span[@class='ec_site']").first.text
+                url = "http://#{url}"
+                {
+                    :text => div.search('a/text()').text.strip,
+                    :href => div.search('a').first['href'],
+                    :host => Addressable::URI.parse(URI.encode(url)).host
+                }
+            end
+
+            def parse_seo(div)
+                a = div.search('a').first
+                if div['class'] == 'card-result wa-ue-card-result'
+                    host = div.search("*[@class='wa-hotelgeneral-gray wa-hotelgeneral-info-sub-title']").text
+                elsif div['class'] == 'result card-result wma-card-box' and div['srcid'] == 'map'
+                    host = 'map.baidu.com'
+                else
+                    host = div.search("*[@class='site']").first.text
+                end
+                {
+                    :text => a.text,
+                    :href => a['href'],
+                    :host => host
+                }
             end
         end
     end

data/lib/query/result/qihu.rb

@@ -0,0 +1,66 @@
+module Query
+  module Result
+    class Qihu
+      include Query::Result
+      def seo_ranks
+        @page.search('//ul[@id="m-result"]/li//h3').map.with_index do |h3,index|
+          a = h3.search('a').first
+          {
+            :rank => index + 1,
+            :href => a['href'],
+            :text => a.text.strip,
+            :host => Addressable::URI.parse(a['href']).host
+          }
+        end
+      end
+
+      def ads_top
+        @page.search("//ul[@id='djbox']/li").map.with_index do |li,index|
+          a = li.search("a").first
+          href = CGI.parse(URI(a['_cs']).query)['aurl'].first
+          {
+            :rank => index + 1,
+            :text => a.text,
+            :href => href,
+            :host => Addressable::URI.parse(URI.encode(href)).host
+          }
+        end
+      end
+
+      def ads_bottom
+        []
+      end
+
+      def ads_right
+        @page.search("//ul[@id='rightbox']/li").map.with_index do |li,index|
+          a = li.search('a').first
+          href = CGI.parse(URI(a['_cs']).query)['aurl'].first
+          host = Addressable::URI.parse(URI.encode(href)).host
+          {
+            :rank => index + 1,
+            :text => a.text,
+            :href => href,
+            :host => host
+          }
+        end
+      end
+
+      def related_keywords
+        []
+      end
+
+      def count
+        @page.search('//span[@class="nums"]').first.text.gsub(/\D/,'').to_i
+      end
+
+      #下一页
+      def next_url
+        next_href = @page.xpath('//a[@id="snext"]').first['href']
+      end
+      #有结果
+      def has_result?
+        !@page.search('//div[@id="main"]/h3').text().include?'没有找到该URL'
+      end
+    end
+  end
+end

data/lib/query/result/{qihoo_mobile.rb → qihu_mobile.rb}

@@ -1,6 +1,6 @@
 module Query
     module Result
-        class QihooMobile
+        class QihuMobile
         end
     end
 end