query 0.1.25 → 0.1.28

data/lib/query/result/baidu_mobile.rb

@@ -1,70 +1,128 @@
 module Query
-    module Result
-        class BaiduMobile
-            include Query::Result
+  module Result
+    class BaiduMobile
+      include Query::Result
 
-            def seo_ranks
-                @seo_ranks ||= @page.search("//*[@class='result']|//*[@class='card-result wa-ue-card-result']|//*[@class='result card-result wma-card-box']").map.with_index do |div,index|
-                    parse_seo(div).merge({:rank => index + 1})
-                end
-            end
+      def seo_ranks
+        s_res =  @page.at("//div[@id='results']")
+        @seo_ranks ||= s_res.css("div.result").map.with_index do |seo_div,index|
+          parse_seo(seo_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
 
-            def ads_top
-                @ads_top ||= @page.search("//*[@class='result']/preceding-sibling::div[@class='ec_wise_ad']/div").map.with_index do |div,index|
-                    puts index
-                    parse_ad(div).merge({:rank => index + 1})
-                end
-            end
+      def ads_top
+        selector = "//*[@class='result']/preceding-sibling::div[not (contains(@class,'result'))]/div/div/a[not (contains(@href,'http://baozhang.baidu.com/guarantee'))]/.."
+        @ads_top ||= @page.search(selector).map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
 
-            def ads_right
-                []
-            end
+      def ads_bottom
+        selector = "//*[@class='result']/following-sibling::div[not (contains(@class,'result'))]/div/div/a[not (contains(@href,'http://baozhang.baidu.com/guarantee'))]/.."
+        @ads_bottom ||= @page.search(selector).map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
 
-            def ads_bottom
-                @ads_bottom ||= @page.search("//*[@class='result']/following-sibling::div[@class='ec_wise_ad']/div/div").map.with_index do |div,index|
-                    parse_ad(div).merge({:rank => index + 1})
-                end
-            end
+      #酒店预订 酒店英文 酒店团购 酒店管理 酒店招聘 快捷酒店 如家快捷酒店 五星级酒店
+      def related_keywords
+        @related_keywords ||= @page.search("//div[@id='relativewords']/div[@class='rw-list']/a").map { |a| a.text }
+      end
 
-            #酒店预订 酒店英文 酒店团购 酒店管理 酒店招聘 快捷酒店 如家快捷酒店 五星级酒店
-            def related_keywords
-                @related_keywords ||= @page.search("div[@class='rw-list']/a").map{|a|a.text}
-            end
+      def html
+        @page.to_html
+      end
 
-            def next_url
-               @next_url ||= @page.xpath('//a[contains(text(),"下一页")]').first['href']
-            end
+      def ads_right
+        []
+      end
 
-            def count
+      def next_url
+      	next_bn = @page.search("//div[@id='pagenav']/a").first
+        url = next_bn.nil? ? "/s?#{@baseuri.query}&pn=#{@pagenumber*10}" : next_bn['href']
+        url
+      end
 
-            end
+      def count
 
-            private
-            def parse_ad(div)
-                url = div.search("span[@class='ec_site']").first.text
-                url = "http://#{url}"
-                {
-                    :text => div.search('a/text()').text.strip,
-                    :href => div.search('a').first['href'],
-                    :host => Addressable::URI.parse(URI.encode(url)).host
-                }
-            end
+      end
 
-            def parse_seo(div)
-                a = div.search('a').first
-                if div['class'] == 'card-result wa-ue-card-result'
-                    host = div.search("*[@class='wa-hotelgeneral-gray wa-hotelgeneral-info-sub-title']").text
-                elsif div['class'] == 'result card-result wma-card-box' and div['srcid'] == 'map'
-                    host = 'map.baidu.com'
-                else
-                    host = div.search("*[@class='site']").first.text
-                end
-                {
-                    :text => a.text,
-                    :href => a['href'],
-                    :host => host
-                }
+    private 
+      def parse_ad(ad_div)
+        begin
+          title_link = ad_div.search('a')[0]
+          url = ad_div.search('link')
+          if url.empty?
+            url = ad_div.search(".//span[contains(text(),'.com')]")[0]
+            url = url.nil? ? "http://m.baidu.com" : "http://#{url.text.strip}"
+            title = title_link.text
+          else
+            url = url[0]['href']
+            title = title_link.xpath("./text() | ./em").text
+          end
+           {
+              :text => title.gsub(/\n|\s/,''),
+              :href => title_link['href'],
+              :host => Addressable::URI.parse(URI.encode(url)).host
+           }
+        rescue Exception => e
+         warn "Error in parse_seo method : " + e.message
+         {}
+        end
+      end
+
+      def parse_seo(seo_div)
+        begin
+          title_link = seo_div.search('a')[0]
+          href = title_link['href']
+          href = href[/m.baidu.com/] ? href : "http://m.baidu.com#{href}"
+          if seo_div['class']=='result'
+            host, is_vr = seo_div.search(".//*[@class='site']")[0], false
+            host = host.nil? ? find_host(seo_div) : host.text.split[0] 
+          elsif seo_div['srcid']=='map'
+            is_vr, host = true, 'map.baidu.com'          
+          elsif seo_div['tpl'] and seo_div['data-log']
+            url = JSON.parse(seo_div['data-log'].gsub("'",'"'))['mu']
+            if url==''
+            	host = find_host(seo_div)
+            else
+              host = Addressable::URI.parse(URI.encode(url)).host
             end
+            is_vr = true
+          else 
+          	is_vr, host = true, find_host(seo_div)
+          end
+          #is_vr = (is_vr.nil? and !host[/baidu|nuomi/]) ? false : true     
+          {   
+            :is_vr => false || is_vr,
+            :text  => title_link.text.gsub(/\n|\s/,'')[0..30],
+            :href  => href,
+            :host  => host
+          }
+        rescue Exception => e
+          warn "Error in parse_seo method : " + e.message
+          {}
+        end
+      end
+      
+			def find_host(node)
+				host = node.search(".//*[name()!='style' and (contains(text(),'.cn') or contains(text(),'com'))]")[0]
+				host.nil? ? 'm.baidu.com' : host.text.split[0]
+			end
+			
+      def redirect(url,limit = 10) 
+        raise ArgumentError, 'Too many HTTP redirects' if limit == 0
+        response = Net::HTTP.get_response(URI(url))
+        case response
+          when Net::HTTPSuccess then
+            return URI(url).host
+          when Net::HTTPRedirection then
+            location = response['location']
+            redirect(location, limit-1)
+          else
+            return "m.baidu.com"
         end
+      end 
     end
-end
+  end
+end

data/lib/query/result/qihu_mobile.rb

@@ -1,6 +1,51 @@
 module Query
-    module Result
-        class QihuMobile
+  module Result
+    class QihuMobile
+      include Query::Result
+      
+      def html
+        @page.to_html
+      end
+      
+      def next_url
+       "#{@baseuri.to_s}&pn=#{@pagenumber+1}"
+      end
+      
+      def seo_ranks
+      	@page.css('div.g-card').map.with_index do |seo_div,index|
+      		begin
+      			cite = seo_div.at('.//*[@class="res-show-url"]/text()')
+      			a = seo_div.at_css('a')	
+      			if cite
+      				cite = cite.to_s.gsub(/ |-/,'') 
+      			else   
+      				url = seo_div.at('.//a[contains(@href,"u=")]') 
+      			  if url
+					  		cite = URI.decode(CGI.parse(URI(URI.encode(url['href'])).query)['u'][0])
+					  		cite = URI(URI.encode(cite)).host
+					  	else
+					  		cite = "m.haosou.com"
+					  	end
+          	end
+          	title = seo_div.at_css('h3') || a        	
+          	{	
+          		:is_vr => seo_div['class']=="g-card r-og-card" ? false : true,
+            	:rank  => index + 1 + (@pagenumber-1)*10,
+            	:href  => a['href'],
+            	:text  => title.text.gsub(/　|\n|\s/,""),
+            	:host  => cite
+          	}
+          rescue Exception => e
+          	warn "Error in parse_seo method : " + e.message
+          	{}
+          end
         end
+      end
+     
+      def related_keywords
+        @related_keywords ||= @page.search("//div[@class='related-search-b']//a").map{|relwd| relwd.text.gsub(/ |\n|\t/,"")}
+      end
+      
     end
-end
+  end
+end

data/lib/query/result/sm_mobile.rb

@@ -0,0 +1,95 @@
+module Query
+  module Result
+    class SMobile
+      include Query::Result
+
+      def seo_ranks
+        @seo_ranks ||= @page.search("//div[@id='results']/div[@class!='ali_row result card']").map.with_index do |seo_div,index|
+          parse_seo(seo_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
+
+      def ads_top
+        selector = "//div[@id='results']/div[@class='result card'][1]/preceding-sibling::div[@class='ali_row result card']"
+        @ads_top ||= @page.search(selector).map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
+
+      def ads_bottom
+        selector = "//div[@id='results']/div[@class='result card'][1]/following-sibling::div[@class='ali_row result card']"
+        @ads_bottom ||= @page.search(selector).map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (index + 1) + (@pagenumber -1) * 10})
+        end
+      end
+
+      #relative words
+      def related_keywords
+        @related_keywords ||= @page.search("//div[@class='rel-keywords card']/ul/li/a").map { |a| a.text }
+      end
+
+      def html
+        @page.to_html
+      end
+
+      def ads_right
+        []
+      end
+
+      def next_url
+        	"#{@baseuri.to_s}&page=#{@pagenumber+1}"
+      end
+
+      def count
+
+      end
+
+    private 
+      def parse_ad(ad_div)
+        begin
+          title_link = ad_div.at_css('a') 
+          title = title_link.search('./text()|./em|./span')
+          url = ad_div.search('.//div[@class="host"]/text()').text
+          url = "http://#{url}" if !url[/http:/]   
+           {
+              :text => title.text.gsub(/\n|\s/,''),
+              :href => title_link['href'],
+              :host => URI(URI.encode(url.gsub(/ |\n|\t|\s/,""))).host
+           }
+        rescue Exception => e
+         warn "Error in parse_ads method : " + e.message
+         {}
+        end
+      end
+
+      def parse_seo(seo_div)	
+        begin
+          title_link = seo_div.at('.//a[contains(@href,"http://")]')
+          href = title_link['href']
+          if seo_div['class']=="result card"
+            is_vr = false
+            url   = seo_div.search('.//div[@class="host"]/span/text()[matches(.,"\w+.\w+")]', XpathFunctions.new)[0] || href
+          else
+            is_vr, url = true, href
+          end
+          url = "http://#{url}" if !url[/http:/]   
+          {   
+            :is_vr => is_vr,
+            :text  => title_link.text.gsub(/\n|\s/,'')[0..30],
+            :href  => href,
+            :host  => URI(URI.encode(url.gsub(/ |\n|\t|\s/,""))).host # remove &nbsp and whitespace
+          }
+        rescue Exception => e
+          warn "Error in parse_seo method : " + e.message
+          {}
+        end
+      end
+      
+      class XpathFunctions 
+      	def matches node_set, regex
+       		node_set.find_all {|node| node.to_s[/#{regex}/] }
+      	end
+      end
+    end
+  end
+end

data/lib/query/result/sogou.rb

@@ -42,11 +42,8 @@ module Query
       end
 
       def count
-        ["//div[@class='zhanzhang']//em", "//span[@id='scd_num']"].each do |xpath|
-          if counter_block = @page.search(xpath).first
-            return counter_block.text.gsub(/\D/,'').to_i
-          end
-        end
+        node = @page.search("//resnum[@id='scd_num']").first
+        node ? node.text.gsub(/\D/,'').to_i : nil
       end
 
       def related_keywords

data/lib/query/result/sogou_mobile.rb

@@ -1,51 +1,91 @@
 require 'cgi'
 module Query
-	module Result
-		class SogouMobile
-			include Query::Result
-			def ads_top
-				@page.search("//ul[@class='searchresult']/li[1]/preceding-sibling::div").map.with_index do |ad_div,index|
-					parse_ad(ad_div).merge({:rank => index + 1})
-				end
-			end
-
-			def ads_right
-				[]
-			end
+  module Result
+    class SogouMobile
+      include Query::Result
+      def ads_top
+        @page.search("//div[@class='results']/div[@class='ec_ad_results'][1]/div[@class='ad_result']").map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (@pagenumber-1)*10 + index + 1})
+        end
+      end
 
-			def ads_bottom
-				@page.search("//ul[@class='searchresult']/li[last()]/following-sibling::div").map.with_index do |div,index|
-					parse_ad(div).merge({:rank => index + 1})
-				end
-			end
+      def ads_right
+        []
+      end
 
-			def seo_ranks
-				@seo_rank ||= @page.search("//ul[@class='searchresult']/li/a").map.with_index do |a,index|
-					href = URI.decode(CGI.parse(URI(URI.encode(a['href'])).query)['url'].first)
-					{
-						:rank => index + 1,
-						:text => a.search('h3').text,
-						:href => href,
-						:host => URI(href).host
-					}
-				end
-			end
+      def ads_bottom
+        @page.search("//div[@class='results']/div[@class='ec_ad_results'][2]/div[@class='ad_result']").map.with_index do |ad_div,index|
+          parse_ad(ad_div).merge({:rank => (@pagenumber-1)*10 + index + 1})
+        end
+      end
 
-			def next_url
-				@page.search("//a[text()='下一页']").first['href']
-			end
+      def seo_ranks
+        @seo_rank ||= @page.search("//div[@class='results']/div[@class='result' or @class='vrResult']").map.with_index do |seo_div,index|
+          parse_seo_ranks(seo_div).merge({:rank => (@pagenumber-1)*10 + index + 1})
+        end
+      end
 
+      def next_url
+        "#{@baseuri.to_s}&p=#{@pagenumber+1}"
+      end
+      
+      def related_keywords
+        @related_keywords ||= @page.search("div[@class='hint']/ul/li/a").map{|relwd| relwd.text.gsub(/ |\n|\t/,"")}
+      end
+      
       def count
       end
+      
+      def html
+        @page.to_html
+      end
 
       private
       def parse_ad(ad_div)
-				{
-					:text => ad_div.search('h3').first.text,
-					:href => ad_div.search('a').first['href'],
-					:host => Addressable::URI.parse("http://#{ad_div.search('span[@class="site"]').text}").host
-				}
-      end
-		end
-	end
+      	begin
+        	site = ad_div.search(".//span[@class='exp_tip']/preceding-sibling::span")[0] || ad_div.search(".//div[@class='bd_citeurl']/text()")[0]
+        	{
+          	:text => ad_div.search('h3')[0].text.gsub(/ |\n|\t/,""),
+          	:href => ad_div.search('a')[0]['href'],
+          	:host => site.text.strip.downcase
+        	}
+        rescue Exception => e
+          warn "Error in parse_ads method : " + e.message
+          {}
+        end
+      end
+      
+      def parse_seo_ranks(seo_div)
+        begin
+        a    = seo_div.search(".//a[contains(@href,'url=')]")[0]
+        cite_url = URI.decode(CGI.parse(URI(URI.encode(a['href'])).query)['url'][0])
+        
+        if cite_url==""
+          cite_url  = seo_div.search(".//div[@class='citeurl']/text()")[0] || "wap.sogou.com"
+          cite_url  = "http://#{cite_url.to_s.gsub(/ |-/,'')}"
+        end
+        
+        if seo_div['class']=='result'       
+          is_vr, title = false, a.search("./text()|./em|./span")     
+        else
+          title     = seo_div.search(".//h3")[0] || a
+          is_vr     = true
+          title.css('script').remove
+        end
+        	url       = a['href'][/wap.sogou.com\/web/].nil? ? "http://wap.sogou.com/web/#{a['href']}" : a['href'] 
+        	
+          {
+            :text   => title.text.gsub(/ |\n|\t/,""),
+            :href   => url,
+            :host   => URI(URI.encode(cite_url)).host,
+            :is_vr  => is_vr
+          }
+        rescue Exception => e
+          warn "Error in parse_seo method : " + e.message
+          {}
+        end
+      end
+      
+    end
+  end
 end