query 0.0.1 → 0.1.4

data/lib/query/result/sogou.rb

@@ -0,0 +1,103 @@
+module Query
+	module Result
+		class Sogou
+      include Query::Result
+			def ads_top
+        return [] if sponsored_divs.empty?
+        sponsored_divs.first.search("li").map.with_index do|li,index|
+					{
+            :rank => index + 1,
+            :text => li.css('h3 a').text,
+            :href => li.css('h3 a')[0]['href'],
+            :host => Addressable::URI.parse(li.css('cite')[0].text).host
+          }
+				end
+			end
+
+			def ads_right
+				@page.css('div#right div#bdfs0').map.with_index do |div,index|
+          {
+            :rank => index + 1,
+            :text => div.css('h3 a').text,
+            :href => div.css('h3 a')[0]['href'],
+            :host => Addressable::URI.parse(div.css('div.fb a cite').text).host
+          }
+				end
+			end
+
+      def ads_bottom
+        return [] if sponsored_divs.size < 2
+      end
+
+      def seo_ranks
+        # @seo_ranks ||= @page.search("div[@class='result']/div/h3").map do |h3|
+        @page.search("//div[@class='results']/div/h3").map.with_index do |h3,index|
+          {
+            :text => h3.search('a').first.text,
+            :href => h3.search('a').first['href'],
+            :host => Addressable::URI.parse(h3.search('a').first['href']).host,
+            :rank => index + 1
+          }
+        end
+      end
+
+      def count
+        ["//div[@class='zhanzhang']//em", "//span[@id='scd_num']"].each do |xpath|
+          if counter_block = @page.search(xpath).first
+            return counter_block.text.gsub(/\D/,'').to_i
+          end
+        end
+      end
+
+      def related_keywords
+        @related_keywords ||= @page.search("table[@id='hint_container']/td").map{|td|td.first.text}
+      end
+
+      def next_url
+        @page.search("//a[text()='下一页>']").first['href']
+      end
+
+      def has_result?
+        @page.search("div[@class='no-result']").empty?
+      end
+
+			# def rank(host)
+   #      raise "unknown host object type:#{host}" unless host.class == Regexp or host.class == String
+
+	  #     result = {}
+
+   #      #顶部广告排名
+   #      ranking_ads_top = 0
+   #      ads_top.each do |line|
+   #        ranking_ads_top += 1
+   #        if host.class == Regexp and line[:host] =~ host
+   #          result[:rank_top] = ranking_ads_top
+   #          break
+   #        elsif host.class == String and line[:host] == host
+   #          result[:rank_top] = ranking_ads_top
+   #          break
+   #        end
+   #      end
+
+   #      #右侧广告排名
+   #      ranking_ads_right = 0
+   #      ads_right.each do |line|
+   #        ranking_ads_right += 1
+   #        if host.class == Regexp and line[:host] =~ host
+   #          result[:rank_right] = ranking_ads_right
+   #          break
+   #        elsif host.class == String and line[:host] == host
+   #          result[:rank_right] = ranking_ads_right
+   #          break
+   #        end
+   #      end
+
+   #      result
+			# end
+      private
+      def sponsored_divs
+        @page.search("div[@class='sponsored']")
+      end
+		end
+	end
+end

data/lib/query/result/sogou_mobile.rb

@@ -0,0 +1,51 @@
+require 'cgi'
+module Query
+	module Result
+		class SogouMobile
+			include Query::Result
+			def ads_top
+				@page.search("//ul[@class='searchresult']/li[1]/preceding-sibling::div").map.with_index do |ad_div,index|
+					parse_ad(ad_div).merge({:rank => index + 1})
+				end
+			end
+
+			def ads_right
+				[]
+			end
+
+			def ads_bottom
+				@page.search("//ul[@class='searchresult']/li[last()]/following-sibling::div").map.with_index do |div,index|
+					parse_ad(div).merge({:rank => index + 1})
+				end
+			end
+
+			def seo_ranks
+				@seo_rank ||= @page.search("//ul[@class='searchresult']/li/a").map.with_index do |a,index|
+					href = URI.decode(CGI.parse(URI(URI.encode(a['href'])).query)['url'].first)
+					{
+						:rank => index + 1,
+						:text => a.search('h3').text,
+						:href => href,
+						:host => URI(href).host
+					}
+				end
+			end
+
+			def next_url
+				@page.search("//a[text()='下一页']").first['href']
+			end
+
+      def count
+      end
+
+      private
+      def parse_ad(ad_div)
+				{
+					:text => ad_div.search('h3').first.text,
+					:href => ad_div.search('a').first['href'],
+					:host => Addressable::URI.parse("http://#{ad_div.search('span[@class="site"]').text}").host
+				}
+      end
+		end
+	end
+end

data/lib/query/result.rb

@@ -1,10 +1,53 @@
 module Query
-    module Result
+  module Result
+    attr_accessor :baseuri,:pagenumber,:perpage
+    def initialize(page)
+      @page = Nokogiri::HTML page
+      @pagenumber = 1
     end
+    def raw_ranks
+      {
+        'ads_top'=>ads_top,
+        'ads_right'=>ads_right,
+        'ads_bottom'=>ads_bottom,
+        'seo_ranks'=>seo_ranks
+      }
+    end
+
+    def rank(host)#on base of ranks
+      @rank ||= %w(seo_ranks ads_top ads_right ads_bottom).map do |type_str|
+        result = nil
+        send(type_str).each_with_index do |line,index|
+          if host.class == Regexp
+            result = index + 1 and break if line[:host] =~ host
+          elsif host.class == String
+            result = index + 1 and break if line[:host] == host
+          else
+            result = false
+          end
+        end
+        result
+      end
+    end
+
+    def next
+      @next_url = URI.join(@baseuri,next_url).to_s
+      next_page = HTTParty.get @next_url
+      next_page = self.class.new(next_page)
+      next_page.baseuri = @next_url
+      next_page.pagenumber = @pagenumber + 1
+      next_page.perpage = @perpage
+      r = next_page
+      r.baseuri = next_url
+      r
+    end
+  end
 end
 require 'nokogiri'
-require 'query/result/base'
+require "addressable/uri"
 require 'query/result/baidu'
 require 'query/result/baidu_mobile'
-require 'query/result/qihoo'
-require 'query/result/qihoo_mobile'
+require 'query/result/qihu'
+require 'query/result/qihu_mobile'
+require 'query/result/sogou'
+require 'query/result/sogou_mobile'

data/lib/query/version.rb

@@ -1,3 +1,3 @@
 module Query
-  VERSION = "0.0.1"
+  VERSION = "0.1.4"
 end

data/lib/query.rb

@@ -1,9 +1,7 @@
-require "query/version"
-require "query/engine"
-require "query/result"
-require "httparty"
-require 'addressable/uri'
-require 'awesome_print'
-module Query
-  # Your code goes here...
+class MyFilter
+  def contains set, str
+    set.any? { |x| x.to_s.downcase == str.downcase}
+  end
 end
+require 'query/result'
+require 'query/engine'

data/query.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = Query::VERSION
   spec.authors       = ["seoaqua"]
   spec.email         = ["seoaqua@me.com"]
-  spec.description   = %q{This GEM is designed to work for SEOers who need to fetch query and parse results from all kinds of search engines}
-  spec.summary       = %q{Now its only support Chinese main search engines}
+  spec.description   = %q{This GEM is designed to work for Chinese SEOers who need to fetch query and parse results from all kinds of search engines}
+  spec.summary       = %q{I dont have time to write the document yet. Usage is almost within rspec tests. Any questions,pls contact me with QQ628552}
   spec.homepage      = "https://github.com/seoaqua/query"
   spec.license       = "MIT"
 
@@ -23,5 +23,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "nokogiri"
   spec.add_dependency "addressable"
   spec.add_dependency "httparty"
-
 end

data/spec/baidu1_spec.rb

@@ -0,0 +1,157 @@
+#coding:UTF-8
+require 'spec_helper'
+describe Query::Result::Baidu do
+  subject{Query::Result::Baidu.new(File.read($sample_baidu1))}
+
+  it "can click the next page button" do
+    subject.next_url.should == '/s?wd=%E5%90%8C%E7%A8%8B%E7%BD%91%E9%85%92%E5%BA%97%E9%A2%84%E8%AE%A2&pn=10&tn=baiduhome_pg&ie=utf-8&f=3&usm=2&rsv_page=1'
+  end
+
+  it "have 69200000 results" do
+    subject.count.should == 69200000
+  end
+
+
+  describe '#seo_ranks' do
+    it "puts www.17u.cn to be on first" do
+      subject.seo_ranks.first[:host].should == 'www.17u.cn'
+    end
+
+    it "should put 同程旅游网客服电话 to be the first title" do
+      subject.seo_ranks.first[:text].should == '同程旅游网客服电话'
+    end
+
+    it "should put 'http://www.17u.cn/' to be the second url" do
+      subject.seo_ranks[1][:href].should == 'http://www.17u.cn/'
+    end
+
+    it "should have href,text,host elements for each seo result" do
+      subject.seo_ranks.each do |seo_rank|
+        seo_rank[:href].should_not == nil
+        seo_rank[:text].should_not == nil
+        seo_rank[:host].should_not == nil
+      end
+    end
+  end
+
+  describe '#ads_top' do
+    it "should have 6 top ads" do
+      subject.ads_top.size.should == 3
+    end
+
+    it "should find hotel.elong.com at the first position in the top ads" do
+      subject.ads_top[0][:host].should == 'www.17u.cn'
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_top" do
+      subject.ads_top.class.should == Array
+      subject.ads_top.each do |ad_top|
+        ad_top.should have_key(:rank)
+        ad_top.should have_key(:host)
+        ad_top.should have_key(:href)
+        ad_top.should have_key(:text)
+      end
+    end
+  end
+
+  describe '#ads_right' do
+    it "should have 5 right ads" do
+      subject.ads_right.size.should == 5
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_right" do
+      subject.ads_right.class.should == Array
+      subject.ads_right.each do |ad_right|
+        ad_right.should have_key(:rank)
+        ad_right.should have_key(:host)
+        ad_right.should have_key(:href)
+        ad_right.should have_key(:text)
+      end
+    end
+  end
+
+  describe '#ads_bottom' do
+    it "should have zero bottom ads" do
+      subject.ads_bottom.size.should == 3
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_bottom" do
+      subject.ads_bottom.class.should == Array
+      subject.ads_bottom.each do |ad_bottom|
+        ad_bottom.should have_key(:rank)
+        ad_bottom.should have_key(:host)
+        ad_bottom.should have_key(:href)
+        ad_bottom.should have_key(:text)
+      end
+    end
+  end
+end
+__END__
+describe Query::Engine::Baidu do
+    page = Query::Engine::Baidu.query '百度'
+
+    it "should return Query::Result::Baidu" do
+        page.class.should == Query::Result::Baidu
+    end
+
+    it "should return 100,000,000" do
+        page.count.should > 100000
+    end
+    it "should return 1" do
+        page.rank('www.baidu.com').should == 1
+    end
+
+    it "should return Query::Result::Baidu" do
+        page.next.class.should == Query::Result::Baidu
+    end
+
+    it "should return true" do
+        bool = Query::Engine::Baidu.popular?'百度'
+        bool.should == true
+    end
+
+    it "should return false" do
+        bool = Query::Engine::Baidu.popular?'lavataliuming'
+        bool.should == false
+    end
+
+    it "should return over 5 words beginning with the query_word" do
+        query_word = '为'
+        suggestions = Query::Engine::Baidu.suggestions(query_word)
+        suggestions.size.should > 5
+        suggestions.each do |suggestion|
+            suggestion[0].should == query_word
+        end
+    end
+
+    it "should return 100,000,000" do
+        result = baidu.pages('baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+
+    it "should return 100,000,000" do
+        result = baidu.links('baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+    it "should return 100,000,000" do
+        result = baidu.pages_with('baidu.com','baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+    it "查询已经被收录的页面收录情况时,应返回true" do
+        baidu.indexed?('http://www.baidu.com').should == true
+    end
+    it "查询一个不存在的页面收录情况时,应返回true" do
+        baidu.indexed?('http://zxv.not-exists.com').should == false
+    end
+    page1 = Query::Engine::Baidu.query('seoaqua.com')
+    it "查询结果应该都能拿到title,href,host" do
+        page1.seo_ranks.each do |id,rank|
+            rank['href'].should_not == nil
+            rank['text'].should_not == nil
+            rank['host'].should_not == nil
+        end
+    end
+end

data/spec/baidu2_spec.rb

@@ -0,0 +1,156 @@
+#coding:UTF-8
+require 'spec_helper'
+describe Query::Result::Baidu do
+  subject{Query::Result::Baidu.new(File.read($sample_baidu2))}
+
+  it "can click the next page button" do
+    subject.next_url.should == '/s?wd=%E9%85%92%E5%BA%97%E9%A2%84%E8%AE%A2%E7%BD%91&pn=10&tn=baiduhome_pg&ie=utf-8&f=3&usm=1&rsv_page=1'
+  end
+
+  it "have 100000000 results" do
+    subject.count.should == 100000000
+  end
+
+  describe '#seo_ranks' do
+    it "should put hotel.qunar.com to be on first" do
+      subject.seo_ranks.first[:host].should == 'hotel.qunar.com'
+    end
+
+    it "should put 北京酒店预订_8371家特惠酒店_百度品质保证_去哪儿网提供 to be the first title" do
+      subject.seo_ranks.first[:text].should == '北京酒店预订_8371家特惠酒店_百度品质保证_去哪儿网提供'
+    end
+
+    it "should put 'http://jiudian.qunar.com/' to be the second url" do
+      subject.seo_ranks[1][:href].should == 'http://www.baidu.com/link?url=ZpGwUrZ8xUUgBQofg1TiNH1n_Ki3QWE62jvjkGvwwZ70wQPxFJxSD1uunh0uDwLM'
+    end
+
+    it "should have href,text,host elements for each seo result" do
+      subject.seo_ranks.each do |seo_rank|
+        seo_rank[:href].should_not == nil
+        seo_rank[:text].should_not == nil
+        seo_rank[:host].should_not == nil
+      end
+    end
+  end
+
+  describe '#ads_top' do
+    it "should have 6 top ads" do
+      subject.ads_top.size.should == 4
+    end
+
+    it "should find hotel.elong.com at the first position in the top ads" do
+      subject.ads_top[0][:host].should == 'www.agoda.com'
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_top" do
+      subject.ads_top.class.should == Array
+      subject.ads_top.each do |ad_top|
+        ad_top.should have_key(:rank)
+        ad_top.should have_key(:host)
+        ad_top.should have_key(:href)
+        ad_top.should have_key(:text)
+      end
+    end
+  end
+
+  describe '#ads_right' do
+    it "should have 8 right ads" do
+      subject.ads_right.size.should == 8
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_right" do
+      subject.ads_right.class.should == Array
+      subject.ads_right.each do |ad_right|
+        ad_right.should have_key(:rank)
+        ad_right.should have_key(:host)
+        ad_right.should have_key(:href)
+        ad_right.should have_key(:text)
+      end
+    end
+  end
+
+  describe '#ads_bottom' do
+    it "should have zero bottom ads" do
+      subject.ads_bottom.size.should == 0
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_bottom" do
+      subject.ads_bottom.class.should == Array
+      subject.ads_bottom.each do |ad_bottom|
+        ad_bottom.should have_key(:rank)
+        ad_bottom.should have_key(:host)
+        ad_bottom.should have_key(:href)
+        ad_bottom.should have_key(:text)
+      end
+    end
+  end
+end
+__END__
+describe Query::Engine::Baidu do
+    page = Query::Engine::Baidu.query '百度'
+
+    it "should return Query::Result::Baidu" do
+        page.class.should == Query::Result::Baidu
+    end
+
+    it "should return 100,000,000" do
+        page.count.should > 100000
+    end
+    it "should return 1" do
+        page.rank('www.baidu.com').should == 1
+    end
+
+    it "should return Query::Result::Baidu" do
+        page.next.class.should == Query::Result::Baidu
+    end
+
+    it "should return true" do
+        bool = Query::Engine::Baidu.popular?'百度'
+        bool.should == true
+    end
+
+    it "should return false" do
+        bool = Query::Engine::Baidu.popular?'lavataliuming'
+        bool.should == false
+    end
+
+    it "should return over 5 words beginning with the query_word" do
+        query_word = '为'
+        suggestions = Query::Engine::Baidu.suggestions(query_word)
+        suggestions.size.should > 5
+        suggestions.each do |suggestion|
+            suggestion[0].should == query_word
+        end
+    end
+
+    it "should return 100,000,000" do
+        result = baidu.pages('baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+
+    it "should return 100,000,000" do
+        result = baidu.links('baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+    it "should return 100,000,000" do
+        result = baidu.pages_with('baidu.com','baidu.com')
+        result.class.should == Query::Result::Baidu
+        result.count.should == 100000000
+    end
+    it "查询已经被收录的页面收录情况时,应返回true" do
+        baidu.indexed?('http://www.baidu.com').should == true
+    end
+    it "查询一个不存在的页面收录情况时,应返回true" do
+        baidu.indexed?('http://zxv.not-exists.com').should == false
+    end
+    page1 = Query::Engine::Baidu.query('seoaqua.com')
+    it "查询结果应该都能拿到title,href,host" do
+        page1.seo_ranks.each do |id,rank|
+            rank['href'].should_not == nil
+            rank['text'].should_not == nil
+            rank['host'].should_not == nil
+        end
+    end
+end