query 0.0.1 → 0.1.4

data/spec/sogou_mobile_spec.rb

@@ -0,0 +1,86 @@
+#coding:UTF-8
+require 'spec_helper'
+describe Query::Result::SogouMobile do
+    subject{Query::Engine::SogouMobile.query('酒店预订')}
+    it "is an instance of #{Query::Result::SogouMobile}" do
+        subject.class.should == Query::Result::SogouMobile
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_top" do
+        subject.ads_top.class.should == Array
+        subject.ads_top.each do |ad_top|
+            ad_top.should have_key(:rank)
+            ad_top.should have_key(:host)
+            ad_top.should have_key(:href)
+            ad_top.should have_key(:text)
+        end
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_right" do
+        subject.ads_right.class.should == Array
+        subject.ads_right.each do |ad_right|
+            ad_right.should have_key(:rank)
+            ad_right.should have_key(:host)
+            ad_right.should have_key(:href)
+            ad_right.should have_key(:text)
+        end
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_bottom" do
+        subject.ads_bottom.class.should == Array
+        subject.ads_bottom.each do |ad_bottom|
+            ad_bottom.should have_key(:rank)
+            ad_bottom.should have_key(:host)
+            ad_bottom.should have_key(:href)
+            ad_bottom.should have_key(:text)
+        end
+    end
+end
+
+result = Query::Engine::SogouMobile.query('酒店预订')
+ads_top = result.ads_top
+describe "types check" do
+    it "should return Query::Result::SogouMobile" do
+	    result.class.should == Query::Result::SogouMobile
+	end
+
+    it "should return Array" do
+		ads_top.class.should == Array
+    end
+
+    it "should has keys" do
+        ads_top[0].should have_key(:rank)
+        ads_top[0].has_key?(:domain)
+        ads_top[0].has_key?(:host)
+        ads_top[0].has_key?(:href)
+        ads_top[0].has_key?(:title)
+    end
+end
+
+describe Query::Result::SogouMobile do
+    subject{Query::Engine::SogouMobile.query '中华人民共和国中央人民政府'}
+
+    it "should be an instance of Query::Result::Sogou" do
+        subject.class.should == Query::Result::SogouMobile
+    end
+
+    it "'s next page is another instance of Query::Result::Sogou" do
+        subject.next.class.should == Query::Result::SogouMobile
+    end
+
+    it "have over 1000 results" do
+        subject.count.should be_nil
+    end
+
+    it "puts www.gov.cn to the first place of seo_ranks" do
+        subject.rank('www.gov.cn')[0].should == 1
+    end
+
+    it "should have href,text,host elements for each seo result" do
+        subject.seo_ranks.each do |seo_rank|
+            seo_rank[:href].should_not == nil
+            seo_rank[:text].should_not == nil
+            seo_rank[:host].should_not == nil
+        end
+    end
+end

data/spec/sogou_spec.rb

@@ -0,0 +1,107 @@
+#coding:UTF-8
+require 'spec_helper'
+
+describe Query::Result::Sogou do
+    subject{Query::Result::Sogou.new(File.read($sample_sogou))}
+
+    it "has an array of hashes with the required keys as the result of ads_top" do
+        subject.ads_top.class.should == Array
+        subject.ads_top.each do |ad_top|
+            ad_top.should have_key(:rank)
+            ad_top.should have_key(:host)
+            ad_top.should have_key(:href)
+            ad_top.should have_key(:text)
+        end
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_right" do
+        subject.ads_right.class.should == Array
+        subject.ads_right.each do |ad_right|
+            ad_right.should have_key(:rank)
+            ad_right.should have_key(:host)
+            ad_right.should have_key(:href)
+            ad_right.should have_key(:text)
+        end
+    end
+
+    it "has an array of hashes with the required keys as the result of ads_bottom" do
+        subject.ads_bottom.class.should == Array
+        subject.ads_bottom.each do |ad_bottom|
+            ad_bottom.should have_key(:rank)
+            ad_bottom.should have_key(:host)
+            ad_bottom.should have_key(:href)
+            ad_bottom.should have_key(:text)
+        end
+    end
+
+end
+
+describe Query::Result::Sogou do
+    subject{Query::Engine::Sogou.query '中华人民共和国中央人民政府'}
+
+    it "should be an instance of Query::Result::Sogou" do
+        subject.class.should == Query::Result::Sogou
+    end
+
+    it "'s next page is another instance of Query::Result::Sogou" do
+        subject.next.class.should == Query::Result::Sogou
+    end
+
+    it "have over 1000 results" do
+        subject.count.should > 1000
+    end
+
+    it "puts www.gov.cn to the first place of seo_ranks" do
+        subject.rank('www.gov.cn')[0].should == 1
+    end
+
+    it "should have href,text,host elements for each seo result" do
+        subject.seo_ranks.each do |seo_rank|
+            seo_rank[:href].should_not == nil
+            seo_rank[:text].should_not == nil
+            seo_rank[:host].should_not == nil
+        end
+    end
+end
+
+describe Query::Engine::Sogou do
+    subject{Query::Engine::Sogou}
+
+    it "have more than 1 million baidu.com pages indexed" do
+        result = subject.pages('baidu.com')
+        result.class.should == Query::Result::Sogou
+        result.count.should > 1000000
+    end
+
+    it "have more than 100 links to baidu.com" do
+        result = subject.links('baidu.com')
+        result.class.should == Query::Result::Sogou
+        result.count.should > 100
+    end
+
+    it "查询已经被收录的页面收录情况时,应返回true" do
+        pending
+        subject.indexed?('http://www.baidu.com').should == true
+    end
+
+    it "查询一个不存在的页面收录情况时,应返回true" do
+        pending
+        subject.indexed?('http://zxv.not-exists.com').should == false
+    end
+
+    describe '#suggestions' do
+        query = '搜狗'
+        subject{Query::Engine::Sogou.suggestions(query)}
+        it 'should have more than one suggestions' do
+            subject.size.should > 1
+        end
+
+        it 'gives all suggestions with the query word at the start' do
+            subject.each do |suggestion|
+                suggestion.should start_with query
+                # (suggestion.start_with?query).should_be true
+            end
+        end
+    end
+end
+

data/spec/spec_helper.rb

@@ -1 +1,12 @@
-require 'query'
+require 'query'
+require 'pathname'
+require 'awesome_print'
+path_root = Pathname.new(__dir__)
+path_samples = File.join(path_root,'samples')
+$sample_qihu = File.join(path_samples,'qihu.html')
+$sample_sogou = File.join(path_samples,'sogou.html')
+$sample_msogou = File.join(path_samples,'msogou.html')
+$sample_baidu1 = File.join(path_samples,'baidu1.html')
+$sample_baidu2 = File.join(path_samples,'baidu2.html')
+$sample_mbaidu1 = File.join(path_samples,'mbaidu1.html')
+$sample_mbaidu2 = File.join(path_samples,'mbaidu2.html')

metadata

@@ -1,94 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: query
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.4
 platform: ruby
 authors:
 - seoaqua
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-03 00:00:00.000000000 Z
+date: 2014-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This GEM is designed to work for SEOers who need to fetch query and parse
-  results from all kinds of search engines
+description: This GEM is designed to work for Chinese SEOers who need to fetch query
+  and parse results from all kinds of search engines
 email:
 - seoaqua@me.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE
 - LICENSE.txt
@@ -98,20 +98,33 @@ files:
 - lib/query/engine.rb
 - lib/query/engine/baidu.rb
 - lib/query/engine/baidu_mobile.rb
-- lib/query/engine/base.rb
-- lib/query/engine/qihoo.rb
-- lib/query/engine/qihoo_mobile.rb
+- lib/query/engine/qihu.rb
+- lib/query/engine/qihu_mobile.rb
+- lib/query/engine/sogou.rb
+- lib/query/engine/sogou_mobile.rb
 - lib/query/result.rb
 - lib/query/result/baidu.rb
 - lib/query/result/baidu_mobile.rb
-- lib/query/result/base.rb
-- lib/query/result/qihoo.rb
-- lib/query/result/qihoo_mobile.rb
+- lib/query/result/qihu.rb
+- lib/query/result/qihu_mobile.rb
+- lib/query/result/sogou.rb
+- lib/query/result/sogou_mobile.rb
 - lib/query/version.rb
 - query.gemspec
-- spec/baidu_mobile_spec.rb
-- spec/baidu_spec.rb
-- spec/qihoo_spec.rb
+- spec/baidu1_spec.rb
+- spec/baidu2_spec.rb
+- spec/mbaidu1_spec.rb
+- spec/msogou_spec.rb
+- spec/qihu_spec.rb
+- spec/samples/baidu1.html
+- spec/samples/baidu2.html
+- spec/samples/mbaidu1.html
+- spec/samples/mbaidu2.html
+- spec/samples/msogou.html
+- spec/samples/qihu.html
+- spec/samples/sogou.html
+- spec/sogou_mobile_spec.rb
+- spec/sogou_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/seoaqua/query
 licenses:
@@ -123,22 +136,34 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.5
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
-summary: Now its only support Chinese main search engines
+summary: I dont have time to write the document yet. Usage is almost within rspec
+  tests. Any questions,pls contact me with QQ628552
 test_files:
-- spec/baidu_mobile_spec.rb
-- spec/baidu_spec.rb
-- spec/qihoo_spec.rb
+- spec/baidu1_spec.rb
+- spec/baidu2_spec.rb
+- spec/mbaidu1_spec.rb
+- spec/msogou_spec.rb
+- spec/qihu_spec.rb
+- spec/samples/baidu1.html
+- spec/samples/baidu2.html
+- spec/samples/mbaidu1.html
+- spec/samples/mbaidu2.html
+- spec/samples/msogou.html
+- spec/samples/qihu.html
+- spec/samples/sogou.html
+- spec/sogou_mobile_spec.rb
+- spec/sogou_spec.rb
 - spec/spec_helper.rb

data/lib/query/engine/base.rb

@@ -1,16 +0,0 @@
-module Query
-    module Engine
-        class Base
-            attr_accessor :perpage
-           #是否收录
-            # def initialize(perpage = 100)
-            #     @perpage = perpage#只允许10或100
-            # end
-            def indexed?(url)
-                URI(url)
-                result = query(url)
-                return result.has_result?
-            end
-        end
-    end
-end

data/lib/query/result/base.rb

@@ -1,50 +0,0 @@
-module Query
-    module Result
-        class Base
-            attr_accessor :baseuri,:pagenumber,:perpage
-            def initialize(page)
-                @page = Nokogiri::HTML page
-                @pagenumber = 1
-            end
-            # def initialize(page,baseuri,pagenumber=1,perpage=100)
-            #     @page = Nokogiri::HTML page
-            #     @baseuri = baseuri
-            #     # @host = URI(baseuri).host
-            #     @pagenumber = pagenumber
-            #     @perpage = perpage
-            # end
-            def whole
-                {
-                    'ads_top'=>ads_top,
-                    'ads_right'=>ads_right,
-                    'ads_bottom'=>ads_bottom,
-                    'ranks'=>ranks
-                }
-            end
-            #返回当前页中host满足条件的结果
-            def ranks_for(specific_host)
-                host_ranks = Hash.new
-                ranks.each do |id,line|
-                    if specific_host.class == Regexp
-                        host_ranks[id] = line if line['host'] =~ specific_host
-                    elsif specific_host.class == String
-                        host_ranks[id] = line if line['host'] == specific_host
-                    end
-                end
-                host_ranks
-            end
-            #return the top rank number from @ranks with the input host
-            def rank(host)#on base of ranks
-                ranks.each do |id,line|
-                    id = id.to_i
-                    if host.class == Regexp
-                        return id if line['host'] =~ host
-                    elsif host.class == String
-                        return id if line['host'] == host
-                    end
-                end
-                return nil
-            end
-        end
-    end
-end

data/lib/query/result/qihoo.rb

@@ -1,75 +0,0 @@
-module Query
-    module Result
-        class Qihoo < Base
-            # include Query::Result
-            Host = 'www.so.com'
-            #返回所有当前页的排名结果
-            def ranks
-                return @ranks unless @ranks.nil?
-                @ranks = Hash.new
-                # id = (@pagenumber - 1) * 10
-                id = 0
-                @page.search('//li[@class="res-list"]').each do |li|
-                    a = li.search("h3/a").first
-                    url = li.search("cite")
-                    next if a['data-pos'].nil?
-                    id += 1
-                    text = a.text.strip
-                    href = a['href']
-                    url = url.first.text
-                    host = Addressable::URI.parse(URI.encode("http://#{url}")).host
-                    @ranks[id.to_s] = {'href'=>a['href'],'text'=>text,'host'=>host}
-                end
-                @ranks
-            end
-            def ads_top
-                id = 0
-                result = []
-                @page.search("//ul[@id='djbox']/li").each do |li|
-                    id += 1
-                    title = li.search("a").first.text
-                    href = li.search("cite").first.text.downcase
-                    host = Addressable::URI.parse(URI.encode(href)).host
-                    result[id] = {'title'=>title,'href'=>href,'host'=>host}
-                end
-                result
-            end
-            def ads_bottom
-                []
-            end
-            def ads_right
-                id = 0
-                result = []
-                @page.search("//ul[@id='rightbox']/li").each do |li|
-                    id += 1
-                    title = li.search("a").first.text
-                    href = li.search("cite").first.text.downcase
-                    host = Addressable::URI.parse(URI.encode(href)).host
-                    result[id] = {'title'=>title,'href'=>href,'host'=>host}
-                end
-                result
-            end
-            def related_keywords
-                []
-            end
-            #下一页
-            def next
-                next_href = @page.xpath('//a[@id="snext"]')
-                return false if next_href.empty?
-                next_href = next_href.first['href']
-                next_href = URI.join(@baseuri,next_href).to_s
-                # next_href = URI.join("http://#{@host}",next_href).to_s
-                next_page = HTTParty.get(next_href).next
-                r =Query::Result::Qihoo.new(next_page)
-                r.baseuri=next_href
-                r.pagenumber=@pagenumber+1
-                r
-                #@page = MbaiduResult.new(Mechanize.new.click(@page.link_with(:text=>/下一页/))) unless @page.link_with(:text=>/下一页/).nil?
-            end
-            #有结果
-            def has_result?
-                !@page.search('//div[@id="main"]/h3').text().include?'没有找到该URL'
-            end
-        end
-    end
-end

data/spec/baidu_mobile_spec.rb

@@ -1,19 +0,0 @@
-#coding:UTF-8
-require 'spec_helper'
-describe Query::Engine::BaiduMobile do
-    mbaidu = Query::Engine::BaiduMobile.new
-    page = mbaidu.query '百度'
-    it "应返回#{Query::Engine::BaiduMobile}" do
-        page.class.should == Query::Result::BaiduMobile
-    end
-    it "下一页也应是Query::Engine::BaiduMobile" do
-        page.next.class.should == Query::Result::BaiduMobile
-        page.next.next.class.should == Query::Result::BaiduMobile
-    end
-    it "百度百科域名应该大于1" do
-        page.rank('wapbaike.baidu.com').should > 1
-    end
-    it "百度无线域名应该在10以内" do
-        page.rank('m.baidu.com').should < 11
-    end
-end

data/spec/baidu_spec.rb

@@ -1,73 +0,0 @@
-#coding:UTF-8
-require 'spec_helper'
-describe Query::Engine::Baidu do
-    baidu = Query::Engine::Baidu.new
-    page = baidu.query '百度'
-
-    it "should return Query::Result::Baidu" do
-        page.class.should == Query::Result::Baidu
-    end
-
-    it "should return 100,000,000" do
-        page.count.should > 100000
-    end
-    it "should return 1" do
-        page.rank('www.baidu.com').should == 1
-    end
-
-    it "should return Query::Result::Baidu" do
-        page.next.class.should == Query::Result::Baidu
-    end
-
-    it "should return true" do
-        bool = Query::Engine::Baidu.popular?'百度'
-        bool.should == true
-    end
-
-    it "should return false" do
-        bool = Query::Engine::Baidu.popular?'lavataliuming'
-        bool.should == false
-    end
-
-    it "should return over 5 words beginning with the query_word" do
-        query_word = '为'
-        suggestions = Query::Engine::Baidu.suggestions(query_word)
-        suggestions.size.should > 5
-        suggestions.each do |suggestion|
-            suggestion[0].should == query_word
-        end
-    end
-
-    it "should return 100,000,000" do
-        result = baidu.pages('baidu.com')
-        result.class.should == Query::Result::Baidu
-        result.count.should == 100000000
-    end
-
-    it "should return 100,000,000" do
-        result = baidu.links('baidu.com')
-        result.class.should == Query::Result::Baidu
-        result.count.should == 100000000
-    end
-    it "should return 100,000,000" do
-        result = baidu.pages_with('baidu.com','baidu.com')
-        result.class.should == Query::Result::Baidu
-        result.count.should == 100000000
-    end
-    it "查询已经被收录的页面收录情况时,应返回true" do
-        baidu.indexed?('http://www.baidu.com').should == true
-    end
-    it "查询一个不存在的页面收录情况时,应返回true" do
-        baidu.indexed?('http://zxv.not-exists.com').should == false
-    end
-    page1 = baidu.query('seoaqua.com')
-    it "查询结果应该都能拿到title,href,host" do
-        page1.ranks.each do |id,rank|
-            rank['href'].should_not == nil
-            rank['text'].should_not == nil
-            rank['host'].should_not == nil
-        end
-    end
-    # ads_page = baidu.query '减肥药'
-
-end

data/spec/qihoo_spec.rb

@@ -1,27 +0,0 @@
-#coding:UTF-8
-require 'spec_helper'
-describe Query::Engine::Qihoo do
-    qihoo = Query::Engine::Qihoo.new
-    page = qihoo.query '奇虎'
-    page2 = page.next
-    page3 = page2.next
-    it "查询关键词'奇虎'后,应返回正确的实例" do
-        page.class.should == Query::Result::Qihoo
-    end
-    it "查询关键词'奇虎'后,下一页也应是MbaiduResult的实例" do
-        page2.class.should == Query::Result::Qihoo
-    end
-    it "查询关键词'奇虎'后,下一页,再下一页也应是MbaiduResult的实例" do
-        page3.class.should == Query::Result::Qihoo
-    end
-
-    it "查询关键词'奇虎'后,奇虎首页域名应该等于1" do
-        page.rank('www.qihoo.com').should == 1
-    end
-    it "查询已经被收录的页面收录情况时,应返回true" do
-        qihoo.indexed?('http://www.360.cn').should == true
-    end
-    it "查询一个不存在的页面收录情况时,应返回true" do
-        qihoo.indexed?('http://zxv.not-exists.com').should == false
-    end
-end