google_scraper_gem 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 277c49c5bcf1ef1089cc6d5ecdac41d99406d284
-  data.tar.gz: c6c612b686fd655435d155d97a9612b0bda4a95c
+  metadata.gz: 698dc969f95efd9b896a34d2673a62dd6c066ef1
+  data.tar.gz: bfd26d90a6d9214c56313c0e8e70d60b09a61bfa
 SHA512:
-  metadata.gz: 5dd8033aca597b6436745f79d63fb8a565ae49208aef48d8ad24bebaad48880112abf933fc2545a024d09f8ef95e2fb3fc08351b663f1b72a08642f0656b9e21
-  data.tar.gz: 41f9e164681290fd8b59b32156107e6a53c663577c027cb90ad3a2372588cf32cf54fc48ecc4e054871f9477ced52ad5590a5c5c5011e524b964f9ffd3514342
+  metadata.gz: ce95160d952b92f393f9156b044412db5a075af34ae2a429a55d4c1af88e24f66645f4cb46f37a1b70a8c3328cec57a66d50939f5dea8bb00e46794e2ecd4a5b
+  data.tar.gz: 21cdae9c65a918a04d66927cb0fad1857cd5a007c45b2c5d52decd4563d62c1bbd030e410f8bd04314d31c847e8df46693cdb07f43bd44bfd8eca91d7aebb54b

data/google_scraper_gem.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
   spec.add_runtime_dependency "mechanize"
   spec.add_runtime_dependency "nokogiri"
 end

data/lib/google_scraper_gem.rb

@@ -14,6 +14,7 @@ class GoogleScraper
       # User Agent list - http://github.com/tenderlove/mechanize/blob/master/lib/mechanize.rb
       agent.user_agent_alias = 'Mac Safari' 
     }
+    # @mech.keep_alive = false
 
     # @mech.set_proxy proxy[:host], proxy[:port], proxy[:username], proxy[:password]
   end
@@ -34,7 +35,7 @@ class GoogleScraper
         rank_count += 1
 
         result = cite.attr('href')
-        puts result
+        # puts result
 
         result.gsub!(%r{^http://}, '')
         result.gsub!(%r{^https://}, '')
@@ -46,8 +47,7 @@ class GoogleScraper
       page_num += 1
 
       # Add random sleep to prevent blocking of IP.
-      # TODO: Replace this with proxy swap.
-      rand(8)
+      # sleep(rand(3..9))
 
       # TODO: Click "next" instead?
       page = page.link_with(:text => page_num.to_s).click
@@ -63,7 +63,7 @@ class GoogleScraper
     rank_count = 0
     page_num = 1
 
-    uri = BASE_URL + extension + SEARCH + URI.encode(keyword)
+    uri = 'http://www.google' + extension + SEARCH + URI.encode(keyword)
 
     page = @mech.get(uri)
     while rank_count < top
@@ -88,7 +88,7 @@ class GoogleScraper
 
       # Add random sleep to prevent blocking of IP.
       # TODO: Replace this with proxy swap.
-      rand(8)
+      # rand(8)
 
       page = page.link_with(:text => page_num.to_s).click
     end

data/lib/google_scraper_gem/version.rb

@@ -1,3 +1,3 @@
 module GoogleScraperGem
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/google_scraper_gem_spec.rb

@@ -1,18 +1,46 @@
-require 'minitest/spec'
-require 'minitest/autorun'
+#require 'minitest/spec'
+#require 'minitest/autorun'
 require_relative '../lib/google_scraper_gem'
 
-@@gs = GoogleScraper.new
-@@results = @@gs.getTopResults('medical assistant')
-
 describe GoogleScraper do
-  it "retrieves the top 10 results by default" do
-    @@results.count.must_equal 10
+  describe "checkRank" do
+    before(:all) do
+      @gs = GoogleScraper.new
+    end
+
+    it "returns the rank of the en/us query" do
+      en_us_rank = @gs.checkRank('medical assistant', 'www.aama-ntl.org')
+      puts "en/us rank: #{en_us_rank}"
+      en_us_rank.should > 0
+    end
+
+    it "returns the rank of the en/ca query" do
+      en_ca_rank = @gs.checkRank('medical assistant', 'www.aama-ntl.org', 'ca', 'en')
+      puts "en/ca rank: #{en_ca_rank}"
+      en_ca_rank.should > 0
+    end
+
+    it "returns the rank of the es/mx query" do
+      es_mx_rank = @gs.checkRank('medical assistant', 'www.medicalassistantsalud.com', 'mx', 'es')
+      puts "es/mx rank: #{es_mx_rank}"
+      es_mx_rank.should > 0
+    end
   end
 
-  it "should not return results that are invalid URLs" do
-    @@results.each do |result|
-      result.wont_include "›"
+  describe "getTopResults" do
+    before(:all) do
+      @gs = GoogleScraper.new
+      @results = @gs.getTopResults('medical assistant')
+    end
+
+    it "retrieves the top 10 results by default" do
+      @results.count.should == 10
+    end
+
+    it "should not return results that are invalid URLs" do
+      @results.each do |result|
+        result.include?("›").should be_false
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google_scraper_gem
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Han Chang
@@ -38,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -102,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.0.rc.2
 signing_key: 
 specification_version: 4
 summary: Scrapes Google Search results