serp_scraper 1.0.0 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30a96845ede19f7ffccdb07ed3931febaeb9569c
-  data.tar.gz: 42efe3459963010693412968e4da65c153a1300f
+  metadata.gz: 37e155ad5c882faa1b543fc3f875b417ed2f903a
+  data.tar.gz: 18089df07af8acfc609da7552d2282eae2ff05d3
 SHA512:
-  metadata.gz: ce9a8e3aabc38778afdf7c3c7af177f89eb3d50ff4426c5b82ca5d4d9bc2816aa35cb484d7485930a407f2379a9389f63a0a6f3ec9b2d3906c667acff108aa1b
-  data.tar.gz: 6c6c06b7d6d304f4a27f6654ff1ab23cbdae6817a170acc3f996e3f998eb9063207bb2099a943f6d1b60840195a1f4c9f5bbb80a45da079bc82c6037bf84b163
+  metadata.gz: 6736db5b2e411387cf6401b888b0e20b723483801e05fd80aab0a7823c65740f9a047b019b51f45c92b45dfa91c3004a94a215e750f039123b6246028e884934
+  data.tar.gz: 0123239f14dc80688a682a362e27f3739731a71b53a8ff836a56f655df04766040e1adfcb7df2f95c5d121d2e1e6b45856954a50287bf4dcdc6cb48a407932cb

data/lib/engines/google.rb

@@ -89,8 +89,10 @@ class SerpScraper::Google
     doc     = Nokogiri::HTML(html)
     results = Array.new
 
-    rows = doc.css('h3.r > a')
-    rows.each_with_index do |row, i|
+    rows = doc.css('div.rc h3.r > a')
+
+    position = 1
+    rows.each do |row|
       begin
         href = Addressable::URI.parse(row["href"])
 
@@ -99,14 +101,19 @@ class SerpScraper::Google
 
         url = Addressable::URI.parse(external_url)
 
+        next unless url.host # Only add valid URL's (ignore images, news etc)
+
         results.push({
-          position: i + 1,
+          position: position,
           title: row.content,
           scheme: url.scheme,
           domain: url.host,
           url: url.request_uri,
           full_url: url.to_s
         })
+
+        position += 1
+
       rescue
         next
       end

data/serp_scraper.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'serp_scraper'
-  s.version     = '1.0.0'
+  s.version     = '1.0.4'
   s.date        = '2017-05-26'
 
   s.homepage    = 'https://github.com/kjellberg'
@@ -15,8 +15,8 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.files       = `git ls-files`.split($/)
 
-  s.add_runtime_dependency 'mechanize', '~> 2.7', '>= 2.7.5'
+  s.add_runtime_dependency 'mechanize', '~> 2.7.0'
   s.add_runtime_dependency 'addressable', '~> 2.5'
-  s.add_runtime_dependency 'nokogiri', '~> 2.9', '>= 2.9.4'
+  s.add_runtime_dependency 'nokogiri', '~> 1.6'
   s.add_runtime_dependency 'deathbycaptcha', '~> 5.0.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: serp_scraper
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.4
 platform: ruby
 authors:
 - Rasmus Kjellberg
@@ -16,20 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.7.5
+        version: 2.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.7.5
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -50,20 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.9.4
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.9.4
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: deathbycaptcha
   requirement: !ruby/object:Gem::Requirement