serp_scraper 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50907d75fe90b6a6eba27dcccf7da802e3d2b999
-  data.tar.gz: 69210b67fa80e1df6774600ffc798e93306a3a70
+  metadata.gz: 57fa4ca58ef4a346fd8839408b73ac3adbf2773c
+  data.tar.gz: 17798f9cb46e7c16011770b1c6a6c25996777fa2
 SHA512:
-  metadata.gz: 030bba279587f40d63c258824c32ef6c0bed017e07a9330a008e6df292c556500b293bc01549a181aa4dd22e8a0ab38c4ec258030464848de117208735a788d9
-  data.tar.gz: cdffcc183ba38b153fce55257a508500dc0378582dc9547dd1c88e80f8dfed1a0a741712ad3e1a4b70271392e4c8cdabb5d0fe5f3f4984400db6c994b0eb49ac
+  metadata.gz: 6bf5846e86571595732c09e66aef92c71070ed59b338d10f1e0a6ddfbd5d46699667ee0ec33c8c29cc912b223b1db5dcb86f31d421b2ea701819f2e1c4fccde3
+  data.tar.gz: febb89dac3c82b1a73f3a8e94bafffe310b926e898d8b78440b29c78c36cbf67eb25684076260f860726e10676e39ade6ec59ce2e426f1167e24637493d1020d

data/EXAMPLES.md

@@ -15,11 +15,14 @@ end
 
 ## Country/TLD specific search
 ```ruby
-# Usees google.se for swedish results
+# Set '.se' as TLD for swedish results
 s = SerpScraper.new(engine: 'google', tld: 'se')
 
-# Set language to Swedish
+# Set language parameter to swedish
 s.engine.parameter('hl', 'sv')
 
-response = s.search('köp bilar online')
-```
+s.search('köp bilar online').results.each do |result|
+  puts result
+  # => {:position=>1, :title=>"kvd.se - Bilauktioner på nätet", :scheme=>"https", :domain=>"www.kvd.se", :url=>"/", :full_url=>"https://www.kvd.se/"}
+end
+```

data/README.md

@@ -15,10 +15,11 @@ gem 'serp_scraper'
 ```
 
 ## Examples
-```ruby
-s = SerpScraper.new(engine: 'google')
-res = s.search('buy cars onlines')
-puts res.results[0]
+
+```ruby 
+google = SerpScraper.new(engine: 'google', tld: 'com')
+first_result = google.search('buy cars onlines').results[0]
+puts first_result
 # => {:position=>1, :title=>"Buying From CarMax", :scheme=>"https", :domain=>"www.carmax.com", :url=>"/car-buying-process", :full_url=>"https://www.carmax.com/car-buying-process"}
 ```
 

data/lib/engines/google.rb

@@ -1,42 +1,75 @@
 class SerpScraper::Google
   attr_accessor :tld
   attr_accessor :user_agent
+  attr_accessor :browser
 
   def initialize(tld)
-    self.tld = tld
+    # Make tld global
+    @tld = tld
 
+    # Create new Mechanize object
     @browser = Mechanize.new { |agent|
       agent.user_agent_alias = 'Mac Safari'
     }
-    
-    @parameters = Hash.new 
-    @parameters['gbv'] = 1
-    @parameters['complete'] = 0
-    @parameters['num'] = 100
-    @parameters['pws'] = 0
-    @parameters['nfrpr'] = 1
-    @parameters['ie'] = 'utf-8'
-    @parameters['oe'] = 'utf-8'
-    @parameters['site'] = 'webhp'
-    @parameters['source'] = 'hp'
+
+    # Set standard query parameters
+    @parameters = {
+      gbv: 1,
+      complete: 0,
+      num: 100,
+      pws: 0,
+      nfrpr: 1,
+      ie: 'utf-8',
+      oe: 'utf-8',
+      site: 'webhp',
+      source: 'hp'
+    }
   end
 
   def search(keyword)
+    # Add keyword to parameters
+    @parameters['q'] = keyword
+
+    # Create build google search url
+    search_url = build_query_url_from_keyword(keyword)
+
     # Do the Googleing
-    http_response = @browser.get(build_query_url_from_keyword(keyword))
+    response = @browser.get(search_url, :referer => "https://www.google.#{@tld}")
+
+    # 503 error = Google Captcha
+    tries = 1 
+    while response.code[/503/] and tries <= 3
+      # Try to solve with captcha 
+      solve_captcha(response.uri.to_s)
+
+      # Do another search
+      response = @browser.get(search_url)
+
+      tries += 1
+    end
     
-    return build_serp_response(http_response) if http_response.code == "200"
+    return build_serp_response(response) if response.code == "200"
 
     # @todo: Look for and solve captchas.
     puts "Did not get a 200 response. Maybe a captcha error?"
   end
 
-  def build_serp_response(http_response)
+  def solve_captcha(captcha_url)
+    puts "trying to solve captcha on url #{captcha_url}"
+    
+    page = @browser.get(captcha_url)
+    doc = Nokogiri::HTML(page.content)
+
+    image_url = Addressable::URI.parse('http://ipv4.google.com/' + doc.css('img')[0]["src"]).normalize
+    puts "Captcha url: " + image_url
+  end
+
+  def build_serp_response(response)
     sr            = SerpScraper::SerpResponse.new
     sr.keyword    = @parameters['q']
     sr.user_agent = @browser.user_agent
-    sr.url        = http_response.uri.to_s
-    sr.html       = http_response.content
+    sr.url        = response.uri.to_s
+    sr.html       = response.content
     sr.results    = extract_results(sr.html)
 
     sr # Return sr
@@ -77,10 +110,8 @@ class SerpScraper::Google
   end
 
   def build_query_url_from_keyword(keyword)
-    @parameters['q'] = keyword
-
     uri = Addressable::URI.new
-    uri.host = "www.google.#{tld}"
+    uri.host = "www.google.#{@tld}"
     uri.scheme = "https"
     uri.path = "/search"
     uri.query_values = @parameters

data/lib/serp_scraper.rb

@@ -1,3 +1,9 @@
+require 'uri'
+require 'mechanize'
+require 'addressable/uri'
+require 'nokogiri'
+require 'deathbycaptcha'
+
 class SerpScraper
   attr_accessor :engine
 
@@ -11,27 +17,14 @@ class SerpScraper
     end
   end
 
+  def set_proxy(address, port, user = nil, password = nil)
+    @engine.browser.set_proxy(address, port, user, password)
+  end
+
   def search(keyword)
     @engine.search(keyword)
   end
 end
 
-def test
-  google = SerpScraper.new(engine: 'google', tld: 'se')
-
-  # Set language to Swedish
-  google.engine.parameter('hl', 'sv')
-
-  # GO, FETCH!
-  response = google.search("casino faktura")
-
-  # Return search results
-  response.results
-end
-
-require 'uri'
-require 'mechanize'
-require 'addressable/uri'
-require 'nokogiri'
 require 'engines/google'
 require 'serp_response'

data/serp_scraper.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'serp_scraper'
-  s.version     = '0.0.2'
+  s.version     = '0.0.3'
   s.date        = '2017-05-26'
 
   s.homepage    = 'https://github.com/kjellberg'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: serp_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Rasmus Kjellberg