serp_scraper 0.0.0 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f8b2245714d035db0eeb3853404259057c653cd
-  data.tar.gz: c3ed1301196c9989edda91bfcdb51b7c6304726c
+  metadata.gz: 50907d75fe90b6a6eba27dcccf7da802e3d2b999
+  data.tar.gz: 69210b67fa80e1df6774600ffc798e93306a3a70
 SHA512:
-  metadata.gz: 7448374ba90679e644e64f1eaec165a2eb5ee0e95f31bfe5654f58bb8218b065735d442a41bce6907340aeead2133e8ab6c0e4d36c9bb918f44d768f86919074
-  data.tar.gz: 0a92203202b7cc0ef273973f58147bc57a21a84bf600a28f7f18526d7be3c2e24d143ef62d1f8012921aaa6a667079d71c80834305e4828742ee803183d8adde
+  metadata.gz: 030bba279587f40d63c258824c32ef6c0bed017e07a9330a008e6df292c556500b293bc01549a181aa4dd22e8a0ab38c4ec258030464848de117208735a788d9
+  data.tar.gz: cdffcc183ba38b153fce55257a508500dc0378582dc9547dd1c88e80f8dfed1a0a741712ad3e1a4b70271392e4c8cdabb5d0fe5f3f4984400db6c994b0eb49ac

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/EXAMPLES.md

@@ -0,0 +1,25 @@
+# SERP Scraper examples
+
+## Basic search
+```ruby
+require 'serp_scraper'
+
+s = SerpScraper.new(engine: 'google')
+response = s.search('buy cars onlines')
+
+response.results.each do |result|
+  puts result
+  # => {:position=>1, :title=>"Buying From CarMax", :scheme=>"https", :domain=>"www.carmax.com", :url=>"/car-buying-process", :full_url=>"https://www.carmax.com/car-buying-process"}
+end
+```
+
+## Country/TLD specific search
+```ruby
+# Usees google.se for swedish results
+s = SerpScraper.new(engine: 'google', tld: 'se')
+
+# Set language to Swedish
+s.engine.parameter('hl', 'sv')
+
+response = s.search('köp bilar online')
+```

data/README.md

@@ -0,0 +1,48 @@
+# SERP Scraper
+SERP Scraper is a ruby library that extracts keyword rankings from Google.
+
+##### Supported search engines
+* Google
+
+## Installation
+Install 'SERP Scraper' from RubyGems:
+```sh
+$ gem install serp_scraper
+```
+Or include it in your project's Gemfile with Bundler:
+```ruby
+gem 'serp_scraper'
+```
+
+## Examples
+```ruby
+s = SerpScraper.new(engine: 'google')
+res = s.search('buy cars onlines')
+puts res.results[0]
+# => {:position=>1, :title=>"Buying From CarMax", :scheme=>"https", :domain=>"www.carmax.com", :url=>"/car-buying-process", :full_url=>"https://www.carmax.com/car-buying-process"}
+```
+
+If you are just starting, check out the [EXAMPLES](https://github.com/kjellberg/serp_scraper/blob/master/EXAMPLES.md) file for more examples.
+
+## Support
+- [github.com/kjellberg/serp_scraper/issues](https://github.com/kjellberg/serp_scraper/issues)
+
+## Contribute
+- [github.com/kjellberg/serp_scraper/issues](https://github.com/kjellberg/serp_scraper/issues)
+
+### Goals
+- Add more search engines like Bing & Yahoo
+- Add DeathByCaptcha support for captcha solving
+
+## Dependencies
+- [mechanize](https://github.com/sparklemotion/mechanize)
+- [nokogiri](https://github.com/sparklemotion/nokogiri)
+- [addressable/uri](https://github.com/sporkmonger/addressable)
+
+## Credits
+- [github.com/kjellberg](https://github.com/kjellberg) 
+
+*Make a [pull request](https://github.com/kjellberg/serp_scraper/#contribute) and add your name here :)*
+
+## License
+This library is distributed under the MIT license.

data/lib/engines/google.rb

@@ -0,0 +1,89 @@
+class SerpScraper::Google
+  attr_accessor :tld
+  attr_accessor :user_agent
+
+  def initialize(tld)
+    self.tld = tld
+
+    @browser = Mechanize.new { |agent|
+      agent.user_agent_alias = 'Mac Safari'
+    }
+    
+    @parameters = Hash.new 
+    @parameters['gbv'] = 1
+    @parameters['complete'] = 0
+    @parameters['num'] = 100
+    @parameters['pws'] = 0
+    @parameters['nfrpr'] = 1
+    @parameters['ie'] = 'utf-8'
+    @parameters['oe'] = 'utf-8'
+    @parameters['site'] = 'webhp'
+    @parameters['source'] = 'hp'
+  end
+
+  def search(keyword)
+    # Do the Googleing
+    http_response = @browser.get(build_query_url_from_keyword(keyword))
+    
+    return build_serp_response(http_response) if http_response.code == "200"
+
+    # @todo: Look for and solve captchas.
+    puts "Did not get a 200 response. Maybe a captcha error?"
+  end
+
+  def build_serp_response(http_response)
+    sr            = SerpScraper::SerpResponse.new
+    sr.keyword    = @parameters['q']
+    sr.user_agent = @browser.user_agent
+    sr.url        = http_response.uri.to_s
+    sr.html       = http_response.content
+    sr.results    = extract_results(sr.html)
+
+    sr # Return sr
+  end
+
+  def extract_results(html)
+    doc     = Nokogiri::HTML(html)
+    results = Array.new
+
+    rows = doc.css('h3.r > a')
+    rows.each_with_index do |row, i|
+      begin
+        href = Addressable::URI.parse(row["href"])
+
+        external_url = href.query_values['q']    unless href.query_values['q'] == nil
+        external_url = href.query_values['url']  unless href.query_values['url'] == nil
+
+        url = Addressable::URI.parse(external_url)
+
+        results.push({
+          position: i + 1,
+          title: row.content,
+          scheme: url.scheme,
+          domain: url.host,
+          url: url.request_uri,
+          full_url: url.to_s
+        })
+      rescue
+        next
+      end
+    end
+
+    results
+  end
+
+  def parameter(key, value)
+    @parameters[key] = value
+  end
+
+  def build_query_url_from_keyword(keyword)
+    @parameters['q'] = keyword
+
+    uri = Addressable::URI.new
+    uri.host = "www.google.#{tld}"
+    uri.scheme = "https"
+    uri.path = "/search"
+    uri.query_values = @parameters
+    uri.to_s
+  end
+end

data/lib/serp_response.rb

@@ -0,0 +1,8 @@
+class SerpScraper::SerpResponse
+  attr_accessor :keyword
+  attr_accessor :url
+  attr_accessor :user_agent
+  attr_accessor :proxy
+  attr_accessor :results
+  attr_accessor :html
+end

data/lib/serp_scraper.rb

@@ -1,5 +1,37 @@
 class SerpScraper
-  def self.hi
-    puts "It Works!"
+  attr_accessor :engine
+
+  def initialize(params)
+    engine = params[:engine] || 'google'
+    tld = params[:tld] || 'com'
+
+    case engine
+    when "google"
+      @engine = Google.new(tld)
+    end
   end
-end
+
+  def search(keyword)
+    @engine.search(keyword)
+  end
+end
+
+def test
+  google = SerpScraper.new(engine: 'google', tld: 'se')
+
+  # Set language to Swedish
+  google.engine.parameter('hl', 'sv')
+
+  # GO, FETCH!
+  response = google.search("casino faktura")
+
+  # Return search results
+  response.results
+end
+
+require 'uri'
+require 'mechanize'
+require 'addressable/uri'
+require 'nokogiri'
+require 'engines/google'
+require 'serp_response'

data/serp_scraper.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |s|
+  s.name        = 'serp_scraper'
+  s.version     = '0.0.2'
+  s.date        = '2017-05-26'
+
+  s.homepage    = 'https://github.com/kjellberg'
+  s.summary     = %q{Get rankings from Search Engines}
+  s.description = "SERP Scraper is a ruby library that extracts keyword rankings from Google."
+  
+
+  s.authors     = ["Rasmus Kjellberg"]
+  s.email       = 'rk@youngmedia.se'
+  s.license     = 'MIT'
+
+  s.require_paths = ["lib"]
+  s.files       = `git ls-files`.split($/)
+
+  s.add_runtime_dependency 'mechanize', '~> 2.7', '>= 2.7.5'
+  s.add_runtime_dependency 'addressable', '~> 2.5'
+  s.add_runtime_dependency 'nokogiri', '~> 2.9', '>= 2.9.4'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: serp_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.2
 platform: ruby
 authors:
 - Rasmus Kjellberg
@@ -9,14 +9,74 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2017-05-26 00:00:00.000000000 Z
-dependencies: []
-description: Scrape search engine keyword positions.
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.5
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.9.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.9.4
+description: SERP Scraper is a ruby library that extracts keyword rankings from Google.
 email: rk@youngmedia.se
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- EXAMPLES.md
+- README.md
+- lib/engines/google.rb
+- lib/serp_response.rb
 - lib/serp_scraper.rb
+- serp_scraper.gemspec
 homepage: https://github.com/kjellberg
 licenses:
 - MIT
@@ -40,5 +100,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Scrape search engine keyword positions.
+summary: Get rankings from Search Engines
 test_files: []