google_play_scraper 0.0.1

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+desc "Run all specs in spec directory (excluding plugin specs)"
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+
+Bundler::GemHelper.install_tasks

data/lib/google_play_scraper.rb

@@ -0,0 +1,9 @@
+require 'google_play_scraper/url'
+require 'google_play_scraper/search_options'
+require 'google_play_scraper/app'
+require 'google_play_scraper/search'
+require 'google_play_scraper/parser'
+require 'nokogiri'
+
+module GooglePlayScraper
+end

data/lib/google_play_scraper/app.rb

@@ -0,0 +1,3 @@
+class GooglePlayScraper::App
+  attr_accessor :id, :url, :name, :developer, :logo_url, :logo_url_small
+end

data/lib/google_play_scraper/parser.rb

@@ -0,0 +1,67 @@
+require 'nokogiri'
+
+class GooglePlayScraper::Parser
+  
+  LOGO_CSS_SELECTOR = '.card-content .cover .cover-image-container .cover-outer-align .cover-inner-align img'
+  
+  attr_reader :raw_html
+  
+  def initialize(raw_html)
+    @raw_html = raw_html
+  end
+  
+  def results
+    doc = Nokogiri::HTML(raw_html)
+    
+    results = []
+    
+    doc.css('.card').each do |app_container|
+      results << create_app(app_container)
+    end
+    
+    results
+  end
+  
+  def create_app(app_container)
+    app = GooglePlayScraper::App.new
+    app.logo_url = extract_logo_url(app_container)
+    app.logo_url_small = extract_logo_url_small(app_container)
+    app.url = extract_app_url(app_container)
+    app.id = extract_app_id(app_container)
+    app.name = extract_app_name(app_container)
+    app.developer = extract_developer(app_container)
+    
+    app
+  end
+  
+  def extract_developer(app_container)
+    span_tag = app_container.css('.card-content .details .subtitle-container .subtitle').first
+    span_tag.content
+  end
+  
+  def extract_app_name(app_container)
+    a_tag = app_container.css('.card-content .details a.title').first
+    a_tag['title']
+  end
+  
+  def extract_app_id(app_container)
+    a_tag = app_container.css('.card-content a.card-click-target').first
+    uri = Addressable::URI.parse(a_tag['href'])
+    uri.query_values['id']
+  end
+  
+  def extract_app_url(app_container)
+    a_tag = app_container.css('.card-content a.card-click-target').first
+    GooglePlayScraper::GOOGLE_PLAY_BASE_URL + a_tag['href']
+  end
+  
+  def extract_logo_url(app_container)
+    img_tag = app_container.css(LOGO_CSS_SELECTOR).first
+    img_tag['data-cover-large']
+  end
+  
+  def extract_logo_url_small(app_container)
+    img_tag = app_container.css(LOGO_CSS_SELECTOR).first
+    img_tag['data-cover-small']
+  end
+end

data/lib/google_play_scraper/search.rb

@@ -0,0 +1,43 @@
+require 'addressable/uri'
+require 'net/http'
+
+class GooglePlayScraper::Search
+  
+  include GooglePlayScraper::SearchOptions
+  
+  attr_reader :query, :options
+  
+  GOOGLE_PLAY_SEARCH_URL = GooglePlayScraper::GOOGLE_PLAY_BASE_URL + "/store/search"
+  
+  DEFAULT_OPTIONS = {
+    CATEGORY => 'apps',
+    LANGUAGE => 'en',
+    APPLY_SORTING => 1, # 0 means no sorting
+    SAFE_SEARCH => 0, # apply safesearch to results
+    NUMBER_OF_RESULTS => 10 # number of results to display
+  }
+  
+  def initialize(query, options = {})
+    @options = DEFAULT_OPTIONS.merge(options)
+    @options['q'] = query
+  end
+  
+  def run
+    uri = build_uri
+    http = Net::HTTP.new(uri.host, 443)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    
+    request = Net::HTTP::Get.new(uri.request_uri)
+    response = http.request(request)
+    
+    parser = GooglePlayScraper::Parser.new(response.body)
+    parser.results
+  end
+  
+  def build_uri
+    uri = Addressable::URI.parse(GOOGLE_PLAY_SEARCH_URL)
+    uri.query_values = options
+    uri
+  end
+end

data/lib/google_play_scraper/search_options.rb

@@ -0,0 +1,9 @@
+module GooglePlayScraper
+  module SearchOptions
+    CATEGORY = :c
+    LANGUAGE = :hl
+    APPLY_SORTING = :sort
+    SAFE_SEARCH = :safe
+    NUMBER_OF_RESULTS = :num
+  end
+end

data/lib/google_play_scraper/url.rb

@@ -0,0 +1,3 @@
+module GooglePlayScraper
+  GOOGLE_PLAY_BASE_URL = 'https://play.google.com'
+end

data/lib/google_play_scraper/version.rb

@@ -0,0 +1,3 @@
+module GooglePlayScraper
+  VERSION = "0.0.1"
+end

data/spec/google_play_scraper/search_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe GooglePlayScraper::Search do
+  context 'searching' do
+    it 'should be able to search for angry birds' do
+      search = GooglePlayScraper::Search.new('Angry Birds')
+      results = search.run
+      results.size.should == 10
+      results.first.name.should == 'Angry Birds'
+      results.first.developer.should == 'Rovio Mobile Ltd.'
+      results.first.id.should == 'com.rovio.angrybirds'
+      results.first.logo_url.should == 'https://lh6.ggpht.com/M9q_Zs_CRt2rbA41nTMhrPqiBxhUEUN8Z1f_mn9m89_TiHbIbUF8hjnc_zwevvLsRIJy=w340'
+      results.first.logo_url_small.should == 'https://lh6.ggpht.com/M9q_Zs_CRt2rbA41nTMhrPqiBxhUEUN8Z1f_mn9m89_TiHbIbUF8hjnc_zwevvLsRIJy=w170'
+      results.first.url.should == 'https://play.google.com/store/apps/details?id=com.rovio.angrybirds'
+    end
+    
+    it 'should be able to search for the sparq.me scanner' do
+      search = GooglePlayScraper::Search.new('SPARQ.ME')
+      results = search.run
+      results.size.should == 1
+      results.first.name.should == 'SPARQ.ME'
+      results.first.developer.should == 'MSKYNET, Inc.'
+      results.first.id.should == 'com.sparqcode.sparqeye.android'
+      results.first.logo_url.should == 'https://lh3.ggpht.com/m8vHSrnD8kqMpYkgyQDxWpren2Pi5Vn-Eemj-xyHwcGUsbsi0rcYy0dR5Qzi5B566ig=w340'
+      results.first.logo_url_small.should == 'https://lh3.ggpht.com/m8vHSrnD8kqMpYkgyQDxWpren2Pi5Vn-Eemj-xyHwcGUsbsi0rcYy0dR5Qzi5B566ig=w170'
+      results.first.url.should == 'https://play.google.com/store/apps/details?id=com.sparqcode.sparqeye.android'
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'google_play_scraper'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: google_play_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mike Emery
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+description: Uses Nokogiri to scrape app search results on the Google Play app search
+  results page.  Returns results as Ruby objects
+email:
+- mike@sparq.it
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/google_play_scraper/app.rb
+- lib/google_play_scraper/parser.rb
+- lib/google_play_scraper/search.rb
+- lib/google_play_scraper/search_options.rb
+- lib/google_play_scraper/url.rb
+- lib/google_play_scraper/version.rb
+- lib/google_play_scraper.rb
+- Rakefile
+- spec/google_play_scraper/search_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: google_play_scraper
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Get Google Play App search results from ruby by scraping their website.
+test_files:
+- spec/google_play_scraper/search_spec.rb
+- spec/spec_helper.rb