market_bot 0.9.1 → 0.10.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9688ff41be35247c4bb0f85621bb800232111936
+  data.tar.gz: 368e41730331ab5f6084ce49d6a51cbe87904d51
+SHA512:
+  metadata.gz: 34de9be2e58bfeab53156bd5b3248fcd412918616431ab03b3521b5065f55d0b07deb913f7f08a07cf512b5151dc1c8859a5e52f70c1e1c84cd86a9129d57fd3
+  data.tar.gz: 96e62fed37bee0a7ee35725a45470a3f912e7651e3868ae102f6135a7d6666e39abab95a76465e5849dafd86384745984bdebfb87949d792fbd50d11a393ef6a

data/Gemfile

@@ -1,12 +1,12 @@
 source 'http://rubygems.org'
 
-gem 'typhoeus'
+gem 'typhoeus', '~> 0.4.2'
 gem 'nokogiri'
 
 group :development do
   gem 'rspec', '~> 2.8.0'
-  gem 'bundler', '~> 1.1.0'
-  gem 'jeweler', '~> 1.6.4'
+  gem 'bundler', '~> 1.3.0'
+  gem 'jeweler', '~> 1.8.0'
   gem 'simplecov', '>= 0'
   gem 'rdoc', '>= 3.9.4'
   gem 'guard', '~> 1.2.3'

data/Gemfile.lock

@@ -1,42 +1,72 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    addressable (2.3.5)
+    builder (3.2.2)
     columnize (0.3.6)
-    debugger (1.1.4)
+    debugger (1.6.1)
       columnize (>= 0.3.1)
-      debugger-linecache (~> 1.1.1)
-      debugger-ruby_core_source (~> 1.1.3)
-    debugger-linecache (1.1.1)
-      debugger-ruby_core_source (>= 1.1.1)
-    debugger-ruby_core_source (1.1.3)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.3)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.3)
     diff-lcs (1.1.3)
-    ffi (1.0.11)
+    faraday (0.8.7)
+      multipart-post (~> 1.1)
+    ffi (1.9.0)
     git (1.2.5)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
     growl (1.0.3)
     guard (1.2.3)
       listen (>= 0.4.2)
       thor (>= 0.14.6)
     guard-rspec (1.1.0)
       guard (>= 1.1)
-    jeweler (1.6.4)
+    hashie (2.0.5)
+    highline (1.6.19)
+    httpauth (0.2.0)
+    jeweler (1.8.6)
+      builder
       bundler (~> 1.0)
       git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
       rake
-    json (1.7.3)
-    listen (0.4.7)
-      rb-fchange (~> 0.0.5)
-      rb-fsevent (~> 0.9.1)
-      rb-inotify (~> 0.8.8)
-    mime-types (1.18)
-    multi_json (1.3.6)
-    nokogiri (1.5.4)
-    rake (0.9.2.2)
-    rb-fchange (0.0.5)
-      ffi
-    rb-fsevent (0.9.1)
-    rb-inotify (0.8.8)
+      rdoc
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    listen (1.2.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+      rb-kqueue (>= 0.2)
+    mime-types (1.23)
+    multi_json (1.7.7)
+    multi_xml (0.5.4)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.1.0)
+    rb-fsevent (0.9.3)
+    rb-inotify (0.9.0)
+      ffi (>= 0.5.0)
+    rb-kqueue (0.2.0)
       ffi (>= 0.5.0)
-    rdoc (3.12)
+    rdoc (4.0.1)
       json (~> 1.4)
     rspec (2.8.0)
       rspec-core (~> 2.8.0)
@@ -46,11 +76,11 @@ GEM
     rspec-expectations (2.8.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.8.0)
-    simplecov (0.6.4)
+    simplecov (0.7.1)
       multi_json (~> 1.0)
-      simplecov-html (~> 0.5.3)
-    simplecov-html (0.5.3)
-    thor (0.15.4)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    thor (0.18.1)
     typhoeus (0.4.2)
       ffi (~> 1.0)
       mime-types (~> 1.18)
@@ -59,14 +89,14 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.1.0)
+  bundler (~> 1.3.0)
   debugger
   growl
   guard (~> 1.2.3)
   guard-rspec (~> 1.1.0)
-  jeweler (~> 1.6.4)
+  jeweler (~> 1.8.0)
   nokogiri
   rdoc (>= 3.9.4)
   rspec (~> 2.8.0)
   simplecov
-  typhoeus
+  typhoeus (~> 0.4.2)

data/README.markdown

@@ -6,6 +6,10 @@ Books, music, movies, etc aren't currently supported.
 It is built on top of Nokogiri and Typhoeus.
 Used in production to power [www.droidmeter.com](http://www.droidmeter.com/?t=github).
 
+**This project is currently seeking developers to help maintain it.
+Please send pull requests or contact me if you are able to help out.
+The app scraper is known to work, but leader board functionality is currently broken.**
+
 ## Dependencies
 
 * Nokogiri
@@ -68,6 +72,11 @@ Used in production to power [www.droidmeter.com](http://www.droidmeter.com/?t=gi
     puts "First place app (#{first_app.title}) price: #{first_app.price}" unless first_app.error
     puts "Last place app (#{last_app.title}) price: #{last_app.price}" unless last_app.error
 
+## Excessive Use
+
+Google will block your IP address if you attempt to scrape large quantities of data.
+Please contact me if you are looking for commercial data solutions.
+
 ## Contributing to Market Bot
 
 1. Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/VERSION

@@ -1 +1 @@
-0.9.1
+0.10.0

data/lib/market_bot/android/app.rb

@@ -19,132 +19,109 @@ module MarketBot
         result = {}
 
         doc = Nokogiri::HTML(html)
-
-        elements = doc.css('.doc-metadata').first.elements[2].elements
-        elem_count = elements.count
-
-        (3..(elem_count - 1)).select{ |n| n.odd? }.each do |i|
-          field_name  = elements[i].text
+        meta_info = doc.css('.meta-info')
+        meta_info.each do |info|
+          field_name = info.css('.title').text.strip
 
           case field_name
-          when 'Updated:'
-            result[:updated] = elements[i + 1].text
-          when 'Current Version:'
-            result[:current_version] = elements[i + 1].text
-          when 'Requires Android:'
-            result[:requires_android] = elements[i + 1].text
-          when 'Category:'
-            result[:category] = elements[i + 1].text
-          when 'Installs:'
-            result[:installs] = elements[i + 1].children.first.text
-          when 'Size:'
-            result[:size] = elements[i + 1].text
-          when 'Price:'
-            result[:price] = elements[i + 1].text
-          when 'Content Rating:'
-            result[:content_rating] = elements[i + 1].text
+            when 'Updated'
+              result[:updated] = info.css('.content').text.strip
+            when 'Installs'
+              result[:installs] = info.css('.content').text.strip
+            when 'Size'
+              result[:size] = info.css('.content').text.strip
+            when 'Current Version'
+              result[:current_version] = info.css('.content').text.strip
+            when 'Requires Android'
+              result[:requires_android] = info.css('.content').text.strip
+            when 'Content Rating'
+              result[:content_rating] = info.css('.content').text.strip
+            when 'Contact Developer'
+              info.css('.dev-link').each do |node|
+                if node.text.strip.eql? 'Email Developer'
+                  result[:email] = node[:href].gsub(/^mailto:/,'')
+                else
+                  redirect_url = node[:href]
+                  if q_param = URI(redirect_url).query.split('&').select{ |p| p =~ /q=/ }.first
+                    actual_url = q_param.gsub('q=', '')
+                  end
+
+                  result[:website_url] = actual_url
+                end
+              end
+
           end
         end
 
-        result[:description] = doc.css('#doc-original-text').first.inner_html
-        result[:title] = doc.css('.doc-banner-title').text
+        node = doc.xpath("//meta[@itemprop='price']").first
+        result[:price] = node[:content].strip rescue 'Free'
 
-        rating_elem = doc.css('.average-rating-value')
-        result[:rating] = rating_elem.first.text unless rating_elem.empty?
+        result[:category] = doc.css('.category').first.text.strip rescue ''
+        result[:description] = doc.xpath("//div[@itemprop='description']").first.inner_html.strip
+        result[:title] = doc.xpath("//div[@itemprop='name']").first.text.strip
 
-        votes_elem = doc.css('.votes')
-        result[:votes] = doc.css('.votes').first.text unless votes_elem.empty?
+        score = doc.css('.score-container').first
+        unless score.nil?
+          node  = score.css('.score').first
+          result[:rating] = node.text.strip
+          node = score.xpath("//meta[@itemprop='ratingCount']").first
+          result[:votes] = node[:content].strip
+        end
 
-        result[:developer] = doc.css('.doc-banner-title-container a').text
+        node = doc.xpath("//div[@itemprop='author']")
+        result[:developer] = node.css('.primary').first.text.strip
 
         result[:more_from_developer] = []
         result[:users_also_installed] = []
         result[:related] = []
 
-        if similar_elem = doc.css('.doc-similar').first
-          similar_elem.children.each do |similar_elem_child|
-            assoc_app_type = similar_elem_child.attributes['data-analyticsid'].text
-
-            next unless %w(more-from-developer users-also-installed related).include?(assoc_app_type)
+        node = doc.css('.recommendation')
+        node.css('.rec-cluster').each do |recommended|
+          assoc_app_type = recommended.css('.heading').first.text.strip.eql?('Similar' ) ? :related : :more_from_developer
+          recommended.css('.card').each do |card|
+            assoc_app = {}
+            assoc_app[:app_id] = card['data-docid'].strip
 
-            assoc_app_type = assoc_app_type.gsub('-', '_').to_sym
-            result[assoc_app_type] ||= []
-
-            similar_elem_child.css('.app-left-column-related-snippet-container').each do |app_elem|
-              assoc_app = {}
-
-              assoc_app[:app_id] = app_elem.attributes['data-docid'].text
-
-              result[assoc_app_type] << assoc_app
-            end
+            result[assoc_app_type] << assoc_app
           end
         end
+        # Users also installed is no longer on the page, adding this for backwards compatibility, well, sort of....
+        result[:users_also_installed] = result[:related]
 
-        result[:banner_icon_url] = doc.css('.doc-banner-icon img').first.attributes['src'].value
-
-        if image_elem = doc.css('.doc-banner-image-container img').first
-          result[:banner_image_url] = image_elem.attributes['src'].value
-        else
-          result[:banner_image_url] = nil
+        node = doc.css('.cover-image').first
+        unless node.nil?
+          result[:banner_icon_url] = node[:src]
+          result[:banner_image_url] = node[:src]
         end
 
-        if website_elem = doc.css('a').select{ |l| l.text.include?("Visit Developer's Website")}.first
-          redirect_url = website_elem.attribute('href').value
-
-          if q_param = URI(redirect_url).query.split('&').select{ |p| p =~ /q=/ }.first
-            actual_url = q_param.gsub('q=', '')
+        result[:youtube_video_ids] = []
+        doc.css('.play-click-target').each do |node|
+          url = node['data-video-url']
+          unless url.nil?
+            result[:youtube_video_ids] << url.split('embed/').last.split('?').first
           end
-
-          result[:website_url] = actual_url
-        end
-
-        if email_elem = doc.css('a').select{ |l| l.text.include?("Email Developer")}.first
-          result[:email] = email_elem.attribute('href').value.gsub(/^mailto:/, '')
-        end
-
-        unless (video_section_elem = doc.css('.doc-video-section')).empty?
-          urls = video_section_elem.children.css('embed').map{ |e| e.attribute('src').value }
-          result[:youtube_video_ids] = urls.map{ |u| /youtube\.com\/v\/(.*)\?/.match(u)[1] }
-        else
-          result[:youtube_video_ids] = []
         end
 
-        screenshots = doc.css('.screenshot-carousel-content-container img')
-
-        if screenshots && screenshots.length > 0
-          result[:screenshot_urls] = screenshots.map { |s| s.attributes['src'].value }
-        else
-          result[:screenshot_urls] = []
+        result[:screenshot_urls] = []
+        doc.css('.screenshot').each do |node|
+          result[:screenshot_urls] << node[:src]
         end
 
-        result[:whats_new] = doc.css('.doc-whatsnew-container').inner_html
+        node = doc.css('.whatsnew').first
+        result[:whats_new] = node.inner_html.strip unless node.nil?
 
+        # Stubbing out for now, can't find them in the redesigned page.
         result[:permissions] = permissions = []
-        perm_types = ['dangerous', 'safe']
-        perm_types.each do |type|
-          doc.css("#doc-permissions-#{type} .doc-permission-group").each do |group_elem|
-            title = group_elem.css('.doc-permission-group-title').text
-            group_elem.css('.doc-permission-description').each do |desc_elem|
-              #permissions << { :security => type, :group => title, :description => desc_elem.text }
-            end
-            descriptions = group_elem.css('.doc-permission-description').map { |e| e.text }
-            descriptions_full = group_elem.css('.doc-permission-description-full').map { |e| e.text }
-            (0...descriptions.length).each do |i|
-              desc = descriptions[i]
-              desc_full = descriptions_full[i]
-              permissions << { :security => type, :group => title, :description => desc, :description_full => desc_full }
-            end
-          end
-        end
 
         result[:rating_distribution] = { 5 => nil, 4 => nil, 3 => nil, 2 => nil, 1 => nil }
 
-        if (histogram = doc.css('div.histogram-table').first)
-          cur_index = 5
-          histogram.css('tr').each do |e|
-            result[:rating_distribution][cur_index] = e.children.last.inner_text.gsub(/[^0-9]/, '').to_i
-            cur_index -= 1
-          end
+        histogram = doc.css('div.rating-histogram')
+        cur_index = 5
+        %w(five four three two one).each do |slot|
+          node = histogram.css(".#{slot.to_s}")
+          result[:rating_distribution][cur_index] = node.css('.bar-number').text.to_i
+          cur_index -= 1
+
         end
 
         result[:html] = html

data/lib/market_bot/android/leaderboard.rb

@@ -30,7 +30,11 @@ module MarketBot
           end
 
           result[:title] = details_node.css('.title').first.attributes['title'].to_s
-          result[:price_usd] = details_node.css('.buy-button-price').children.first.text.gsub(' Buy', '')
+
+          if (price_elem = details_node.css('.buy-button-price').children.first)
+            result[:price_usd] = price_elem.text.gsub(' Buy', '')
+          end
+
           result[:developer] = details_node.css('.attribution').children.first.text
           result[:market_id] = details_node.css('.title').first.attributes['href'].to_s.gsub('/store/apps/details?id=', '').gsub(/&feature=.*$/, '')
           result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
@@ -69,6 +73,7 @@ module MarketBot
         @hydra = options[:hydra] || MarketBot.hydra
         @request_opts = options[:request_opts] || {}
         @parsed_results = []
+        @pending_pages = []
       end
 
       def market_urls(options={})
@@ -92,7 +97,8 @@ module MarketBot
         results
       end
 
-      def enqueue_update(options={})
+      def enqueue_update(options={},&block)
+        @callback = block
         if @identifier.to_s.downcase == 'editors_choice' && category == nil
           url = 'https://play.google.com/store/apps/collection/editors_choice?&hl=en'
           process_page(url, 1)
@@ -130,6 +136,7 @@ module MarketBot
 
     private
       def process_page(url, page_num)
+        @pending_pages << page_num
         request = Typhoeus::Request.new(url, @request_opts)
         request.on_complete do |response|
           # HACK: Typhoeus <= 0.4.2 returns a response, 0.5.0pre returns the request.
@@ -143,6 +150,8 @@ module MarketBot
 
       def update_callback(result, page)
         @parsed_results[page] = result
+        @pending_pages.delete(page)
+        @callback.call(self) if @callback and @pending_pages.empty?
       end
     end
 

data/lib/market_bot/android/search_query.rb

@@ -4,7 +4,7 @@ module MarketBot
     # Search query pages are extremely similar to leaderboard pages.
     # Amazingly, this inheritence hack works!
     class SearchQuery < MarketBot::Android::Leaderboard
-      def initialze(query, options={})
+      def initialize(query, options={})
         super(query, nil, options)
       end
 

data/market_bot.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "market_bot"
-  s.version = "0.9.1"
+  s.version = "0.10.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Chad Remesch"]
-  s.date = "2012-10-10"
+  s.date = "2013-07-19"
   s.description = "Market Bot is a high performance Ruby scraper for Google's Android Market with a simple to use API.  It is built on top of Nokogiri and Typhoeus."
   s.email = "chad@remesch.com"
   s.extra_rdoc_files = [
@@ -51,18 +51,18 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/chadrem/market_bot"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "2.0.3"
   s.summary = "Market Bot: High performance Ruby scraper for Google's Android Market"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<typhoeus>, [">= 0"])
+      s.add_runtime_dependency(%q<typhoeus>, ["~> 0.4.2"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.1.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.3.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.0"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
       s.add_development_dependency(%q<rdoc>, [">= 3.9.4"])
       s.add_development_dependency(%q<guard>, ["~> 1.2.3"])
@@ -70,11 +70,11 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<growl>, [">= 0"])
       s.add_development_dependency(%q<debugger>, [">= 0"])
     else
-      s.add_dependency(%q<typhoeus>, [">= 0"])
+      s.add_dependency(%q<typhoeus>, ["~> 0.4.2"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_dependency(%q<bundler>, ["~> 1.1.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<bundler>, ["~> 1.3.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.0"])
       s.add_dependency(%q<simplecov>, [">= 0"])
       s.add_dependency(%q<rdoc>, [">= 3.9.4"])
       s.add_dependency(%q<guard>, ["~> 1.2.3"])
@@ -83,11 +83,11 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<debugger>, [">= 0"])
     end
   else
-    s.add_dependency(%q<typhoeus>, [">= 0"])
+    s.add_dependency(%q<typhoeus>, ["~> 0.4.2"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-    s.add_dependency(%q<bundler>, ["~> 1.1.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<bundler>, ["~> 1.3.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.0"])
     s.add_dependency(%q<simplecov>, [">= 0"])
     s.add_dependency(%q<rdoc>, [">= 3.9.4"])
     s.add_dependency(%q<guard>, ["~> 1.2.3"])