market_bot 0.17.0 → 1.0.0

data/lib/market_bot.rb

@@ -5,19 +5,15 @@ require 'nokogiri'
 
 require 'market_bot/version'
 require 'market_bot/exceptions'
-require 'market_bot/android/app'
-require 'market_bot/android/leaderboard/constants'
-require 'market_bot/android/leaderboard'
-require 'market_bot/android/search_query'
-require 'market_bot/android/developer'
-require 'market_bot/movie/leaderboard'
-require 'market_bot/movie/search_query'
+require 'market_bot/util'
+require 'market_bot/play/constants'
+require 'market_bot/play/app/constants'
+require 'market_bot/play/app'
+require 'market_bot/play/chart/constants'
+require 'market_bot/play/chart'
+require 'market_bot/play/developer'
 
 module MarketBot
-  def self.hydra
-    @hydra ||= Typhoeus::Hydra.new(:max_concurrency => 5)
-  end
-
   def self.timeout
     @timeout ||= 10
   end
@@ -26,12 +22,21 @@ module MarketBot
     @timeout = val
   end
 
-  def self.connecttimeout
-    @connecttimeout ||= 10
+  def self.connect_timeout
+    @connect_timeout ||= 10
+  end
+
+  def self.connect_timeout=(val)
+    @connect_timeout = val
+  end
+
+  def self.user_agent
+    @user_agent ||= "MarketBot/#{MarketBot::VERSION} / " \
+      "(+https://github.com/chadrem/market_bot)"
   end
 
-  def self.connecttimeout=(val)
-    @connecttimeout = val
+  def self.user_agent=(val)
+    @user_agent = val
   end
 end
 

data/lib/market_bot/exceptions.rb

@@ -1,5 +1,6 @@
 module MarketBot
   class MarketBotError < StandardError; end
   class ResponseError < MarketBotError; end
-  class AppNotFoundError < ResponseError; end
+  class NotFoundError < ResponseError; end
+  class UnavailableError < ResponseError; end
 end

data/lib/market_bot/play/app.rb

@@ -0,0 +1,188 @@
+module MarketBot
+  module Play
+    class App
+      attr_reader *ATTRIBUTES
+      attr_reader :package
+      attr_reader :lang
+      attr_reader :result
+
+      def self.parse(html, opts={})
+        result = {}
+
+        doc = Nokogiri::HTML(html)
+        meta_info = doc.css('.meta-info')
+        meta_info.each do |info|
+          field_name = info.css('.title').text.strip
+
+          case field_name
+          when 'Updated'
+            result[:updated] = info.at_css('.content').text.strip
+          when 'Installs'
+            result[:installs] = info.at_css('.content').text.strip
+          when 'Size'
+            result[:size] = info.at_css('.content').text.strip
+          when 'Current Version'
+            result[:current_version] = info.at_css('.content').text.strip
+          when 'Requires Android'
+            result[:requires_android] = info.at_css('.content').text.strip
+          when 'Contact Developer', 'Developer'
+            info.css('.dev-link').each do |node|
+              node_href = node[:href]
+              if node_href =~ /^mailto:/
+                result[:email] = node_href.gsub(/^mailto:/,'')
+              else
+                if q_param = URI(node_href).query.split('&').select{ |p| p =~ /q=/ }.first
+                  actual_url = q_param.gsub('q=', '')
+                end
+
+                result[:website_url] = actual_url
+              end
+            end
+
+          end
+        end
+
+        result[:content_rating] = doc.at_css("div.content[itemprop='contentRating']").text
+
+        result[:price] = doc.at_css('meta[itemprop="price"]')[:content]
+
+        category_div = doc.at_css('.category')
+        result[:category] = category_div.text.strip rescue nil
+        result[:category_url] = File.split(category_div["href"])[1]
+
+        result[:description] = doc.at_css('div[itemprop="description"]').inner_html.strip
+        result[:title] = doc.at_css('div.id-app-title').text
+
+        score = doc.at_css('.score-container')
+        unless score.nil?
+          node  = score.at_css('.score')
+          result[:rating] = node.text.strip
+          node = score.at_css('meta[itemprop="ratingCount"]')
+          result[:votes] = node[:content].strip.to_i
+        end
+
+        node = doc.at_css('div[itemprop="author"]')
+        result[:developer] = node.at_css('.primary').text.strip
+
+        result[:more_from_developer] = []
+        result[:similar] = []
+
+        node = doc.css('.recommendation')
+        node.css('.rec-cluster').each do |recommended|
+          assoc_app_type = recommended.at_css('.heading').text.strip.eql?('Similar' ) ? :similar : :more_from_developer
+          recommended.css('.card').each do |card|
+            assoc_app = {}
+            assoc_app[:package] = card['data-docid'].strip
+
+            result[assoc_app_type] << assoc_app
+          end
+        end
+
+        node = doc.at_css('.cover-image')
+        unless node.nil?
+          url = MarketBot::Util.fix_content_url(node[:src])
+          result[:cover_image_url] = url
+        end
+
+        result[:screenshot_urls] = []
+        doc.css('.screenshot').each do |node|
+          result[:screenshot_urls] << MarketBot::Util.fix_content_url(node[:src])
+        end
+
+        result[:full_screenshot_urls] = []
+        doc.css('.full-screenshot').each do |node|
+          result[:full_screenshot_urls] << MarketBot::Util.fix_content_url(node[:src])
+        end
+
+        result[:reviews] = []
+        unless opts[:skip_reviews] # Review parsing is CPU intensive.
+          doc.css('.single-review').each do |node|
+            review = {}
+            review[:author] = node.at_css('.author-name').text.strip if node.at_css('.author-name')
+            raw_tag = node.at_css('.current-rating').to_s
+            if raw_tag.match(/100%;/i)
+              review[:score] = 5
+            elsif raw_tag.match(/80%;/i)
+              review[:score] = 4
+            elsif raw_tag.match(/60%;/i)
+              review[:score] = 3
+            elsif raw_tag.match(/40%;/i)
+              review[:score] = 2
+            elsif raw_tag.match(/20%;/i)
+              review[:score] = 1
+            end
+            if node.at_css('.review-title')
+              review[:title] = node.at_css('.review-title').text.strip
+            end
+            if node.at_css('.review-body')
+              review[:text] = node.at_css('.review-body').text
+                .sub!(review[:title],'')
+                .sub!(node.at_css('.review-link').text, '')
+                .strip
+            end
+            if review
+              result[:reviews] << review
+            end
+          end
+        end
+
+        result[:rating_distribution] = { 5 => nil, 4 => nil, 3 => nil, 2 => nil, 1 => nil }
+
+        histogram = doc.css('div.rating-histogram')
+        cur_index = 5
+        %w(five four three two one).each do |slot|
+          node = histogram.at_css(".#{slot.to_s}")
+          result[:rating_distribution][cur_index] = node.css('.bar-number').text.gsub(/,/,'').to_i
+          cur_index -= 1
+
+        end
+
+        result[:html] = html
+
+        result
+      end
+
+      def initialize(package, opts={})
+        @package = package
+        @lang = opts[:lang] || MarketBot::Play::DEFAULT_LANG
+        @request_opts = MarketBot::Util.build_request_opts(opts[:request_opts])
+      end
+
+      def store_url
+        "https://play.google.com/store/apps/details?id=#{@package}&hl=#{@lang}"
+      end
+
+      def update
+        req = Typhoeus::Request.new(store_url, @request_opts)
+        req.run
+        response_handler(req.response)
+
+        self
+      end
+
+      private
+
+      def response_handler(response)
+        if response.success?
+          @result = self.class.parse(response.body)
+
+          ATTRIBUTES.each do |a|
+            attr_name = "@#{a}"
+            attr_value = @result[a]
+            instance_variable_set(attr_name, attr_value)
+          end
+        else
+          codes = "code=#{response.code}, return_code=#{response.return_code}"
+          case response.code
+          when 404
+            raise MarketBot::NotFoundError.new("Unable to find app in store: #{codes}")
+          when 403
+            raise MarketBot::UnavailableError.new("Unavailable app (country restriction?): #{codes}")
+          else
+            raise MarketBot::ResponseError.new("Unhandled response: #{codes}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/market_bot/play/app/constants.rb

@@ -0,0 +1,33 @@
+module MarketBot
+  module Play
+    class App
+      ATTRIBUTES = [
+        :category,
+        :category_url,
+        :content_rating,
+        :cover_image_url,
+        :current_version,
+        :description,
+        :developer,
+        :email,
+        :full_screenshot_urls,
+        :html,
+        :installs,
+        :more_from_developer,
+        :price,
+        :rating,
+        :rating_distribution,
+        :requires_android,
+        :reviews,
+        :screenshot_urls,
+        :similar,
+        :size,
+        :title,
+        :updated,
+        :votes,
+        :website_url,
+        :whats_new
+      ]
+    end
+  end
+end

data/lib/market_bot/play/chart.rb

@@ -0,0 +1,118 @@
+module MarketBot
+	module Play
+		class Chart
+			attr_reader :collection
+			attr_reader :category
+			attr_reader :country
+			attr_reader :lang
+			attr_reader :result
+
+			def self.parse(html, opts={})
+				opts[:lang] ||= MarketBot::Play::DEFAULT_LANG
+
+				results = []
+				doc = Nokogiri::HTML(html)
+
+				doc.css('.card').each do |snippet_node|
+					result = {}
+
+					details_node = snippet_node.css('.details')
+
+					unless snippet_node.css('img').empty?
+						result[:icon_url] = MarketBot::Util.fix_content_url(snippet_node.css('img').first.attributes['src'].value)
+					end
+
+					unless snippet_node.css('.current-rating').empty?
+						stars_style = snippet_node.css('.current-rating').first.attributes['style'].value
+						stars_width_percent = stars_style[/width:\s+([0-9.]+)%/, 1].to_f
+						result[:stars] = (5 * stars_width_percent / 100).round(1).to_s
+					else
+						result[:stars] = nil
+					end
+
+					title_node = details_node.css('.title').first
+					result[:title] = title_node.attributes['title'].to_s
+					result[:rank] = title_node.text.gsub(/\..*/, '').to_i
+
+					if (price_elem = details_node.css('.buy span').first)
+						result[:price] = price_elem.text
+					end
+
+					result[:developer] = details_node.css('.subtitle').first.attributes['title'].to_s
+					result[:package] = details_node.css('.title').first.attributes['href'].to_s.gsub('/store/apps/details?id=', '').gsub(/&feature=.*$/, '')
+					result[:store_url] = "https://play.google.com/store/apps/details?id=#{result[:package]}&hl=#{opts[:lang]}"
+
+					result[:price] = '0' if result[:price] == 'Free'
+
+					results << result
+				end
+
+				results
+			end
+
+			def initialize(collection, category=nil, opts={})
+				@collection = collection
+				@category = category
+				@request_opts = MarketBot::Util.build_request_opts(opts[:request_opts])
+				@lang = opts[:lang] || MarketBot::Play::DEFAULT_LANG
+				@country = opts[:country] || MarketBot::Play::DEFAULT_COUNTRY
+        @max_pages = opts[:max_pages] || MarketBot::Play::Chart::MAX_PAGES
+			end
+
+			def store_urls
+				urls = []
+				start = 0
+				num = 100
+
+				@max_pages.times do |i|
+					url = 'https://play.google.com/store/apps'
+					url << "/category/#{@category}" if @category
+					url << "/collection/#{@collection}?"
+					url << "start=#{start}&"
+					url << "gl=#{@country}&"
+					url << "num=#{num}&"
+					url << "hl=#{@lang}"
+
+					urls << url
+					start += num
+				end
+
+				urls
+			end
+
+			def update(opts={})
+				@result = []
+
+				store_urls.each do |url|
+					req = Typhoeus::Request.new(url, @request_opts)
+					req.run
+
+					break unless response_handler(req.response)
+				end
+
+				@result.flatten!
+
+				self
+			end
+
+			private
+
+			def response_handler(response)
+				if response.success?
+					r = self.class.parse(response.body, lang: @lang)
+
+					if @result.empty? ||
+							(!@result.empty? &&@result[-1][-1][:rank] + 1 == r[0][:rank])
+						@result << r
+						return true
+					end
+
+					return false
+				else
+					codes = "code=#{response.code}, return_code=#{response.return_code}"
+					raise MarketBot::ResponseError.new("Unhandled response: #{codes}")
+				end
+			end
+		end
+	end
+end

data/lib/market_bot/play/chart/constants.rb

@@ -0,0 +1,74 @@
+module MarketBot
+  module Play
+    class Chart
+      MAX_PAGES = 6
+
+      COLLECTIONS = %w{
+        topselling_free
+        topselling_paid
+        topgrossing
+        topselling_free
+        movers_shakers
+        topgrossing
+        topselling_new_free
+        topselling_new_paid
+      }
+
+      CATEGORIES = %w{
+				ANDROID_WEAR
+				BOOKS_AND_REFERENCE
+				BUSINESS
+				COMICS
+				COMMUNICATION
+				EDUCATION
+				ENTERTAINMENT
+				FINANCE
+				HEALTH_AND_FITNESS
+				LIBRARIES_AND_DEMO
+				LIFESTYLE
+				APP_WALLPAPER
+				MEDIA_AND_VIDEO
+				MEDICAL
+				MUSIC_AND_AUDIO
+				NEWS_AND_MAGAZINES
+				PERSONALIZATION
+				PHOTOGRAPHY
+				PRODUCTIVITY
+				SHOPPING
+				SOCIAL
+				SPORTS
+				TOOLS
+				TRANSPORTATION
+				TRAVEL_AND_LOCAL
+				WEATHER
+				APP_WIDGETS
+				GAME_ACTION
+				GAME_ADVENTURE
+				GAME_ARCADE
+				GAME_BOARD
+				GAME_CARD
+				GAME_CASINO
+				GAME_CASUAL
+				GAME_EDUCATIONAL
+				GAME_MUSIC
+				GAME_PUZZLE
+				GAME_RACING
+				GAME_ROLE_PLAYING
+				GAME_SIMULATION
+				GAME_SPORTS
+				GAME_STRATEGY
+				GAME_TRIVIA
+				GAME_WORD
+				FAMILY?age=AGE_RANGE1
+				FAMILY?age=AGE_RANGE2
+				FAMILY?age=AGE_RANGE3
+				FAMILY_ACTION
+				FAMILY_BRAINGAMES
+				FAMILY_CREATE
+				FAMILY_EDUCATION
+				FAMILY_MUSICVIDEO
+				FAMILY_PRETEND
+      }
+    end
+  end
+end