market_bot 0.17.0 → 1.0.0

data/spec/market_bot/play/developer_spec.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+describe MarketBot::Play::Developer do
+  shared_context('parsing a developer') do
+    it 'should be a valid length' do
+      expect(@parsed.length).to be > 1
+    end
+
+    it 'should have attributes' do
+      expect(@parsed).to all(have_key(:package)).and \
+        all(have_key(:title)).and all(have_key(:store_url)).and \
+        all(have_key(:developer)).and all(have_key(:icon_url))
+    end
+  end
+
+  describe "(zynga)" do
+    include_context 'parsing a developer'
+
+    before(:all) do
+      @name = 'zynga'
+      @html = read_play_data('developer-zynga.txt')
+      @parsed = MarketBot::Play::Chart.parse(@html)
+    end
+  end
+
+  it 'should generate store_urls' do
+    name ='zynga'
+    dev = MarketBot::Play::Developer.new(name)
+
+    expect(dev.store_urls.length).to eq(1)
+
+    dev.store_urls.each_with_index do |url, i|
+      msg = "i=#{i}, url=#{url}"
+      pattern = /\Ahttps:\/\/play\.google\.com\/store\/apps\/developer\?id=zynga&start=0&gl=us&num=100&hl=en\z/
+      expect(url).to match(Regexp.new(pattern)), msg
+    end
+  end
+
+  it 'should update' do
+    name = 'zynga'
+    dev = MarketBot::Play::Developer.new(name)
+    code = 200
+
+    dev.store_urls.each_with_index do |url, i|
+      html = read_play_data('developer-zynga.txt')
+      response = Typhoeus::Response.new(code: code, headers: '', body: html)
+      Typhoeus.stub(url).and_return(response)
+    end
+
+    dev.update
+  end
+end

data/spec/market_bot_spec.rb

@@ -1,4 +1,18 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "MarketBot" do
+  it 'should support changing the default request timeout' do
+    MarketBot.timeout = 5
+    expect(MarketBot.timeout).to eq(5)
+  end
+
+  it 'should support changing the default request connect timeout' do
+    MarketBot.connect_timeout = 5
+    expect(MarketBot.connect_timeout).to eq(5)
+  end
+
+  it 'should support changing the default request user agent' do
+    MarketBot.user_agent = 'My Custom Bot'
+    expect(MarketBot.user_agent).to eq('My Custom Bot')
+  end
 end

data/spec/spec_helper.rb

@@ -7,13 +7,7 @@ require 'rspec'
 
 begin
   require 'byebug'
-rescue Exception => e
-end
-
-begin
-  require 'simplecov'
-  SimpleCov.start
-rescue Exception => e
+rescue Exception
 end
 
 require 'market_bot'
@@ -29,8 +23,8 @@ Typhoeus.configure do |config|
   config.block_connection = true
 end
 
-def read_file(*path)
-  data = nil
-  File.open(File.join(path), 'r') { |f| data = f.read }
-  return data
+def read_play_data(name)
+  path = File.join(File.expand_path(File.dirname(__FILE__)),
+                   'market_bot', 'play', 'data', name)
+  File.read(path)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: market_bot
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Chad Remesch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-06 00:00:00.000000000 Z
+date: 2016-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -42,49 +42,54 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: '3.4'
 description: Market Bot is a high performance Ruby scraper for Google's Android Market
   with a simple to use API.  It is built on top of Nokogiri and Typhoeus.
 email:
 - chad@remesch.com
-executables: []
+executables:
+- app_categories
+- benchmark_parser_app
+- console
+- setup
+- update_test_data
 extensions: []
 extra_rdoc_files: []
 files:
@@ -92,38 +97,40 @@ files:
 - ".rspec"
 - ".travis.yml"
 - Gemfile
+- Guardfile
 - LICENSE.txt
-- NOTES.txt
 - README.markdown
 - Rakefile
+- bin/app_categories
+- bin/benchmark_parser_app
+- bin/console
+- bin/setup
+- bin/update_test_data
 - lib/market_bot.rb
-- lib/market_bot/android/app.rb
-- lib/market_bot/android/developer.rb
-- lib/market_bot/android/leaderboard.rb
-- lib/market_bot/android/leaderboard/constants.rb
-- lib/market_bot/android/search_query.rb
 - lib/market_bot/exceptions.rb
-- lib/market_bot/movie/leaderboard.rb
-- lib/market_bot/movie/leaderboard/constants.rb
-- lib/market_bot/movie/search_query.rb
+- lib/market_bot/play/app.rb
+- lib/market_bot/play/app/constants.rb
+- lib/market_bot/play/chart.rb
+- lib/market_bot/play/chart/constants.rb
+- lib/market_bot/play/constants.rb
+- lib/market_bot/play/developer.rb
+- lib/market_bot/util.rb
 - lib/market_bot/version.rb
 - market_bot.gemspec
-- spec/market_bot/android/app_spec.rb
-- spec/market_bot/android/data/app_1.txt
-- spec/market_bot/android/data/app_2.txt
-- spec/market_bot/android/data/app_3.txt
-- spec/market_bot/android/data/app_4.txt
-- spec/market_bot/android/data/developer-zynga.txt
-- spec/market_bot/android/data/leaderboard-apps_editors_choice.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page1.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page2.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page3.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page4.txt
-- spec/market_bot/android/developer_spec.rb
-- spec/market_bot/android/leaderboard_spec.rb
-- spec/market_bot/android/search_query_spec.rb
-- spec/market_bot/movie/data/leaderboard-movies_comedy_topselling_paid.txt
-- spec/market_bot/movie/leaderboard_spec.rb
+- spec/market_bot/play/app_spec.rb
+- spec/market_bot/play/chart_spec.rb
+- spec/market_bot/play/data/app-com.bluefroggaming.popdat.txt
+- spec/market_bot/play/data/app-com.mg.android.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-0.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-1.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-2.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-3.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-4.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-5.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-6.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-7.txt
+- spec/market_bot/play/data/developer-zynga.txt
+- spec/market_bot/play/developer_spec.rb
 - spec/market_bot_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/chadrem/market_bot
@@ -150,21 +157,19 @@ signing_key:
 specification_version: 4
 summary: 'Market Bot: High performance Ruby scraper for Google''s Android Market'
 test_files:
-- spec/market_bot/android/app_spec.rb
-- spec/market_bot/android/data/app_1.txt
-- spec/market_bot/android/data/app_2.txt
-- spec/market_bot/android/data/app_3.txt
-- spec/market_bot/android/data/app_4.txt
-- spec/market_bot/android/data/developer-zynga.txt
-- spec/market_bot/android/data/leaderboard-apps_editors_choice.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page1.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page2.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page3.txt
-- spec/market_bot/android/data/leaderboard-apps_topselling_paid-page4.txt
-- spec/market_bot/android/developer_spec.rb
-- spec/market_bot/android/leaderboard_spec.rb
-- spec/market_bot/android/search_query_spec.rb
-- spec/market_bot/movie/data/leaderboard-movies_comedy_topselling_paid.txt
-- spec/market_bot/movie/leaderboard_spec.rb
+- spec/market_bot/play/app_spec.rb
+- spec/market_bot/play/chart_spec.rb
+- spec/market_bot/play/data/app-com.bluefroggaming.popdat.txt
+- spec/market_bot/play/data/app-com.mg.android.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-0.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-1.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-2.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-3.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-4.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-5.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-6.txt
+- spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-7.txt
+- spec/market_bot/play/data/developer-zynga.txt
+- spec/market_bot/play/developer_spec.rb
 - spec/market_bot_spec.rb
 - spec/spec_helper.rb

data/NOTES.txt

@@ -1,61 +0,0 @@
-URLs:
-- id can be one of the below IDs or an application ID which usually starts with com.
-- category is optional and works with a subset of the below IDs.
-https://market.android.com/details?id=&cat=
-https://market.android.com/details?id=com.angrymobgames.muffinknightfree
-
-Data Models:
-Android::Leaderboard
-Android::Leaderboard::Stat
-Android::App
-Android::App::Stat
-
-IDs:
-apps_topselling_paid
-apps_topselling_free
-apps_topselling_new_free
-apps_topselling_new_paid
-apps_editors_choice
-apps_topselling_paid_game
-apps_movers_shakers
-apps_featured
-apps_tablet_featured
-apps_topgrossing
-
-Categories:
-GAME
-ARCADE
-BRAIN
-CARDS
-CASUAL
-GAME_WALLPAPER
-RACING
-SPORTS_GAMES
-GAME_WIDGETS
-APPLICATION
-BOOKS_AND_REFERENCE
-BUSINESS
-COMICS
-COMMUNICATION
-EDUCATION
-ENTERTAINMENT
-FINANCE
-HEALTH_AND_FITNESS
-LIBRARIES_AND_DEMO
-LIFESTYLE
-APP_WALLPAPER
-MEDIA_AND_VIDEO
-MEDICAL
-MUSIC_AND_AUDIO
-NEWS_AND_MAGAZINES
-PERSONALIZATION
-PHOTOGRAPHY
-PRODUCTIVITY
-SHOPPING
-SOCIAL
-SPORTS
-TOOLS
-TRANSPORTATION
-TRAVEL_AND_LOCAL
-WEATHER
-APP_WIDGETS

data/lib/market_bot/android/app.rb

@@ -1,273 +0,0 @@
-module MarketBot
-  module Android
-
-    class App
-      MARKET_ATTRIBUTES = [
-        :title,
-        :rating,
-        :updated,
-        :current_version,
-        :requires_android,
-        :category,
-        :installs,
-        :size,
-        :price,
-        :content_rating,
-        :description,
-        :votes,
-        :developer,
-        :more_from_developer,
-        :users_also_installed,
-        :related,
-        :banner_icon_url,
-        :banner_image_url,
-        :website_url,
-        :email,
-        :screenshot_urls,
-        :whats_new,
-        :permissions,
-        :rating_distribution,
-        :html,
-        :category_url,
-        :full_screenshot_urls,
-        :reviews,
-      ]
-
-      attr_reader :app_id
-      attr_reader *MARKET_ATTRIBUTES
-      attr_reader :hydra
-      attr_reader :lang
-      attr_reader :callback
-      attr_reader :error
-      attr_reader :result
-
-      def self.parse(html)
-        result = {}
-
-        doc = Nokogiri::HTML(html)
-        meta_info = doc.css('.meta-info')
-        meta_info.each do |info|
-          field_name = info.css('.title').text.strip
-
-          case field_name
-          when 'Updated'
-            result[:updated] = info.css('.content').text.strip
-          when 'Installs'
-            result[:installs] = info.css('.content').text.strip
-          when 'Size'
-            result[:size] = info.css('.content').text.strip
-          when 'Current Version'
-            result[:current_version] = info.css('.content').text.strip
-          when 'Requires Android'
-            result[:requires_android] = info.css('.content').text.strip
-          when 'Contact Developer', 'Developer'
-            info.css('.dev-link').each do |node|
-              node_href = node[:href]
-              if node_href =~ /^mailto:/
-                result[:email] = node_href.gsub(/^mailto:/,'')
-              else
-                if q_param = URI(node_href).query.split('&').select{ |p| p =~ /q=/ }.first
-                  actual_url = q_param.gsub('q=', '')
-                end
-
-                result[:website_url] = actual_url
-              end
-            end
-
-          end
-        end
-
-        result[:content_rating] = doc.css("div.content[itemprop='contentRating']").text
-
-        node = doc.xpath("//meta[@itemprop='price']").first
-        result[:price] = node[:content].strip rescue 'Free'
-
-        category_div = doc.css('.category').first
-        result[:category] = category_div.text.strip rescue ''
-        cat_link = category_div["href"]
-        path, cat_name = File.split(cat_link)
-        result[:category_url] = cat_name
-
-        result[:description] = doc.xpath("//div[@itemprop='description']").first.inner_html.strip
-        result[:title] = doc.css('div.id-app-title').text
-
-        score = doc.css('.score-container').first
-        unless score.nil?
-          node  = score.css('.score').first
-          result[:rating] = node.text.strip
-          node = score.xpath("//meta[@itemprop='ratingCount']").first
-          result[:votes] = node[:content].strip
-        end
-
-        node = doc.xpath("//div[@itemprop='author']")
-        result[:developer] = node.css('.primary').first.text.strip
-
-        result[:more_from_developer] = []
-        result[:users_also_installed] = []
-        result[:related] = []
-
-        node = doc.css('.recommendation')
-        node.css('.rec-cluster').each do |recommended|
-          assoc_app_type = recommended.css('.heading').first.text.strip.eql?('Similar' ) ? :related : :more_from_developer
-          recommended.css('.card').each do |card|
-            assoc_app = {}
-            assoc_app[:app_id] = card['data-docid'].strip
-
-            result[assoc_app_type] << assoc_app
-          end
-        end
-        # Users also installed is no longer on the page, adding this for backwards compatibility, well, sort of....
-        result[:users_also_installed] = result[:related]
-
-        node = doc.css('.cover-image').first
-        unless node.nil?
-          url = fix_content_url(node[:src])
-          result[:banner_icon_url] = url
-          result[:banner_image_url] = url
-        end
-
-        result[:screenshot_urls] = []
-        doc.css('.screenshot').each do |node|
-          result[:screenshot_urls] << fix_content_url(node[:src])
-        end
-
-        result[:full_screenshot_urls] = []
-        doc.css('.full-screenshot').each do |node|
-          result[:full_screenshot_urls] << fix_content_url(node[:src])
-        end
-
-        node = doc.css('.whatsnew').first
-        result[:whats_new] = node.inner_html.strip unless node.nil?
-
-        # Stubbing out for now, can't find them in the redesigned page.
-        result[:permissions] = permissions = []
-
-        result[:reviews] = []
-        doc.css('.single-review').each do |node|
-          review = {}
-          review[:author_name] = node.css('.author-name').text.strip if node.css('.author-name')
-          raw_tag = node.css('.current-rating').to_s
-          if raw_tag.match(/100%;/i)
-            review[:review_score] = 5
-          elsif raw_tag.match(/80%;/i)
-            review[:review_score] = 4
-          elsif raw_tag.match(/60%;/i)
-            review[:review_score] = 3
-          elsif raw_tag.match(/40%;/i)
-            review[:review_score] = 2
-          elsif raw_tag.match(/20%;/i)
-            review[:review_score] = 1
-          end
-          if node.css('.review-title')
-            review[:review_title] = node.css('.review-title').text.strip
-          end
-          if node.css('.review-body')
-            review[:review_text] = node.css('.review-body').text
-              .sub!(review[:review_title],'')
-              .sub!(node.css('.review-link').text,'')
-              .strip
-          end
-          if review
-            result[:reviews] << review
-          end
-        end
-
-        result[:rating_distribution] = { 5 => nil, 4 => nil, 3 => nil, 2 => nil, 1 => nil }
-
-        histogram = doc.css('div.rating-histogram')
-        cur_index = 5
-        %w(five four three two one).each do |slot|
-          node = histogram.css(".#{slot.to_s}")
-          result[:rating_distribution][cur_index] = node.css('.bar-number').text.gsub(/,/,'').to_i
-          cur_index -= 1
-
-        end
-
-        result[:html] = html
-
-        result
-      end
-
-      def initialize(app_id, options={})
-        @app_id = app_id
-        @hydra = options[:hydra] || MarketBot.hydra
-        @lang = options[:lang] || 'en'
-        @callback = nil
-        @error = nil
-        @request_opts = options[:request_opts] || {}
-        @request_opts[:timeout] ||= MarketBot.timeout
-        @request_opts[:connecttimeout] ||= MarketBot.connecttimeout
-      end
-
-      def market_url
-        "https://play.google.com/store/apps/details?id=#{@app_id}&hl=#{lang}"
-      end
-
-      def update
-        req = Typhoeus::Request.new(market_url, @request_opts)
-        req.run
-        result = handle_response(req.response)
-        update_callback(result)
-
-        self
-      end
-
-      def enqueue_update(&block)
-        @callback = block
-        @error = nil
-
-        request = Typhoeus::Request.new(market_url, @request_opts)
-
-        request.on_complete do |response|
-          result = nil
-
-          begin
-            result = handle_response(response)
-          rescue Exception => e
-            @error = e
-          end
-
-          update_callback(result)
-        end
-
-        hydra.queue(request)
-
-        self
-      end
-
-      private
-
-      def self.fix_content_url(url)
-        url =~ /\A\/\// ? "https:#{url}" : url
-      end
-
-      def handle_response(response)
-        if response.success?
-          App.parse(response.body)
-        else
-          codes = "code=#{response.code}, return_code=#{response.return_code}"
-          case response.code
-          when 404
-            raise MarketBot::AppNotFoundError.new("Unable to find app in store: #{codes}")
-          else
-            raise MarketBot::ResponseError.new("Unhandled response: #{codes}")
-          end
-        end
-      end
-
-      def update_callback(result)
-        unless @error
-          @result = result
-
-          MARKET_ATTRIBUTES.each do |a|
-            attr_name = "@#{a}"
-            attr_value = result[a]
-            instance_variable_set(attr_name, attr_value)
-          end
-        end
-
-        @callback.call(self) if @callback
-      end
-    end
-  end
-end