market_bot 0.17.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b9b827496e5425d3887b85c676d993abe3d0998
-  data.tar.gz: 1387fb5cf278a03a366120f96758623388a710db
+  metadata.gz: 9206239cc019edb10efdf93e3c56e0bda9817e4b
+  data.tar.gz: 50dd856eb3b5aa3a69b494ba641b70b10ce75bf0
 SHA512:
-  metadata.gz: 7aae31488f75f6c4c56fbc7d7e0bdf14a949dcf380d6de7a006024dd1e8b73e8f24c8ec118622932dbda16b8bcbd0147bbff129fc7c1db87b4c855ca58900d57
-  data.tar.gz: 17e6cb00e21febbca548e6a8d4750383890335d2d6d2e4204aebc456c2fe7ed0f509c9b71b471f2c31e45c5edb3158ddd1a1fd35aa912eb4ad76884db07f759e
+  metadata.gz: 23560b228f34cd479889578a7ee1b56d974a6c88a910e6522eb18680de1265be6554e9a5169e9374abb081c0a17f410669cce92f53f7d982ea8f445e961bd489
+  data.tar.gz: 851d0bd20f7b00d17d6524857c356067a0fea1c5dfd4b46589f9bbaa74da1c251b7dc9a74b6f31fab57566bfbcb300d3ac1979716d2ac94c26a198208fece940

data/.travis.yml

@@ -1,3 +1,3 @@
 language: ruby
 rvm:
-  - 2.2.2
+  - 2.3.1

data/Guardfile

@@ -0,0 +1,70 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+  # Rails files
+  rails = dsl.rails(view_extensions: %w(erb haml slim))
+  dsl.watch_spec_files_for(rails.app_files)
+  dsl.watch_spec_files_for(rails.views)
+
+  watch(rails.controllers) do |m|
+    [
+      rspec.spec.("routing/#{m[1]}_routing"),
+      rspec.spec.("controllers/#{m[1]}_controller"),
+      rspec.spec.("acceptance/#{m[1]}")
+    ]
+  end
+
+  # Rails config changes
+  watch(rails.spec_helper)     { rspec.spec_dir }
+  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
+  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
+
+  # Capybara features specs
+  watch(rails.view_dirs)     { |m| rspec.spec.("features/#{m[1]}") }
+  watch(rails.layouts)       { |m| rspec.spec.("features/#{m[1]}") }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
+    Dir[File.join("**/#{m[1]}.feature")][0] || "spec/acceptance"
+  end
+end

data/README.markdown

@@ -1,14 +1,13 @@
-# Market Bot - [![Build Status](https://travis-ci.org/chadrem/market_bot.svg?branch=master)](https://travis-ci.org/chadrem/market_bot) [![Coverage Status](https://coveralls.io/repos/chadrem/market_bot/badge.svg?branch=master&service=github)](https://coveralls.io/github/chadrem/market_bot?branch=master)
+# Market Bot - [![Build Status](https://travis-ci.org/chadrem/market_bot.svg?branch=master)](https://travis-ci.org/chadrem/market_bot) [![Coverage Status](https://coveralls.io/repos/github/chadrem/market_bot/badge.svg?branch=master)](https://coveralls.io/github/chadrem/market_bot?branch=master)
 
-Market Bot is a high performance Ruby scraper for Google's (Google Play) Android Market with a simple to use API.
-Currently it can: scrape apps, app leaderboards, movies, and tv shows.  It can also search for apps.
-It is built on top of Nokogiri and Typhoeus.
+Market Bot is a web scraper (web robot, web spider) for the Google Play Android app store.
+It can collect data on apps, charts, and developers.
 
-**This project is currently seeking developers to help maintain it.
-Please send pull requests or contact me if you are able to help out.
-The best ways to get started are to fix bugs, improve documentation,
-and merge useful changes from public forks.  By working together we
-can make the best scraper for the Google Play Store!**
+##### Commercial Support
+
+Want to buy data instead of collect it yourself?
+Want to add a feature to Market Bot, but don't know how?
+Please contact me ([chad@remesch.com](mailto:chad@remesch.com)) for commercial solutions.
 
 ## Dependencies
 
@@ -17,97 +16,49 @@ can make the best scraper for the Google Play Store!**
 
 ## Installation
 
-    gem install market_bot
+Add this line to your application's Gemfile to use the latest stable version:
+
+    gem 'market_bot'
+
+And then execute:
 
-## Getting Started Example
+    $ bundle
 
-    require 'rubygems'
-    require 'market_bot'
+## App API example
 
-    # Download all the data for the Facebook app.
-    app = MarketBot::Android::App.new('com.facebook.katana')
+    # Download/parse the app.
+    app = MarketBot::Play::App.new('com.facebook.katana')
     app.update
 
-    # Here we will print out the title of the app.
+    # Print out the app title.
     puts app.title
 
-    # Here we will print out the rating of the app.
-    puts app.rating
-
-    # And the price...
-    puts app.price
-
-    # market_bot has many other attributes for each app.
-    # You can see what attributes are available in your version of
-    # market_bot by printing a simple constant included in the gem.
-    puts MarketBot::Android::App::MARKET_ATTRIBUTES.inspect
-
-## Simple API Examples
-
-    require 'rubygems'
-    require 'market_bot'
-
-    # Download/parse the leaderboard.
-    lb = MarketBot::Android::Leaderboard.new(:topselling_free, :game)
-    lb.update
-
-    # Download/parse the details for the first and last entries of the leaderboard.
-    first_app = MarketBot::Android::App.new(lb.results.first[:market_id])
-    last_app = MarketBot::Android::App.new(lb.results.last[:market_id])
-    first_app.update
-    last_app.update
-    puts "First place app (#{first_app.title}) price: #{first_app.price}"
-    puts "Last place app (#{last_app.title}) price: #{last_app.price}"
-
-    # Search for apps.
-    sq = MarketBot::Android::SearchQuery.new('donkeys')
-    sq.update
-    puts "Results found: #{sq.results.count}"
-
-    # Download/parse developer pages.
-    developer = MarketBot::Android::Developer.new('Zynga')
-    developer.update
-    puts "Results found: #{developer.results.count}"
-
-    # Print the rating for an app.
-    puts MarketBot::Android::App.new('com.king.candycrushsaga').update.rating
-
-    # Check if an app exists and has a title.
-    def app_exists?(app_id)
-      begin
-        return !!MarketBot::Android::App.new(app_id).update.title
-      rescue
-        return false
-      end
-    end
-    app_exists?('com.king.candycrushsaga')  # Return's true.
-    app_exists?('com.some.fake.app.that.does.not.exist')  # Return's false.
-
-## Advanced API Examples
-
-    require 'rubygems'
-    require 'market_bot'
-
-    # Create a reusable hydra object with 5 http workers.
-    hydra = Typhoeus::Hydra.new(:max_concurrency => 5)
-
-    # Download/parse the leaderboard.
-    lb = MarketBot::Android::Leaderboard.new(:topselling_free, :game, :hyda => hydra)
-    lb.update
-
-    # Download/parse the details for the first and last entries of the leaderboard using the batch API.
-    first_app = MarketBot::Android::App.new(lb.results.first[:market_id], :hydra => hydra)
-    last_app = MarketBot::Android::App.new(lb.results.last[:market_id], :hydra => hydra)
-    first_app.enqueue_update
-    last_app.enqueue_update do |a|
-      # Callback example.  This block will execute on a successful hydra.run.
-      puts "Callback... title: #{a.title}"
-    end
-    hydra.run
-
-    # You must manually check if an error occurred when using the batch API without callbacks.
-    puts "First place app (#{first_app.title}) price: #{first_app.price}" unless first_app.error
-    puts "Last place app (#{last_app.title}) price: #{last_app.price}" unless last_app.error
+    # Print all the other attributes you can find on an app object.
+    puts MarketBot::Play::App::ATTRIBUTES.inspect
+
+## Charts API example
+
+Charts are defined by a collection and an option category.
+
+    # Download/parse the chart (collection=topselling_free, category=GAME).
+    chart = MarketBot::Play::Chart.new('topselling_free', 'GAME')
+    chart.update
+
+    # Print the first app.
+    puts chart.result.first.inspect
+
+    # Print all the chart collections & categories.
+    puts MarketBot::Play::Chart::COLLECTIONS.inspect
+    puts MarketBot::Play::Chart::CATEGORIES.inspect
+
+## Developer API example
+
+    # Download/parse developer.
+    dev = MarketBot::Play::Developer.new('Zynga')
+    dev.update
+
+    # Print the first app.
+    puts dev.result.first.inspect
 
 ## Excessive Use
 
@@ -126,6 +77,5 @@ Please contact me if you are looking for commercial data solutions.
 
 ## Copyright
 
-Copyright (c) 2011 - 2015 Chad Remesch. See LICENSE.txt for
+Copyright (c) 2011 - 2016 Chad Remesch. See LICENSE.txt for
 further details.
-

data/Rakefile

@@ -8,92 +8,3 @@ end
 
 task :default => :spec
 
-desc 'Start an IRB console with Workers loaded'
-task :console do
-  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
-
-  require 'market_bot'
-  require 'irb'
-
-  ARGV.clear
-
-  IRB.start
-end
-
-namespace :spec do
-  namespace :data do
-    require 'typhoeus'
-
-    def download(url, save_path)
-      resp = Typhoeus::Request.get(url)
-      File.open(save_path, "w") do |file|
-        resp.body.force_encoding('utf-8')
-        file.puts(resp.body)
-      end
-    end
-
-    desc 'download the latest test data used by the specs'
-    task :update do
-      download('https://play.google.com/store/apps/details?id=com.bluefroggaming.popdat',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'app_1.txt'))
-
-      download('https://play.google.com/store/apps/details?id=com.evernote',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'app_2.txt'))
-
-      download('https://play.google.com/store/apps/details?id=com.mg.android',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'app_3.txt'))
-
-      download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?num=24',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page1.txt'))
-
-      download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=24&num=24',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page2.txt'))
-
-      download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=48&num=24',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page3.txt'))
-
-      download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=96&num=24',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page4.txt'))
-
-      download('https://play.google.com/store/apps/collection/editors_choice',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_editors_choice.txt'))
-
-      download('https://play.google.com/store/apps/developer?id=Zynga',
-               File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'developer-zynga.txt'))
-    end
-  end
-end
-
-namespace :benchmark do
-  task :android do
-    require 'market_bot'
-    require 'benchmark'
-
-    hydra = Typhoeus::Hydra.new(:max_concurrency => 20)
-
-    leaderboard = nil
-    leaderboard_ms = Benchmark.realtime {
-      leaderboard = MarketBot::Android::Leaderboard.new(:apps_topselling_paid, nil, :hydra => hydra)
-      leaderboard.update
-    }
-
-    puts '----------------------------------------------------'
-    puts 'Benchmark Leaderboard: Top Selling Paid Apps'
-    puts '----------------------------------------------------'
-    puts "app count: #{leaderboard.results.length}"
-    puts "time: #{leaderboard_ms.round(3)} seconds"
-    puts
-
-    apps = nil
-    apps_ms = Benchmark.realtime {
-      apps = leaderboard.results.map{ |r| MarketBot::Android::App.new(r[:market_id], :hydra => hydra).enqueue_update }
-      hydra.run
-    }
-
-    puts '----------------------------------------------------'
-    puts 'Benchmark Apps: top Selling Paid Apps'
-    puts '----------------------------------------------------'
-    puts "app count: #{apps.length}"
-    puts "time: #{apps_ms.round(3)} seconds"
-  end
-end

data/bin/app_categories

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'nokogiri'
+require 'typhoeus'
+
+res = Typhoeus.get('https://play.google.com/store/apps', timeout: 5, connecttimeout: 5)
+doc = Nokogiri::HTML(res.body)
+elems = doc.css('a.child-submenu-link')
+links = elems.map { |e| e.attributes['href'].value }
+
+# Filter out non-app links such as 'Popular Characters'.
+links = links.select { |l| l =~ /apps\/category/ }
+
+categories = links.map { |l| l.gsub('/store/apps/category/', '') }
+
+categories.each { |c| puts c }

data/bin/benchmark_parser_app

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'market_bot'
+
+require 'benchmark'
+
+loops = 5
+test_files = %w{app_1.txt app_2.txt app_3.txt}
+test_files.each do |test_file|
+  puts '----------------------------------------------------'
+  puts "Benchmark App Parser: #{test_file}"
+  puts '----------------------------------------------------'
+  file_path = File.join(File.dirname(__FILE__), '..', 'spec', 'market_bot', 'play', 'data', test_file)
+  html = File.read(file_path)
+
+  total = 0.0
+  loops.times do
+    total += Benchmark.realtime { MarketBot::Play::App.parse(html) }
+  end
+  average = total.to_f / loops.to_f
+
+  puts "average: #{average} seconds"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "market_bot"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/update_test_data

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'typhoeus'
+
+def download(url, name)
+  path = play_path(name)
+  puts "Downloading: #{name}..."
+  resp = Typhoeus::Request.get(url)
+  File.open(path, 'w') do |file|
+    resp.body.force_encoding('utf-8')
+    file.puts(resp.body)
+  end
+end
+
+def play_path(name)
+  File.join(File.dirname(__FILE__), '..', 'spec', 'market_bot', 'play', 'data', name)
+end
+
+download('https://play.google.com/store/apps/details?id=com.bluefroggaming.popdat', 'app-com.bluefroggaming.popdat.txt')
+download('https://play.google.com/store/apps/details?id=com.mg.android', 'app-com.mg.android.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=0&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-0.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=100&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-1.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=200&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-2.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=300&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-3.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=400&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-4.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=500&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-5.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=600&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-6.txt')
+download('https://play.google.com/store/apps/category/GAME_ARCADE/collection/topselling_paid?start=700&gl=us&num=100&hl=en', 'chart-topselling_paid-GAME_ARCADE-7.txt')
+download('https://play.google.com/store/apps/developer?id=Zynga', 'developer-zynga.txt')