market_bot 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9688ff41be35247c4bb0f85621bb800232111936
-  data.tar.gz: 368e41730331ab5f6084ce49d6a51cbe87904d51
+  metadata.gz: b957c274ab5e86e3c0415b1af975ce4a115bbd78
+  data.tar.gz: 17e27abcbf8f10e5e9c338ece121d8067a4fb878
 SHA512:
-  metadata.gz: 34de9be2e58bfeab53156bd5b3248fcd412918616431ab03b3521b5065f55d0b07deb913f7f08a07cf512b5151dc1c8859a5e52f70c1e1c84cd86a9129d57fd3
-  data.tar.gz: 96e62fed37bee0a7ee35725a45470a3f912e7651e3868ae102f6135a7d6666e39abab95a76465e5849dafd86384745984bdebfb87949d792fbd50d11a393ef6a
+  metadata.gz: 226c78c540a1a8a4424e6a59839b89e50bf0b9e1390746e4df41c97fc8dade7811d8a3b2fc666930de1fb831c7faf8bef4c701ec9d79b8f0e7e1085f694ae033
+  data.tar.gz: bbdefb263e00eadc4b27aab602136a9222993b2868f9207c5a4612862483d47f6cbfc4c95c1655ffc8bcd8dffdab7b6bc793607dbe1e218af36ce23ea1356ca7

data/.gitignore

@@ -0,0 +1,21 @@
+coverage
+rdoc
+doc
+.yardoc
+.bundle
+pkg
+.DS_Store
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+.redcar
+*.rbc
+.rvmrc
+vendor
+.idea
+.ruby-version
+.ruby-gemset
+vendor_bundle

data/Gemfile

@@ -1,16 +1,4 @@
-source 'http://rubygems.org'
+source 'https://rubygems.org'
 
-gem 'typhoeus', '~> 0.4.2'
-gem 'nokogiri'
-
-group :development do
-  gem 'rspec', '~> 2.8.0'
-  gem 'bundler', '~> 1.3.0'
-  gem 'jeweler', '~> 1.8.0'
-  gem 'simplecov', '>= 0'
-  gem 'rdoc', '>= 3.9.4'
-  gem 'guard', '~> 1.2.3'
-  gem 'guard-rspec', '~> 1.1.0'
-  gem 'growl'
-  platform(:ruby) { gem 'debugger' }
-end
+# Specify your gem's dependencies in workers.gemspec
+gemspec

data/Gemfile.lock

@@ -1,73 +1,22 @@
+PATH
+  remote: .
+  specs:
+    market_bot (0.11.0)
+      nokogiri
+      typhoeus (>= 0.6.0)
+
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
-    addressable (2.3.5)
-    builder (3.2.2)
-    columnize (0.3.6)
-    debugger (1.6.1)
-      columnize (>= 0.3.1)
-      debugger-linecache (~> 1.2.0)
-      debugger-ruby_core_source (~> 1.2.3)
-    debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.2.3)
     diff-lcs (1.1.3)
-    faraday (0.8.7)
-      multipart-post (~> 1.1)
-    ffi (1.9.0)
-    git (1.2.5)
-    github_api (0.10.1)
-      addressable
-      faraday (~> 0.8.1)
-      hashie (>= 1.2)
-      multi_json (~> 1.4)
-      nokogiri (~> 1.5.2)
-      oauth2
-    growl (1.0.3)
-    guard (1.2.3)
-      listen (>= 0.4.2)
-      thor (>= 0.14.6)
-    guard-rspec (1.1.0)
-      guard (>= 1.1)
-    hashie (2.0.5)
-    highline (1.6.19)
-    httpauth (0.2.0)
-    jeweler (1.8.6)
-      builder
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      github_api (= 0.10.1)
-      highline (>= 1.6.15)
-      nokogiri (= 1.5.10)
-      rake
-      rdoc
-    json (1.8.0)
-    jwt (0.1.8)
-      multi_json (>= 1.5)
-    listen (1.2.2)
-      rb-fsevent (>= 0.9.3)
-      rb-inotify (>= 0.9)
-      rb-kqueue (>= 0.2)
-    mime-types (1.23)
-    multi_json (1.7.7)
-    multi_xml (0.5.4)
-    multipart-post (1.2.0)
-    nokogiri (1.5.10)
-    oauth2 (0.9.2)
-      faraday (~> 0.8)
-      httpauth (~> 0.2)
-      jwt (~> 0.1.4)
-      multi_json (~> 1.0)
-      multi_xml (~> 0.5)
-      rack (~> 1.2)
-    rack (1.5.2)
-    rake (10.1.0)
-    rb-fsevent (0.9.3)
-    rb-inotify (0.9.0)
-      ffi (>= 0.5.0)
-    rb-kqueue (0.2.0)
-      ffi (>= 0.5.0)
-    rdoc (4.0.1)
-      json (~> 1.4)
+    ethon (0.6.1)
+      ffi (>= 1.3.0)
+      mime-types (~> 1.18)
+    ffi (1.9.3)
+    mime-types (1.25.1)
+    mini_portile (0.5.2)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
     rspec (2.8.0)
       rspec-core (~> 2.8.0)
       rspec-expectations (~> 2.8.0)
@@ -76,27 +25,12 @@ GEM
     rspec-expectations (2.8.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.8.0)
-    simplecov (0.7.1)
-      multi_json (~> 1.0)
-      simplecov-html (~> 0.7.1)
-    simplecov-html (0.7.1)
-    thor (0.18.1)
-    typhoeus (0.4.2)
-      ffi (~> 1.0)
-      mime-types (~> 1.18)
+    typhoeus (0.6.6)
+      ethon (~> 0.6.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.3.0)
-  debugger
-  growl
-  guard (~> 1.2.3)
-  guard-rspec (~> 1.1.0)
-  jeweler (~> 1.8.0)
-  nokogiri
-  rdoc (>= 3.9.4)
-  rspec (~> 2.8.0)
-  simplecov
-  typhoeus (~> 0.4.2)
+  market_bot!
+  rspec (= 2.8.0)

data/README.markdown

@@ -8,7 +8,7 @@ Used in production to power [www.droidmeter.com](http://www.droidmeter.com/?t=gi
 
 **This project is currently seeking developers to help maintain it.
 Please send pull requests or contact me if you are able to help out.
-The app scraper is known to work, but leader board functionality is currently broken.**
+The app, leaderboard, and app search features are known to work, but the developer scraper is currently broken.**
 
 ## Dependencies
 

data/Rakefile

@@ -1,29 +1,4 @@
-# encoding: utf-8
-
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "market_bot"
-  gem.homepage = "http://github.com/chadrem/market_bot"
-  gem.license = "MIT"
-  gem.summary = %Q{Market Bot: High performance Ruby scraper for Google's Android Market}
-  gem.description = %Q{Market Bot is a high performance Ruby scraper for Google's Android Market with a simple to use API.  It is built on top of Nokogiri and Typhoeus.}
-  gem.email = "chad@remesch.com"
-  gem.authors = ["Chad Remesch"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
+require 'bundler/gem_tasks'
 
 require 'rspec/core'
 require 'rspec/core/rake_task'
@@ -33,17 +8,17 @@ end
 
 task :default => :spec
 
-require 'rdoc/task'
-RDoc::Task.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+desc 'Start an IRB console with Workers loaded'
+task :console do
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
 
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "market_bot #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+  require 'market_bot'
+  require 'irb'
 
-require 'typhoeus'
+  ARGV.clear
+
+  IRB.start
+end
 
 namespace :spec do
   namespace :data do
@@ -63,19 +38,19 @@ namespace :spec do
       download('https://play.google.com/store/apps/details?id=com.evernote',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'app_2.txt'))
 
-      download('https://play.google.com/store/apps/details?id=kooistar.solutions.Thermometer',
+      download('https://play.google.com/store/apps/details?id=com.mg.android',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'app_3.txt'))
 
-      download('https://play.google.com/store/apps/collection/topselling_paid',
+      download('https://play.google.com/store/apps/category/ARCADE/collection/topselling_paid?num=24',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page1.txt'))
 
-      download('https://play.google.com/store/apps/collection/topselling_paid?start=24&num=24',
+      download('https://play.google.com/store/apps/category/ARCADE/collection/topselling_paid?start=24&num=24',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page2.txt'))
 
-      download('https://play.google.com/store/apps/collection/topselling_paid?start=48&num=24',
+      download('https://play.google.com/store/apps/category/ARCADE/collection/topselling_paid?start=48&num=24',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page3.txt'))
 
-      download('https://play.google.com/store/apps/collection/topselling_paid?start=456&num=24',
+      download('https://play.google.com/store/apps/category/ARCADE/collection/topselling_paid?start=96&num=24',
                File.join(File.dirname(__FILE__), 'spec', 'market_bot', 'android', 'data', 'leaderboard-apps_topselling_paid-page4.txt'))
 
       download('https://play.google.com/store/apps/collection/editors_choice',

data/lib/market_bot.rb

@@ -3,6 +3,7 @@ require 'uri'
 require 'typhoeus'
 require 'nokogiri'
 
+require 'market_bot/version'
 require 'market_bot/android/app'
 require 'market_bot/android/leaderboard/constants'
 require 'market_bot/android/leaderboard'

data/lib/market_bot/android/app.rb

@@ -95,7 +95,7 @@ module MarketBot
         end
 
         result[:youtube_video_ids] = []
-        doc.css('.play-click-target').each do |node|
+        doc.css('.play-action-container').each do |node|
           url = node['data-video-url']
           unless url.nil?
             result[:youtube_video_ids] << url.split('embed/').last.split('?').first
@@ -119,7 +119,7 @@ module MarketBot
         cur_index = 5
         %w(five four three two one).each do |slot|
           node = histogram.css(".#{slot.to_s}")
-          result[:rating_distribution][cur_index] = node.css('.bar-number').text.to_i
+          result[:rating_distribution][cur_index] = node.css('.bar-number').text.gsub(/,/,'').to_i
           cur_index -= 1
 
         end

data/lib/market_bot/android/leaderboard.rb

@@ -5,6 +5,9 @@ module MarketBot
       attr_reader :identifier, :category
       attr_reader :hydra
 
+      MAX_STARS = 5
+      PERCENT_DENOM = 100
+
       def self.parse(html)
         if html.include?('<title>Editor&#39;s Choice')
           parse_editors_choice_page(html)
@@ -17,25 +20,26 @@ module MarketBot
         results = []
         doc = Nokogiri::HTML(html)
 
-        doc.css('.snippet').each do |snippet_node|
+        doc.css('.card').each do |snippet_node|
           result = {}
 
           details_node = snippet_node.css('.details')
 
-          unless snippet_node.css('.ratings').empty?
-            stars_text = snippet_node.css('.ratings').first.attributes['title'].value
-            result[:stars] = /Rating: (.+) stars .*/.match(stars_text)[1]
+          unless snippet_node.css('.current-rating').empty?
+            stars_style = snippet_node.css('.current-rating').first.attributes['style'].value
+            stars_width_percent = stars_style[/width:\s+([0-9.]+)%/, 1].to_f
+            result[:stars] = (MAX_STARS * stars_width_percent/PERCENT_DENOM).round(1).to_s
           else
             result[:stars] = nil
           end
 
           result[:title] = details_node.css('.title').first.attributes['title'].to_s
 
-          if (price_elem = details_node.css('.buy-button-price').children.first)
-            result[:price_usd] = price_elem.text.gsub(' Buy', '')
+          if (price_elem = details_node.css('.buy span').first)
+            result[:price_usd] = price_elem.text
           end
 
-          result[:developer] = details_node.css('.attribution').children.first.text
+          result[:developer] = details_node.css('.subtitle').first.attributes['title'].to_s
           result[:market_id] = details_node.css('.title').first.attributes['href'].to_s.gsub('/store/apps/details?id=', '').gsub(/&feature=.*$/, '')
           result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
 

data/lib/market_bot/version.rb

@@ -0,0 +1,3 @@
+module MarketBot
+  VERSION = '0.11.0'
+end

data/market_bot.gemspec

@@ -1,99 +1,23 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'market_bot/version'
 
-Gem::Specification.new do |s|
-  s.name = "market_bot"
-  s.version = "0.10.0"
+Gem::Specification.new do |gem|
+  gem.name          = 'market_bot'
+  gem.version       = MarketBot::VERSION
+  gem.authors       = ['Chad Remesch']
+  gem.email         = ['chad@remesch.com']
+  gem.description   = %q{Market Bot is a high performance Ruby scraper for Google's Android Market with a simple to use API.  It is built on top of Nokogiri and Typhoeus.}
+  gem.summary       = %q{Market Bot: High performance Ruby scraper for Google's Android Market}
+  gem.homepage      = 'https://github.com/chadrem/market_bot'
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Chad Remesch"]
-  s.date = "2013-07-19"
-  s.description = "Market Bot is a high performance Ruby scraper for Google's Android Market with a simple to use API.  It is built on top of Nokogiri and Typhoeus."
-  s.email = "chad@remesch.com"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.markdown"
-  ]
-  s.files = [
-    ".document",
-    ".rspec",
-    "Gemfile",
-    "Gemfile.lock",
-    "Guardfile",
-    "LICENSE.txt",
-    "NOTES.txt",
-    "README.markdown",
-    "Rakefile",
-    "VERSION",
-    "lib/market_bot.rb",
-    "lib/market_bot/android/app.rb",
-    "lib/market_bot/android/developer.rb",
-    "lib/market_bot/android/leaderboard.rb",
-    "lib/market_bot/android/leaderboard/constants.rb",
-    "lib/market_bot/android/search_query.rb",
-    "market_bot.gemspec",
-    "spec/market_bot/android/app_spec.rb",
-    "spec/market_bot/android/data/app_1.txt",
-    "spec/market_bot/android/data/app_2.txt",
-    "spec/market_bot/android/data/app_3.txt",
-    "spec/market_bot/android/data/leaderboard-apps_editors_choice.txt",
-    "spec/market_bot/android/data/leaderboard-apps_topselling_paid-page1.txt",
-    "spec/market_bot/android/data/leaderboard-apps_topselling_paid-page2.txt",
-    "spec/market_bot/android/data/leaderboard-apps_topselling_paid-page3.txt",
-    "spec/market_bot/android/data/leaderboard-apps_topselling_paid-page4.txt",
-    "spec/market_bot/android/leaderboard_spec.rb",
-    "spec/market_bot/android/search_query_spec.rb",
-    "spec/market_bot_spec.rb",
-    "spec/spec_helper.rb"
-  ]
-  s.homepage = "http://github.com/chadrem/market_bot"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.3"
-  s.summary = "Market Bot: High performance Ruby scraper for Google's Android Market"
+  gem.add_dependency('typhoeus', '>= 0.6.0')
+  gem.add_dependency('nokogiri', '>= 0')
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<typhoeus>, ["~> 0.4.2"])
-      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.3.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.8.0"])
-      s.add_development_dependency(%q<simplecov>, [">= 0"])
-      s.add_development_dependency(%q<rdoc>, [">= 3.9.4"])
-      s.add_development_dependency(%q<guard>, ["~> 1.2.3"])
-      s.add_development_dependency(%q<guard-rspec>, ["~> 1.1.0"])
-      s.add_development_dependency(%q<growl>, [">= 0"])
-      s.add_development_dependency(%q<debugger>, [">= 0"])
-    else
-      s.add_dependency(%q<typhoeus>, ["~> 0.4.2"])
-      s.add_dependency(%q<nokogiri>, [">= 0"])
-      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_dependency(%q<bundler>, ["~> 1.3.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.8.0"])
-      s.add_dependency(%q<simplecov>, [">= 0"])
-      s.add_dependency(%q<rdoc>, [">= 3.9.4"])
-      s.add_dependency(%q<guard>, ["~> 1.2.3"])
-      s.add_dependency(%q<guard-rspec>, ["~> 1.1.0"])
-      s.add_dependency(%q<growl>, [">= 0"])
-      s.add_dependency(%q<debugger>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<typhoeus>, ["~> 0.4.2"])
-    s.add_dependency(%q<nokogiri>, [">= 0"])
-    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-    s.add_dependency(%q<bundler>, ["~> 1.3.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.8.0"])
-    s.add_dependency(%q<simplecov>, [">= 0"])
-    s.add_dependency(%q<rdoc>, [">= 3.9.4"])
-    s.add_dependency(%q<guard>, ["~> 1.2.3"])
-    s.add_dependency(%q<guard-rspec>, ["~> 1.1.0"])
-    s.add_dependency(%q<growl>, [">= 0"])
-    s.add_dependency(%q<debugger>, [">= 0"])
-  end
+  gem.add_development_dependency('rspec', '= 2.8.0')
 end
-