dmg_android_api 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f00475c072f0b51f86bf9a47fabccbdde1aeed92
+  data.tar.gz: 9227bd5ee02738b4019c7dc2b0af8bff85c5332e
+SHA512:
+  metadata.gz: 3787fe315a5766ea665e39d365ace862f6af3e5010ef1e474910b1bd3bfb11186be5489841856425799c6a9960220acf03a44e3865da27c3957f4c2019ac03f9
+  data.tar.gz: a9ee189e59a4a8b0ebc971a63a08c19144598dcd946db6ed58bfee4969dc8a50a7a22b993372b144fa45a9cdd95ad415a651bc41bc1ff6722fc8f74ef1306549

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,15 @@
+source 'http://rubygems.org'
+
+gem 'typhoeus'
+gem 'nokogiri'
+
+group :development do
+  gem 'rspec', '~> 2.8.0'
+  gem 'bundler', '~> 1.3.5'
+  gem 'jeweler', '~> 1.6.4'
+  gem 'simplecov', '>= 0'
+  gem 'rdoc', '>= 3.9.4'
+  gem 'guard', '~> 1.2.3'
+  gem 'guard-rspec', '~> 1.1.0'
+  gem 'growl'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Nhat Anh Nguyen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# DmgAndroidApi
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'dmg_android_api'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dmg_android_api
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/dmg_android_api.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'dmg_android_api/version'
+
+
+Gem::Specification.new do |spec|
+  spec.name          = "dmg_android_api"
+  spec.version       = DmgAndroidApi::VERSION
+  spec.authors       = ["Nhat Anh Nguyen"]
+  spec.email         = ["n.nguyen@thisisdmg.com"]
+  spec.description   = %q{DMG Android Api}
+  spec.summary       = %q{DMG Android Api}
+  spec.homepage      = "http://www.thisisdmg.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/dmg_android_api/android/app.rb

@@ -0,0 +1,221 @@
+module DmgAndroidApi
+  module Android
+
+    class App
+      MARKET_ATTRIBUTES = [:title, :rating, :updated, :current_version, :requires_android,
+                          :category, :installs, :size, :price, :content_rating, :description,
+                          :votes, :developer, :more_from_developer, :users_also_installed,
+                          :related, :banner_icon_url, :banner_image_url, :website_url, :email,
+                          :youtube_video_ids, :screenshot_urls, :whats_new, :permissions,
+                          :rating_distribution, :html]
+
+      attr_reader :app_id
+      attr_reader *MARKET_ATTRIBUTES
+      attr_reader :hydra
+      attr_reader :callback
+      attr_reader :error
+
+      def self.parse(html)
+        result = {}
+
+        doc = Nokogiri::HTML(html)
+
+        extra_info = doc.css('.meta-info')
+
+        extra_info.each do |info|
+
+          field_name  = info.css('.title').text
+
+          case field_name
+          when 'Updated'
+            result[:updated] = info.css('.content').text
+          when 'Current Version'
+            result[:current_version] = info.css('.content').text
+          when 'Requires Android'
+            result[:requires_android] = info.css('.content').text
+          when 'Category'
+            result[:category] = info.css('.content').text
+          when 'Installs'
+            result[:installs] = info.css('.content').text
+          when 'Size'
+            result[:size] = info.css('.content').text
+          when 'Content Rating'
+            result[:content_rating] = info.css('.content').text
+          end
+        end
+        result[:description] = doc.css('.text-body div').first.inner_html
+
+        result[:title] = doc.css('.document-title').first.text
+
+        rating_elem = doc.css('.score')
+        result[:rating] = rating_elem.text unless rating_elem.empty?
+
+        votes_elem = doc.css('.reviews-num')
+        result[:votes] = votes_elem.text unless votes_elem.empty?
+
+        result[:developer] = doc.css('.document-subtitle.primary').text
+
+        result[:more_from_developer] = []
+        result[:users_also_installed] = []
+        result[:related] = []
+
+        if similar_elem = doc.css('.doc-similar').first
+          similar_elem.children.each do |similar_elem_child|
+            assoc_app_type = similar_elem_child.attributes['data-analyticsid'].text
+
+            next unless %w(more-from-developer users-also-installed related).include?(assoc_app_type)
+
+            assoc_app_type = assoc_app_type.gsub('-', '_').to_sym
+            result[assoc_app_type] ||= []
+
+            similar_elem_child.css('.app-left-column-related-snippet-container').each do |app_elem|
+              assoc_app = {}
+
+              assoc_app[:app_id] = app_elem.attributes['data-docid'].text
+
+              result[assoc_app_type] << assoc_app
+            end
+          end
+        end
+
+        result[:banner_icon_url] = doc.css('.cover-container img').first.attributes['src'].value
+
+        if image_elem = doc.css('.doc-banner-image-container img').first
+          result[:banner_image_url] = image_elem.attributes['src'].value
+        else
+          result[:banner_image_url] = nil
+        end
+
+        if website_elem = doc.css('a').select{ |l| l.text.include?("Visit Developer's Website")}.first
+          redirect_url = website_elem.attribute('href').value
+
+          if q_param = URI(redirect_url).query.split('&').select{ |p| p =~ /q=/ }.first
+            actual_url = q_param.gsub('q=', '')
+          end
+
+          result[:website_url] = actual_url
+        end
+
+        if email_elem = doc.css('a').select{ |l| l.text.include?("Email Developer")}.first
+          result[:email] = email_elem.attribute('href').value.gsub(/^mailto:/, '')
+        end
+
+        unless (video_section_elem = doc.css('.doc-video-section')).empty?
+          urls = video_section_elem.children.css('embed').map{ |e| e.attribute('src').value }
+          result[:youtube_video_ids] = urls.map{ |u| /youtube\.com\/v\/(.*)\?/.match(u)[1] }
+        else
+          result[:youtube_video_ids] = []
+        end
+
+        
+        screenshots = doc.css('.thumbnails img')
+
+        if screenshots && screenshots.length > 0
+          result[:screenshot_urls] = screenshots.map { |s| s.attributes['src'].value }
+        else
+          result[:screenshot_urls] = []
+        end
+
+        recent_changes = doc.css('.recent-change')
+        result[:whats_new] = ''
+        recent_changes.each do |change|
+
+          result[:whats_new] << change.text
+          result[:whats_new] << '<br>'
+
+        end
+        result[:permissions] = permissions = []
+        perm_types = ['dangerous', 'safe']
+        perm_types.each do |type|
+          doc.css("#doc-permissions-#{type} .doc-permission-group").each do |group_elem|
+            title = group_elem.css('.doc-permission-group-title').text
+            group_elem.css('.doc-permission-description').each do |desc_elem|
+              #permissions << { :security => type, :group => title, :description => desc_elem.text }
+            end
+            descriptions = group_elem.css('.doc-permission-description').map { |e| e.text }
+            descriptions_full = group_elem.css('.doc-permission-description-full').map { |e| e.text }
+            (0...descriptions.length).each do |i|
+              desc = descriptions[i]
+              desc_full = descriptions_full[i]
+              permissions << { :security => type, :group => title, :description => desc, :description_full => desc_full }
+            end
+          end
+        end
+
+        result[:rating_distribution] = { 5 => nil, 4 => nil, 3 => nil, 2 => nil, 1 => nil }
+
+        if (histogram = doc.css('div.histogram-table').first)
+          cur_index = 5
+          histogram.css('tr').each do |e|
+            result[:rating_distribution][cur_index] = e.children.last.inner_text.gsub(/[^0-9]/, '').to_i
+            cur_index -= 1
+          end
+        end
+
+        result[:html] = html
+
+        result
+      end
+
+      def initialize(app_id, options={})
+        @app_id = app_id
+        @hydra = options[:hydra] || DmgAndroidApi.hydra
+        @request_opts = options[:request_opts] || {}
+        @callback = nil
+        @error = nil
+      end
+
+      def market_url
+        "https://play.google.com/store/apps/details?id=#{@app_id}&hl=en"
+      end
+
+      def update
+        resp = Typhoeus::Request.get(market_url, @request_opts)
+        result = App.parse(resp.body)
+        update_callback(result)
+
+        self
+      end
+
+      def enqueue_update(&block)
+        @callback = block
+        @error = nil
+
+        request = Typhoeus::Request.new(market_url, @request_opts)
+
+        request.on_complete do |response|
+          # HACK: Typhoeus <= 0.4.2 returns a response, 0.5.0pre returns the request.
+          response = response.response if response.is_a?(Typhoeus::Request)
+
+          result = nil
+
+          begin
+            result = App.parse(response.body)
+          rescue Exception => e
+            @error = e
+          end
+
+          update_callback(result)
+        end
+
+        hydra.queue(request)
+
+        self
+      end
+
+    private
+      def update_callback(result)
+        unless @error
+          MARKET_ATTRIBUTES.each do |a|
+            attr_name = "@#{a}"
+            attr_value = result[a]
+            instance_variable_set(attr_name, attr_value)
+          end
+        end
+
+        @callback.call(self) if @callback
+      end
+    end
+
+  end
+end

data/lib/dmg_android_api/android/developer.rb

@@ -0,0 +1,33 @@
+module DmgAndroidApi
+  module Android
+
+    # Developer pages are extremely similar to leaderboard pages.
+    # Amazingly, this inheritence hack works!
+    class Developer < DmgAndroidApi::Android::Leaderboard
+      def initialze(developer, options={})
+        super(query, nil, options)
+      end
+
+      def market_urls(options={})
+        results = []
+
+        min_page = options[:min_page] || 1
+        max_page = options[:max_page] || 25
+
+        (min_page..max_page).each do |page|
+          start_val = (page - 1) * 12
+
+          url = "https://play.google.com/store/apps/developer?"
+          url << "id=#{URI.escape(identifier)}&"
+          url << "start=#{start_val}&"
+          url << "num=12&hl=en"
+
+          results << url
+        end
+
+        results
+      end
+    end
+
+  end
+end

data/lib/dmg_android_api/android/leaderboard/constants.rb

@@ -0,0 +1,59 @@
+module DmgAndroidApi
+  module Android
+
+    class Leaderboard
+      IDENTIFIERS = [
+        :editors_choice,
+        :featured,
+        :movers_shakers,
+        :tablet_featured,
+        :topgrossing,
+        :topselling_free,
+        :topselling_new_free,
+        :topselling_new_paid,
+        :topselling_paid,
+        :topselling_paid_game
+      ]
+
+      CATEGORIES = [
+        :application,
+        :app_wallpaper,
+        :app_widgets,
+        :arcade,
+        :books_and_reference,
+        :brain,
+        :business,
+        :cards,
+        :casual,
+        :comics,
+        :communication,
+        :education,
+        :entertainment,
+        :finance,
+        :game,
+        :game_wallpaper,
+        :game_widgets,
+        :health_and_fitness,
+        :libraries_and_demo,
+        :lifestyle,
+        :media_and_video,
+        :medical,
+        :music_and_audio,
+        :news_and_magazines,
+        :personalization,
+        :photography,
+        :productivity,
+        :racing,
+        :shopping,
+        :social,
+        :sports,
+        :sports_games,
+        :tools,
+        :transportation,
+        :travel_and_local,
+        :weather
+      ]
+    end
+
+  end
+end

data/lib/dmg_android_api/android/leaderboard.rb

@@ -0,0 +1,153 @@
+module DmgAndroidApi
+  module Android
+
+    class Leaderboard
+      attr_reader :identifier, :category
+      attr_reader :hydra
+
+      def self.parse(html)
+        if html.include?('<title>Editor&#39;s Choice')
+          parse_editors_choice_page(html)
+        else
+          parse_normal_page(html)
+        end
+      end
+
+      def self.parse_normal_page(html)
+        results = []
+        doc = Nokogiri::HTML(html)
+        doc.css('.card').each do |snippet_node|
+          result = {}
+          
+          details_node = snippet_node.css('.details')
+
+
+          unless snippet_node.css('.reason-set').empty?
+
+            ratings_node = snippet_node.css('.reason-set').css('.stars-container')
+            price_node   = snippet_node.css('.reason-set').css('.price-container')
+
+            ratings_season_node = ratings_node.css('.reason-set-star-rating').css('.tiny-star')
+            current_rating      = ratings_season_node.css('.current-rating')
+            result[:stars] = current_rating.text
+            result[:price_usd] = price_node.css('.buy-button-container').css('.buy').text
+          else
+            result[:stars] = nil
+          end
+
+          result[:title] = details_node.css('.title').text
+          result[:developer]  = details_node.css('.subtitle-container').children.first.text
+          result[:market_id]  = snippet_node.attributes['data-docid'].text
+          result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
+
+          result[:price_usd] = '$0.00' if result[:price_usd] == 'Free'
+
+          results << result
+        end
+
+        results
+      end
+
+      def self.parse_editors_choice_page(html)
+        results = []
+
+        doc = Nokogiri::HTML(html)
+
+        doc.css('.fsg-snippet').each do |snippet_node|
+          result = {}
+
+          result[:title]      = snippet_node.css('.title').text
+          result[:price_usd]  = nil
+          result[:developer]  = snippet_node.css('.attribution').text
+          result[:market_id]  = snippet_node.attributes['data-docid'].text
+          result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
+
+          results << result
+        end
+
+        results
+      end
+
+      def initialize(identifier, category=nil, options={})
+        @identifier = identifier
+        @category = category
+        @hydra = options[:hydra] || DmgAndroidApi.hydra
+        @request_opts = options[:request_opts] || {}
+        @parsed_results = []
+      end
+
+      def market_urls(options={})
+        results = []
+
+        min_page = options[:min_page] || 1
+        max_page = options[:max_page] || 10
+
+        (min_page..max_page).each do |page|
+          start_val = (page - 1) * 46
+
+          url = 'https://play.google.com/store/apps'
+          url << "/category/#{category.to_s.upcase}" if category
+          url << "/collection/#{identifier.to_s}?"
+          url << "start=#{start_val}"
+          url << "&num=46&hl=en"
+          results << url
+        end
+
+        results
+      end
+
+      def enqueue_update(options={})
+        if @identifier.to_s.downcase == 'editors_choice' && category == nil
+          url = 'https://play.google.com/store/apps/collection/editors_choice?&hl=en'
+          process_page(url, 1)
+        else
+          min_rank = options[:min_rank] || 1
+          max_rank = options[:max_rank] || 500
+
+          min_page = rank_to_page(min_rank)
+          max_page = rank_to_page(max_rank)
+
+          @parsed_results = []
+
+          urls = market_urls(:min_page => min_page, :max_page => max_page)
+          urls.each_index{ |i| process_page(urls[i], i+1) }
+        end
+
+        self
+      end
+
+      def update(options={})
+        enqueue_update(options)
+        @hydra.run
+
+        self
+      end
+
+      def rank_to_page(rank)
+        ((rank - 1) / 24) + 1
+      end
+
+      def results
+        raise 'Results do not exist yet.' unless @parsed_results
+        @parsed_results.reject{ |page| page.nil? || page.empty? }.flatten
+      end
+
+    private
+      def process_page(url, page_num)
+        request = Typhoeus::Request.new(url, @request_opts)
+        request.on_complete do |response|
+          # HACK: Typhoeus <= 0.4.2 returns a response, 0.5.0pre returns the request.
+          response = response.response if response.is_a?(Typhoeus::Request)
+          result = Leaderboard.parse(response.body)
+          update_callback(result, page_num)
+        end
+        @hydra.queue(request)
+      end
+
+      def update_callback(result, page)
+        @parsed_results[page] = result
+      end
+    end
+
+  end
+end

data/lib/dmg_android_api/android/search_query.rb

@@ -0,0 +1,31 @@
+module DmgAndroidApi
+  module Android
+
+    # Search query pages are extremely similar to leaderboard pages.
+    # Amazingly, this inheritence hack works!
+    class SearchQuery < DmgAndroidApi::Android::Leaderboard
+      def initialze(query, options={})
+        super(query, nil, options)
+      end
+
+      def market_urls(options={})
+        results = []
+        min_page = options[:min_page] || 1
+        max_page = 9
+        (min_page..max_page).each do |page|
+          start_val = (page - 1) * 46
+
+          url = "https://play.google.com/store/search?"
+          url << "q=#{URI.escape(identifier)}&"
+          url << "c=apps&start=#{start_val}&"
+          url << "num=46&hl=en"
+          
+          results << url
+        end
+
+        results
+      end
+    end
+
+  end
+end

data/lib/dmg_android_api/version.rb

@@ -0,0 +1,3 @@
+module DmgAndroidApi
+  VERSION = "0.0.4"
+end

data/lib/dmg_android_api.rb

@@ -0,0 +1,18 @@
+require "dmg_android_api/version"
+
+require 'uri'
+
+require 'typhoeus'
+require 'nokogiri'
+
+require 'dmg_android_api/android/app'
+require 'dmg_android_api/android/leaderboard/constants'
+require 'dmg_android_api/android/leaderboard'
+require 'dmg_android_api/android/search_query'
+require 'dmg_android_api/android/developer'
+
+module DmgAndroidApi
+  def self.hydra
+    @hydra ||= Typhoeus::Hydra.new(:max_concurrency => 5)
+  end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: dmg_android_api
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Nhat Anh Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: DMG Android Api
+email:
+- n.nguyen@thisisdmg.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- dmg_android_api.gemspec
+- lib/dmg_android_api.rb
+- lib/dmg_android_api/android/app.rb
+- lib/dmg_android_api/android/developer.rb
+- lib/dmg_android_api/android/leaderboard.rb
+- lib/dmg_android_api/android/leaderboard/constants.rb
+- lib/dmg_android_api/android/search_query.rb
+- lib/dmg_android_api/version.rb
+homepage: http://www.thisisdmg.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: DMG Android Api
+test_files: []