scraper_google_play 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f22341e7b048e1b93074bdca9e6828d6d2c9dfc
-  data.tar.gz: bba6530ccbfd6ee2104225da3f458fab38b4a570
+  metadata.gz: cd442c04dbbd5e5472150aee957fc015bf4d6eb2
+  data.tar.gz: 3f2e546053bc811f4e63fc762d5e8cab3f640e30
 SHA512:
-  metadata.gz: 63fd3c0c9696c7b0e3b2338ca4aca6ba96f97d090d3eb0c92c5a6c45e387ff031cd627ac1583f91a313c49a9da991778d441ae89c9050995464e2436aab2e731
-  data.tar.gz: 2626d31452f687fc2a4b9e9bf17108daddc681ab0669f47ef3e53fb3de86d262ffd9ac59cf2c660f279e414f3bdb52e192106b48b0e8aa4634def26caf02efd9
+  metadata.gz: 3c89b2283a1f5c11c4c7dc80de6c7538305b1eaf0a989fc14cddd70f2ca6036b459954d12d391f588c6b445b03567240e439906dc7694d01194f5bfd09eba7da
+  data.tar.gz: 779da175f04c9518bed67c673ad7972781946381e0194521c83ebfb0e0bc3ccb1ef219f6f87d2da21f8398c5d0a9eb643f6d8afd123da942ae1b06b340597c24

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.gem

data/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
 
+gem 'mechanize'
 # Specify your gem's dependencies in scraper_google_play.gemspec
 gemspec

data/lib/scraper_google_play.rb

@@ -1,4 +1,9 @@
-require "scraper_google_play/version"
+require 'mechanize' # and any other gems you need
+require 'active_support/all'
+require 'scraper_google_play/base'
+require 'scraper_google_play/app'
+require 'scraper_google_play/category'
+require 'scraper_google_play/search'
 
 module ScraperGooglePlay
   def self.lorem

data/lib/scraper_google_play/app.rb

@@ -0,0 +1,83 @@
+class ScraperGooglePlay::App < ScraperGooglePlay::Base
+  attr_accessor :package_name, :app_name, :screen_shots, :developer, :address, :email, :title,
+  							:genre_text, :genre_id, :version, :description, :android_version, :android_version_text,
+ 								:size, :vide, :score
+
+  def initialize(package_name, options = {})
+  	
+  	@package_name = package_name
+  	@agent = Mechanize.new
+		@query = {
+			id: package_name,
+		}.merge(options).to_query
+
+		unless exist?("#{ScraperGooglePlay::APPURL}?#{@query}")
+			@page = nil
+		else 
+			@page = @agent.get("#{ScraperGooglePlay::APPURL}?#{@query}")
+		end
+  end
+
+  def info
+  	return {} unless @page
+  	detailsInfo = @page.at('.details-info')
+		additionalInfo = @page.search('.details-section-contents')
+
+		mainGenre = detailsInfo.at('.category')
+		
+		android_version_text = @page.at('.details-section-contents div.content[itemprop="operatingSystems"]').text().strip()
+		@app_info = {
+			name: name,
+			logo: logo,
+			screen_shots: screen_shots,
+			developer: detailsInfo.at('span[itemprop="name"]').text,
+			address: @page.at('.content.physical-address').try(:text),
+			email: @page.at('.content.contains-text-link .dev-link[href^="mailto:"]') ? @page.at('.content.contains-text-link .dev-link[href^="mailto:"]').attr('href').split(':')[-1] : nil , 
+			title: detailsInfo.at('.document-title').text.strip,
+			genre_text: mainGenre.text().strip,
+			genre_id: mainGenre.attr('href').split('/')[4],
+			version: @page.at('.details-section-contents div.content[itemprop="softwareVersion"]').text().strip(),
+			description: @page.at('.details-section-contents div[itemprop=description] div').text().gsub(/<\/p>/, '\n</p>'),
+			android_version_text: android_version_text,
+			android_version: normalizeAndroidVersion(android_version_text),
+			contentRating: @page.at('.details-section-contents div.content[itemprop="contentRating"]').text().strip(),
+			size: @page.at('.details-section-contents div.content[itemprop="fileSize"]').try(:text),
+			video: @page.at('.screenshots span.preview-overlay-container[data-video-url]') ? @page.at('.screenshots span.preview-overlay-container[data-video-url]').attr('data-video-url') : nil,
+			score: @page.at('.rating-box div.score').text().to_s.gsub(',', '.'),
+
+		}		
+  end
+
+  def similar_app
+  	return nil unless @page
+  	apps = []
+  	recommendation = @page.at(".recommendation")
+  	recommendation.search(".cards id-card-list .card.apps").each do |app|
+  		apps <<  app.attr('data-docid')
+  	end
+  	apps
+  end
+
+  private
+  
+  def normalizeAndroidVersion (androidVersionText) 
+	  matches = androidVersionText.match(/^([0-9\.]+)[^0-9\.].+/)
+
+	  if (!matches) 
+	    return 'VARY'
+	  end
+	  return matches[1];
+	end
+
+  def name
+  	elem = @page.search(".id-app-title")
+		elem ? elem.first.text : ""
+  end
+
+  def logo
+		elem = @page.search(".main-content .apps img.cover-image")
+		elem ? "http:#{elem.first.attributes["src"].text}" : ""
+  end
+
+
+end

data/lib/scraper_google_play/base.rb

@@ -0,0 +1,24 @@
+class ScraperGooglePlay::Base
+
+  def exist?(url)
+  	uri = URI("#{url}")
+		status_code = Net::HTTP.get_response(uri).code
+		case status_code
+		when '200'
+			return true
+		when '404'
+			return false
+		else
+			return false
+		end
+  end
+
+   def list_packname_with_page(page)
+  	return nil if page.nil?
+  	apps = []
+  	page.search('.id-card-list.card-list .card.apps').each do |elm|
+  		apps << elm.attr('data-docid')
+  	end
+  	apps  	
+  end
+end

data/lib/scraper_google_play/category.rb

@@ -0,0 +1,52 @@
+class ScraperGooglePlay::Category < ScraperGooglePlay::Base
+
+  def initialize(category_name)
+  	@category_name = category_name.upcase
+  	@agent = Mechanize.new
+
+		unless exist?("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}")
+			@page = nil
+		else 
+			@page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}")
+		end
+  end
+
+  def all_app
+  	return nil unless @page
+  	list_packname_with_page(@page)
+  end
+
+  def topselling_paid
+  	return nil unless @page
+  	sub_page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}/collection/topselling_paid")
+  	list_packname_with_page(sub_page)
+  end
+
+  def topselling_free
+  	return nil unless @page
+  	sub_page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}/collection/topselling_free")
+  	list_packname_with_page(sub_page)
+  end
+
+  def topgrossing
+  	return nil unless @page
+  	sub_page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}/collection/topgrossing")
+  	list_packname_with_page(sub_page)
+  end
+
+  def topselling_new_paid
+  	return nil unless @page
+  	sub_page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}/collection/topselling_new_paid")
+  	list_packname_with_page(sub_page)
+  end
+
+  def topselling_new_free
+  	return nil unless @page
+  	sub_page = @agent.get("#{ScraperGooglePlay::CATEGORYURL}#{@category_name}/collection/topselling_new_free")
+  	list_packname_with_page(sub_page)
+  end
+  
+  private
+
+
+end

data/lib/scraper_google_play/search.rb

@@ -0,0 +1,18 @@
+class ScraperGooglePlay::Search < ScraperGooglePlay::Base
+
+  def initialize(keyword)
+  	@agent = Mechanize.new
+  	@query = {
+  			c: "apps",
+  			q: keyword
+  		}.to_query
+			@page = @agent.get("#{ScraperGooglePlay::SEARCHURL}?#{@query}")
+  end
+
+  def apps
+  	return nil unless @page
+  	list_packname_with_page(@page)
+  end
+
+
+end

data/lib/scraper_google_play/version.rb

@@ -1,3 +1,6 @@
 module ScraperGooglePlay
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
+  APPURL = "https://play.google.com/store/apps/details"
+  CATEGORYURL = "https://play.google.com/store/apps/category/"
+  SEARCHURL = "https://play.google.com/store/search"
 end

data/scraper_google_play.gemspec

@@ -31,4 +31,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "activesupport", "~> 4.2.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_google_play
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Nguyen Bach
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-30 00:00:00.000000000 Z
+date: 2017-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.5
 description: Write a longer description or delete this line.
 email:
 - trannguyenbach1992@gmail.com
@@ -52,6 +80,10 @@ files:
 - bin/console
 - bin/setup
 - lib/scraper_google_play.rb
+- lib/scraper_google_play/app.rb
+- lib/scraper_google_play/base.rb
+- lib/scraper_google_play/category.rb
+- lib/scraper_google_play/search.rb
 - lib/scraper_google_play/version.rb
 - scraper_google_play-0.1.0.gem
 - scraper_google_play.gemspec