itunes_app_reviews 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/Gemfile +15 -0
- data/LICENSE.txt +22 -0
- data/README.md +52 -0
- data/Rakefile +1 -0
- data/app_reviews.rb +19 -0
- data/itunes_app_reviews.gemspec +19 -0
- data/lib/app_store_reviews.rb +11 -0
- data/lib/itunes_app_reviews/reviews.rb +42 -0
- data/lib/itunes_app_reviews/reviews_page.rb +60 -0
- data/lib/itunes_app_reviews/stores.rb +83 -0
- data/lib/itunes_app_reviews/version.rb +3 -0
- data/lib/itunes_app_reviews.rb +11 -0
- data/spec/fixtures/reviews_page.xml +3591 -0
- data/spec/fixtures/reviews_page_with_errors.xml +3591 -0
- data/spec/review_page_spec.rb +56 -0
- data/spec/spec_helper.rb +9 -0
- metadata +72 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Grzegorz Biesiadecki
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# ITunesAppReviews
|
2
|
+
|
3
|
+
Download user reviews from iTunes (iOS App Store)
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'app_store_reviews'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install app_store_reviews
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
Create reviews object with selected store ID and application ID.
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
app_id = 123456789
|
25
|
+
store_id = ITunesAppReviews::Stores['United States']
|
26
|
+
|
27
|
+
reviews = ITunesAppReviews::Reviews.new(store_id, app_id)
|
28
|
+
reviews.each do |review|
|
29
|
+
puts "title: #{review[:title]}"
|
30
|
+
puts "name: #{review[:name]}"
|
31
|
+
puts "text: #{review[:text]}"
|
32
|
+
puts "rating: #{review[:rating]}"
|
33
|
+
puts "date: #{review[:date]}"
|
34
|
+
puts "version: #{review[:version]}"
|
35
|
+
puts
|
36
|
+
end
|
37
|
+
```ruby
|
38
|
+
|
39
|
+
For logging provide logger to `ITunesAppReviews` module and set desired logging level. By default logs go to `/dev/null` with level `ERROR`.
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
ITunesAppReviews.log = Logger.new(STDOUT)
|
43
|
+
ITunesAppReviews.log.level = Logger::DEBUG
|
44
|
+
```ruby
|
45
|
+
|
46
|
+
## Contributing
|
47
|
+
|
48
|
+
1. Fork it
|
49
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
50
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
51
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
52
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/app_reviews.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative 'lib/app_store_reviews'
|
3
|
+
puts "Starting\n"
|
4
|
+
|
5
|
+
app_id = 539124565
|
6
|
+
store_id = 143507
|
7
|
+
|
8
|
+
reviews = AppStoreReviews::Reviews.new(store_id, app_id)
|
9
|
+
reviews.each do |review|
|
10
|
+
puts "title: #{review[:title]}"
|
11
|
+
puts "name: #{review[:name]}"
|
12
|
+
puts "text: #{review[:text]}"
|
13
|
+
puts "rating: #{review[:rating]}"
|
14
|
+
puts "date: #{review[:date]}"
|
15
|
+
puts "version: #{review[:version]}"
|
16
|
+
puts
|
17
|
+
end
|
18
|
+
|
19
|
+
puts "Done."
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'itunes_app_reviews/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "itunes_app_reviews"
|
8
|
+
gem.version = ITunesAppReviews::VERSION
|
9
|
+
gem.authors = ["Grzegorz Biesiadecki"]
|
10
|
+
gem.email = ["gbiesiadecki@gmail.com"]
|
11
|
+
gem.description = %q{Scrapes iTunes for app reviews}
|
12
|
+
gem.summary = %q{iTunes App Reviews}
|
13
|
+
gem.homepage = ""
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative 'reviews_page'
|
3
|
+
require 'httpclient'
|
4
|
+
require 'open-uri'
|
5
|
+
require 'open_uri_redirections'
|
6
|
+
require 'csv'
|
7
|
+
require 'date'
|
8
|
+
|
9
|
+
module ITunesAppReviews
|
10
|
+
class Reviews
|
11
|
+
attr_writer :start_date
|
12
|
+
|
13
|
+
def initialize(store_id, app_id)
|
14
|
+
@store_id = store_id
|
15
|
+
@app_id = app_id
|
16
|
+
end
|
17
|
+
|
18
|
+
def each
|
19
|
+
ITunesAppReviews::log.debug("Loading reviews (store_id: #{@store_id}, app_id: #{@app_id})")
|
20
|
+
page = 0
|
21
|
+
while true
|
22
|
+
page += 1
|
23
|
+
reviews = reviews_page(page)
|
24
|
+
|
25
|
+
break if reviews.empty?
|
26
|
+
|
27
|
+
reviews.each do |review|
|
28
|
+
yield review
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def reviews_page(page_number)
|
34
|
+
url = "http://phobos.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?sortOrdering=4&onlyLatestVersion=false&sortAscending=true&pageNumber=#{(page_number - 1)}&type=Purple+Software&id=#{@app_id}"
|
35
|
+
f = open(url, "User-Agent" => "iTunes-iPhone/2.2 (2)", "X-Apple-Store-Front" => "#{@store_id}-1", :allow_safe_redirections => true)
|
36
|
+
page_text = f.read
|
37
|
+
ITunesAppReviews.log.debug("Loading page #{page_number} (store_id: #{@store_id}, app_id: #{@app_id})")
|
38
|
+
ITunesAppReviews.log.debug(page_text)
|
39
|
+
ITunesAppReviews::ReviewsPage.new(page_text).reviews
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'rexml/document'
|
4
|
+
include REXML
|
5
|
+
|
6
|
+
module ITunesAppReviews
|
7
|
+
class ReviewsPage
|
8
|
+
LINK_SELECTOR = "Document > View > ScrollView > VBoxView > View > MatrixView > VBoxView > VBoxView > VBoxView"
|
9
|
+
|
10
|
+
def initialize(page_content)
|
11
|
+
doc = Nokogiri::XML(page_content)
|
12
|
+
@links = doc.css(LINK_SELECTOR)
|
13
|
+
end
|
14
|
+
|
15
|
+
def reviews
|
16
|
+
@links.reduce([]) do |reviews, current|
|
17
|
+
review = parse(current)
|
18
|
+
reviews << review if review
|
19
|
+
reviews
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def parse(link)
|
26
|
+
ITunesAppReviews.log.debug("Parsing link")
|
27
|
+
ITunesAppReviews.log.debug(link)
|
28
|
+
|
29
|
+
node = link.css('TextView > SetFontStyle')
|
30
|
+
info = node[2].content.split(" - ")
|
31
|
+
version = info[1].strip()[-5, 5]
|
32
|
+
date = info[2].strip()
|
33
|
+
|
34
|
+
name_el = node.css('GotoURL > b')
|
35
|
+
name = name_el.first.content.strip
|
36
|
+
|
37
|
+
title_el = node[0].css('b')
|
38
|
+
title = title_el.first.content
|
39
|
+
|
40
|
+
text = node[3].content.strip.gsub(/(\s){2,}/, " ")
|
41
|
+
|
42
|
+
node = link.css('HBoxView > HBoxView > HBoxView')
|
43
|
+
rating = node.attr('alt').value.delete(" Stars")
|
44
|
+
|
45
|
+
{
|
46
|
+
title: title,
|
47
|
+
name: name,
|
48
|
+
text: text,
|
49
|
+
rating: rating,
|
50
|
+
date: date,
|
51
|
+
version: version
|
52
|
+
}
|
53
|
+
|
54
|
+
rescue StandardError => e
|
55
|
+
ITunesAppReviews.log.error("Error while parsing")
|
56
|
+
ITunesAppReviews.log.error(e)
|
57
|
+
nil
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module ITunesAppReviews
|
4
|
+
STORES = {
|
5
|
+
'Argentina' => 143505,
|
6
|
+
'Australia' => 143460,
|
7
|
+
'Belgium' => 143446,
|
8
|
+
'Brazil' => 143503,
|
9
|
+
'Canada' => 143455,
|
10
|
+
'Chile' => 143483,
|
11
|
+
'China' => 143465,
|
12
|
+
'Colombia' => 143501,
|
13
|
+
'Costa Rica' => 143495,
|
14
|
+
'Croatia' => 143494,
|
15
|
+
'Czech Republic' => 143489,
|
16
|
+
'Denmark' => 143458,
|
17
|
+
'Deutschland' => 143443,
|
18
|
+
'El Salvador' => 143506,
|
19
|
+
'Espana' => 143454,
|
20
|
+
'Finland' => 143447,
|
21
|
+
'France' => 143442,
|
22
|
+
'Greece' => 143448,
|
23
|
+
'Guatemala' => 143504,
|
24
|
+
'Hong Kong' => 143463,
|
25
|
+
'Hungary' => 143482,
|
26
|
+
'India' => 143467,
|
27
|
+
'Indonesia' => 143476,
|
28
|
+
'Ireland' => 143449,
|
29
|
+
'Israel' => 143491,
|
30
|
+
'Italia' => 143450,
|
31
|
+
'Korea' => 143466,
|
32
|
+
'Kuwait' => 143493,
|
33
|
+
'Lebanon' => 143497,
|
34
|
+
'Luxembourg' => 143451,
|
35
|
+
'Malaysia' => 143473,
|
36
|
+
'Mexico' => 143468,
|
37
|
+
'Nederland' => 143452,
|
38
|
+
'New Zealand' => 143461,
|
39
|
+
'Norway' => 143457,
|
40
|
+
'Osterreich' => 143445,
|
41
|
+
'Pakistan' => 143477,
|
42
|
+
'Panama' => 143485,
|
43
|
+
'Peru' => 143507,
|
44
|
+
'Phillipines' => 143474,
|
45
|
+
'Poland' => 143478,
|
46
|
+
'Portugal' => 143453,
|
47
|
+
'Qatar' => 143498,
|
48
|
+
'Romania' => 143487,
|
49
|
+
'Russia' => 143469,
|
50
|
+
'Saudi Arabia' => 143479,
|
51
|
+
'Schweiz/Suisse' => 143459,
|
52
|
+
'Singapore' => 143464,
|
53
|
+
'Slovakia' => 143496,
|
54
|
+
'Slovenia' => 143499,
|
55
|
+
'South Africa' => 143472,
|
56
|
+
'Sri Lanka' => 143486,
|
57
|
+
'Sweden' => 143456,
|
58
|
+
'Taiwan' => 143470,
|
59
|
+
'Thailand' => 143475,
|
60
|
+
'Turkey' => 143480,
|
61
|
+
'United Arab Emirates' => 143481,
|
62
|
+
'United Kingdom' => 143444,
|
63
|
+
'United States' => 143441,
|
64
|
+
'Venezuela' => 143502,
|
65
|
+
'Vietnam' => 143471,
|
66
|
+
'Japan' => 143462,
|
67
|
+
'Dominican Republic' => 143508,
|
68
|
+
'Ecuador' => 143509,
|
69
|
+
'Egypt' => 143516,
|
70
|
+
'Estonia' => 143518,
|
71
|
+
'Honduras' => 143510,
|
72
|
+
'Jamaica' => 143511,
|
73
|
+
'Kazakhstan' => 143517,
|
74
|
+
'Latvia' => 143519,
|
75
|
+
'Lithuania' => 143520,
|
76
|
+
'Macau' => 143515,
|
77
|
+
'Malta' => 143521,
|
78
|
+
'Moldova' => 143523,
|
79
|
+
'Nicaragua' => 143512,
|
80
|
+
'Paraguay' => 143513,
|
81
|
+
'Uruguay' => 143514
|
82
|
+
}
|
83
|
+
end
|