app-reviews 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in app-reviews.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Mu-ik Jeon
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# AppReviews
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'app-reviews'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install app-reviews
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/app-reviews.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/app-reviews/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Mu-ik Jeon"]
+  gem.email         = ["muikor@gmail.com"]
+  gem.description   = %q{Mobile App Review Crawler}
+  gem.summary       = %q{mobile app review crawler}
+  gem.homepage      = ""
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "app-reviews"
+  gem.require_paths = ["lib"]
+  gem.version       = AppReviews::VERSION
+end

data/bin/app-reviews-crawler

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '../lib/app-reviews-crawler')
+
+AppReviewsCrawler.new.execute ARGV

data/lib/app-reviews/version.rb

@@ -0,0 +1,3 @@
+module AppReviews
+  VERSION = "0.0.1"
+end

data/lib/app-reviews-crawler.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "app-reviews/version"
+require 'app_store_reviews'
+require 'play_store_reviews'
+require 'tstore_reviews'
+require 'active_support/core_ext'
+
+class AppReviewsCrawler
+  def execute(argv)
+    return print_usage unless validate argv
+    store, app_id, from_date_str = argv
+    start_page = 1
+    end_page = 10000
+    from_date = Date.parse(from_date_str)
+    puts "store: #{store}"
+    puts "app_id: #{app_id}"
+    puts "from_date: #{from_date}"
+    puts "page: #{start_page} ~ #{end_page}"
+
+    reviews = create_reviews(store, app_id)
+    return print_usage unless reviews
+    reviews.set_page start_page, end_page
+    reviews.set_from_date from_date
+    require 'yaml'
+    reviews.each do |item|
+      puts item.to_yaml
+    end
+
+    puts "Review Count: #{reviews.count}"
+    puts "Review Last Date: #{reviews.last[:date]}"
+  end
+
+  def create_reviews(store, app_id)
+    case store
+    when 'appstore'
+      AppStoreReviews.new app_id
+    when 'play'
+      PlayStoreReviews.new app_id
+    when 'tstore'
+      TstoreReviews.new app_id
+    end
+  end
+
+  def validate(argv)
+    return false if argv.size < 3
+    true
+  end
+
+  def print_usage
+    puts "USAGE: app-reviews-crawler appstore|play|tstore store_app_id from_date"
+    puts "example# app-reviews-crawler appstore 383844387 2012-03-26"
+    puts "example# app-reviews-crawler play com.thinkreals.pocketstyle2 2012-03-26"
+    puts "example# app-reviews-crawler tstore 0000033534 2012-03-26"
+  end
+end

data/lib/app-reviews.rb

@@ -0,0 +1,2 @@
+module AppReviews
+end

data/lib/app_store_review_page.rb

@@ -0,0 +1,62 @@
+require 'nokogiri'
+require 'rexml/document'
+include REXML
+
+class AppStoreReviewPage
+  def initialize(text, page)
+    @doc = Nokogiri::XML(text)
+    @page = page
+  end
+
+  def items
+    path = "Document > View > ScrollView > VBoxView > View > MatrixView > VBoxView > VBoxView > VBoxView"
+    @doc.css(path).each do |link|
+      begin
+        review = parse link
+        next if review.nil?
+        break unless yield review
+      rescue Exception => e
+        print_parse_error e, link
+      end
+    end
+  end
+
+  def last_page
+    @doc.css("MatrixView > VBoxView > VBoxView > HBoxView:nth-child(2) > TextView > SetFontStyle").each do |link|
+      link.content.split(' ').last.to_i 
+    end
+  end
+
+  private
+  def parse(link)
+    node = link.css('TextView > SetFontStyle')
+    date = node[2].content.gsub("\n", '').strip
+    index = date.rindex('- ')
+    return if index.nil?
+    index += 1
+    date = date[index..-1].strip
+    name_el = node.css('GotoURL > b')
+    title_el = node[0].css('b')
+    name = name_el.first.content.strip
+    title = title_el.first.content
+    text = node[3].content.strip
+    node = link.css('HBoxView > HBoxView > HBoxView')
+    rating = node.attr('alt').value
+
+    {
+      title: title,
+      name: name,
+      text: text,
+      rating: rating,
+      date: date,
+    }
+  end
+
+  def print_parse_error(e, link)
+    puts e
+    puts e.backtrace
+    puts "page: #{@page}"
+    puts link
+  end
+end
+

data/lib/app_store_reviews.rb

@@ -0,0 +1,45 @@
+require 'base_app_reviews'
+require 'app_store_review_page'
+require 'httpclient'
+require 'open-uri'
+
+class AppStoreReviews
+  include BaseAppReviews
+
+  def each
+    unless @list.nil?
+      return @list.each do |item|
+        yield item
+      end
+    end
+
+    @list = []
+    country_codes = [143441, 143466, 143463]
+    country_codes.each do |country_code| 
+      get_reviews(country_code) do |review|
+        @list << review
+        yield review
+      end
+    end
+  end
+
+  private
+  def get_reviews(country)
+    (@start_page..@end_page).each do |page|
+      url = "http://phobos.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?sortOrdering=4&onlyLatestVersion=false&sortAscending=true&pageNumber=#{(page - 1)}&type=Purple+Software&id=#{@app_id}"
+
+      f = open(url,	"User-Agent" => "iTunes-iPhone/2.2 (2)", "X-Apple-Store-Front" => "#{country}-1") 
+#      File.new('review.xml', 'w').puts f.read
+#      exit
+      review_page = AppStoreReviewPage.new f.read, page
+      break unless review_page.items do |item|
+        return false if Date.parse(item[:date]) < @from_date
+        yield item
+        true
+      end
+
+      break if review_page.last_page < page + 1
+    end
+  end
+end
+

data/lib/base_app_reviews.rb

@@ -0,0 +1,30 @@
+module BaseAppReviews
+  include Enumerable
+
+  def initialize(app_id)
+    @app_id = app_id
+    @list = nil
+  end
+
+  def set_last_date(date)
+    @last_date = date
+  end
+
+  def last_date
+    @last_date
+  end
+
+  def set_page(start_page, end_page)
+    @start_page = start_page
+    @end_page = end_page
+  end
+
+  def set_from_date(date)
+    @from_date = date
+  end
+
+  def last
+    @list.last
+  end
+end
+

data/lib/play_store_review_page.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require 'rexml/document'
+include REXML
+
+class PlayStoreReviewPage
+  def initialize(text, page)
+    result = JSON(text[5..-1])
+    html = result['htmlContent'].gsub('<hr>', '<hr />')
+    @xml = Document.new("<body>#{html}</body>")
+    @page = page
+  end
+
+  def items
+    @xml.root.elements.each do |item|
+      next if item.elements.size < 1
+      begin
+        name_el = item.elements['span'].elements['strong']
+        next unless name_el
+        date = item.elements[2, 'span'].text.sub('님이', '').strip
+
+        title = item.elements['div'].elements['h4'].text.strip
+        if item.elements['p']
+          text = item.elements['p'].text 
+        else
+          text = nil
+        end
+        text = text.strip if text
+        name = name_el.text.strip
+        rating = item.elements['div'].elements['div'].attribute('title').value.strip
+        review = {
+          title: title,
+          text: text,
+          name: name,
+          rating: rating,
+          date: date,
+        }
+        break unless yield review
+      rescue Exception => e
+        print_error e, item
+      end
+    end
+  end
+
+  def print_error(e, item)
+    puts e
+    puts e.backtrace
+    puts "page: #{@page}"
+    puts item
+  end
+end
+

data/lib/play_store_reviews.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+require 'httpclient'
+require 'play_store_review_page'
+
+class PlayStoreReviews
+  include BaseAppReviews
+
+  def each
+    unless @list.nil?
+      return @list.each do |item|
+        yield item
+      end
+    end
+
+    @list = []
+    url = "https://play.google.com/store/getreviews"
+    params = {
+      id: @app_id,
+      reviewSortOrder: 0,
+      reviewType: 1,
+      pageNum: 0,
+    }
+
+    (@start_page..@end_page).each do |page|
+      params[:pageNum] = page - 1
+      client = HTTPClient.new
+      content = client.post_content(url, params.to_query)
+      review_page = PlayStoreReviewPage.new content, page
+      break unless review_page.items do |item|
+        return false if Date.strptime(item[:date], '%Y년 %m월 %d일') < @from_date
+        @list << item
+        yield item
+        true
+      end
+    end
+  end
+end
+

data/lib/tstore_review_page.rb

@@ -0,0 +1,40 @@
+require 'rexml/document'
+include REXML
+
+class TstoreReviewPage
+  def initialize(content, page)
+    content = content.gsub(/<textarea[^<]+<\/textarea>/, '')
+    content = content.gsub('gif""', 'gif"')
+    xml = Document.new(content)
+    @table = xml.root.elements['body'].elements[2, 'form'].elements[2, 'div'].elements['table']
+  end
+
+  def items
+    @table.elements.each do |item|
+      begin
+        td = item.elements['td']
+        next unless td
+        date = td.elements['p'].elements['span'].text.strip
+        name = td.elements['p'].elements[2, 'strong'].text.strip
+        text = td.elements[2, 'p'].elements['span'].elements['div'].text.gsub('&nbsp;', ' ').strip
+        review = {
+          text: text,
+          name: name,
+          date: date,
+        }
+        break unless yield review
+      rescue Exception => e
+        print_error e, item
+      end
+    end
+  end
+
+  private
+  def print_error(e, item)
+    puts e
+    puts e.backtrace
+    puts "page: #{@page}"
+    puts item
+  end
+end
+

data/lib/tstore_reviews.rb

@@ -0,0 +1,37 @@
+require 'httpclient'
+require 'tstore_review_page'
+
+class TstoreReviews
+  include BaseAppReviews
+
+  def each
+    unless @list.nil?
+      return @list.each do |item|
+        yield item
+      end
+    end
+
+    @list = []
+    url = "http://www.tstore.co.kr/userpoc/multi/popReply.omp"
+    params = {
+      prodId: @app_id,
+      currentPage: 0,
+      flag: 'L',
+      replyType: 0,
+    }
+
+    (@start_page..@end_page).each do |page|
+      params[:currentPage] = page
+      client = HTTPClient.new
+      content = client.post_content(url, params.to_query)
+      review_page = TstoreReviewPage.new content, page
+      break unless review_page.items do |item|
+        return false if Date.strptime(item[:date], '%Y-%m-%d') < @from_date
+        @list << item
+        yield item
+        true
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: app-reviews
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mu-ik Jeon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-29 00:00:00.000000000Z
+dependencies: []
+description: Mobile App Review Crawler
+email:
+- muikor@gmail.com
+executables:
+- app-reviews-crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- app-reviews.gemspec
+- bin/app-reviews-crawler
+- lib/app-reviews-crawler.rb
+- lib/app-reviews.rb
+- lib/app-reviews/version.rb
+- lib/app_store_review_page.rb
+- lib/app_store_reviews.rb
+- lib/base_app_reviews.rb
+- lib/play_store_review_page.rb
+- lib/play_store_reviews.rb
+- lib/tstore_review_page.rb
+- lib/tstore_reviews.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.19
+signing_key: 
+specification_version: 3
+summary: mobile app review crawler
+test_files: []