app-reviews 0.0.1 → 0.0.2

data/bin/app-reviews-crawler

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require File.join(File.dirname(__FILE__), '../lib/app-reviews-crawler')
+require File.join(File.dirname(__FILE__), '../lib/app-reviews/crawler')
 
-AppReviewsCrawler.new.execute ARGV
+AppReviews::Crawler.new.execute ARGV

data/lib/app-reviews.rb

@@ -1,2 +1,16 @@
+require 'app-reviews/app_store/reviews'
+require 'app-reviews/play_store/reviews'
+require 'app-reviews/t_store/reviews'
+
 module AppReviews
+  def self.create(store, app_id)
+    case store
+    when 'appstore'
+      AppStore::Reviews.new app_id
+    when 'play'
+      PlayStore::Reviews.new app_id
+    when 'tstore'
+      TStore::Reviews.new app_id
+    end
+  end
 end

data/lib/app-reviews/app_store/review_page.rb

@@ -0,0 +1,66 @@
+require 'nokogiri'
+require 'rexml/document'
+include REXML
+
+module AppReviews
+  module AppStore
+    class ReviewPage
+      def initialize(text, page)
+        @doc = Nokogiri::XML(text)
+        @page = page
+      end
+
+      def items
+        path = "Document > View > ScrollView > VBoxView > View > MatrixView > VBoxView > VBoxView > VBoxView"
+        @doc.css(path).each do |link|
+          begin
+            review = parse link
+            next if review.nil?
+            break unless yield review
+          rescue Exception => e
+            print_parse_error e, link
+          end
+        end
+      end
+
+      def last_page
+        @doc.css("MatrixView > VBoxView > VBoxView > HBoxView:nth-child(2) > TextView > SetFontStyle").each do |link|
+          link.content.split(' ').last.to_i 
+        end
+      end
+
+      private
+      def parse(link)
+        node = link.css('TextView > SetFontStyle')
+        date = node[2].content.gsub("\n", '').strip
+        index = date.rindex('- ')
+        return if index.nil?
+        index += 1
+        date = date[index..-1].strip
+        name_el = node.css('GotoURL > b')
+        title_el = node[0].css('b')
+        name = name_el.first.content.strip
+        title = title_el.first.content
+        text = node[3].content.strip
+        node = link.css('HBoxView > HBoxView > HBoxView')
+        rating = node.attr('alt').value
+
+        {
+          title: title,
+          name: name,
+          text: text,
+          rating: rating,
+          date: date,
+        }
+      end
+
+      def print_parse_error(e, link)
+        puts e
+        puts e.backtrace
+        puts "page: #{@page}"
+        puts link
+      end
+    end
+  end
+end
+

data/lib/app-reviews/app_store/reviews.rb

@@ -0,0 +1,49 @@
+require 'app-reviews/base_reviews'
+require 'app-reviews/app_store/review_page'
+require 'httpclient'
+require 'open-uri'
+
+module AppReviews
+  module AppStore
+    class Reviews
+      include BaseReviews
+
+      def each
+        unless @list.nil?
+          return @list.each do |item|
+            yield item
+          end
+        end
+
+        @list = []
+        country_codes = [143441, 143466, 143463]
+        country_codes.each do |country_code| 
+          get_reviews(country_code) do |review|
+            @list << review
+            yield review
+          end
+        end
+      end
+
+      private
+      def get_reviews(country)
+        (@start_page..@end_page).each do |page|
+          url = "http://phobos.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?sortOrdering=4&onlyLatestVersion=false&sortAscending=true&pageNumber=#{(page - 1)}&type=Purple+Software&id=#{@app_id}"
+
+          f = open(url,	"User-Agent" => "iTunes-iPhone/2.2 (2)", "X-Apple-Store-Front" => "#{country}-1") 
+#      File.new('review.xml', 'w').puts f.read
+#      exit
+          review_page = AppStore::ReviewPage.new f.read, page
+          break unless review_page.items do |item|
+            return false if Date.parse(item[:date]) < @from_date
+            yield item
+            true
+          end
+
+          break if review_page.last_page < page + 1
+        end
+      end
+    end
+  end
+end
+

data/lib/app-reviews/base_reviews.rb

@@ -0,0 +1,37 @@
+module AppReviews
+  module BaseReviews
+    include Enumerable
+
+    def set_last_date(date)
+      @last_date = date
+    end
+
+    def last_date
+      @last_date
+    end
+
+    def set_page(start_page, end_page)
+      @start_page = start_page
+      @end_page = end_page
+    end
+
+    def set_from_date(date)
+      @from_date = date
+    end
+
+    def count
+      @list.count
+    end
+
+    def last
+      @list.last
+    end
+
+    private
+    def initialize(app_id)
+      @app_id = app_id
+      @list = nil
+    end
+  end
+end
+

data/lib/app-reviews/crawler.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "app-reviews/version"
+require 'active_support/core_ext'
+require 'app-reviews'
+require 'yaml'
+
+module AppReviews
+  class Crawler
+    def execute(argv)
+      return print_usage unless validate argv
+      store, app_id, from_date_str = argv
+      start_page = 1
+      end_page = 10000
+      from_date = Date.parse(from_date_str)
+      puts "store: #{store}"
+      puts "app_id: #{app_id}"
+      puts "from_date: #{from_date}"
+      puts "page: #{start_page} ~ #{end_page}"
+
+      reviews = AppReviews.create(store, app_id)
+      return print_usage unless reviews
+      reviews.set_page start_page, end_page
+      reviews.set_from_date from_date
+      reviews.each do |item|
+        puts item.to_yaml
+      end
+
+      puts "Review Count: #{reviews.count}"
+      puts "Review Last Date: #{reviews.last[:date]}"
+    end
+
+    def create_reviews(store, app_id)
+    end
+
+    def validate(argv)
+      return false if argv.size < 3
+      true
+    end
+
+    def print_usage
+      puts "USAGE: app-reviews-crawler appstore|play|tstore store_app_id from_date"
+      puts "example# app-reviews-crawler appstore 383844387 2012-03-26"
+      puts "example# app-reviews-crawler play com.thinkreals.pocketstyle2 2012-03-26"
+      puts "example# app-reviews-crawler tstore 0000033534 2012-03-26"
+    end
+  end
+end

data/lib/app-reviews/play_store/review_page.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require 'rexml/document'
+include REXML
+
+module AppReviews
+  module PlayStore
+    class ReviewPage
+      def initialize(text, page)
+        result = JSON(text[5..-1])
+        html = result['htmlContent'].gsub('<hr>', '<hr />')
+        @xml = Document.new("<body>#{html}</body>")
+        @page = page
+      end
+
+      def items
+        @xml.root.elements.each do |item|
+          next if item.elements.size < 1
+          begin
+            name_el = item.elements['span'].elements['strong']
+            next unless name_el
+            date = item.elements[2, 'span'].text.sub('님이', '').strip
+
+            title = item.elements['div'].elements['h4'].text.strip
+            if item.elements['p']
+              text = item.elements['p'].text 
+            else
+              text = nil
+            end
+            text = text.strip if text
+            name = name_el.text.strip
+            rating = item.elements['div'].elements['div'].attribute('title').value.strip
+            review = {
+              title: title,
+              text: text,
+              name: name,
+              rating: rating,
+              date: date,
+            }
+            break unless yield review
+          rescue Exception => e
+            print_error e, item
+          end
+        end
+      end
+
+      def print_error(e, item)
+        puts e
+        puts e.backtrace
+        puts "page: #{@page}"
+        puts item
+      end
+    end
+  end
+end
+

data/lib/app-reviews/play_store/reviews.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'app-reviews/base_reviews'
+require 'app-reviews/play_store/review_page'
+require 'httpclient'
+
+module AppReviews
+  module PlayStore
+    class Reviews
+      include BaseReviews
+
+      def each
+        unless @list.nil?
+          return @list.each do |item|
+            yield item
+          end
+        end
+
+        @list = []
+        url = "https://play.google.com/store/getreviews"
+        params = {
+          id: @app_id,
+          reviewSortOrder: 0,
+          reviewType: 1,
+          pageNum: 0,
+        }
+
+        (@start_page..@end_page).each do |page|
+          params[:pageNum] = page - 1
+          client = HTTPClient.new
+          content = client.post_content(url, params.to_query)
+          review_page = PlayStore::ReviewPage.new content, page
+          break unless review_page.items do |item|
+            return false if Date.strptime(item[:date], '%Y년 %m월 %d일') < @from_date
+            @list << item
+            yield item
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/app-reviews/t_store/review_page.rb

@@ -0,0 +1,44 @@
+require 'rexml/document'
+include REXML
+
+module AppReviews
+  module TStore
+    class ReviewPage
+      def initialize(content, page)
+        content = content.gsub(/<textarea[^<]+<\/textarea>/, '')
+        content = content.gsub('gif""', 'gif"')
+        xml = Document.new(content)
+        @table = xml.root.elements['body'].elements[2, 'form'].elements[2, 'div'].elements['table']
+      end
+
+      def items
+        @table.elements.each do |item|
+          begin
+            td = item.elements['td']
+            next unless td
+            date = td.elements['p'].elements['span'].text.strip
+            name = td.elements['p'].elements[2, 'strong'].text.strip
+            text = td.elements[2, 'p'].elements['span'].elements['div'].text.gsub('&nbsp;', ' ').strip
+            review = {
+              text: text,
+              name: name,
+              date: date,
+            }
+            break unless yield review
+          rescue Exception => e
+            print_error e, item
+          end
+        end
+      end
+
+      private
+      def print_error(e, item)
+        puts e
+        puts e.backtrace
+        puts "page: #{@page}"
+        puts item
+      end
+    end
+  end
+end
+

data/lib/app-reviews/t_store/reviews.rb

@@ -0,0 +1,41 @@
+require 'app-reviews/base_reviews'
+require 'app-reviews/t_store/review_page'
+require 'httpclient'
+
+module AppReviews
+  module TStore
+    class Reviews
+      include BaseReviews
+
+      def each
+        unless @list.nil?
+          return @list.each do |item|
+            yield item
+          end
+        end
+
+        @list = []
+        url = "http://www.tstore.co.kr/userpoc/multi/popReply.omp"
+        params = {
+          prodId: @app_id,
+          currentPage: 0,
+          flag: 'L',
+          replyType: 0,
+        }
+
+        (@start_page..@end_page).each do |page|
+          params[:currentPage] = page
+          client = HTTPClient.new
+          content = client.post_content(url, params.to_query)
+          review_page = TStore::ReviewPage.new content, page
+          break unless review_page.items do |item|
+            return false if Date.strptime(item[:date], '%Y-%m-%d') < @from_date
+            @list << item
+            yield item
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/app-reviews/version.rb

@@ -1,3 +1,3 @@
 module AppReviews
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: app-reviews
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -26,16 +26,16 @@ files:
 - Rakefile
 - app-reviews.gemspec
 - bin/app-reviews-crawler
-- lib/app-reviews-crawler.rb
 - lib/app-reviews.rb
+- lib/app-reviews/app_store/review_page.rb
+- lib/app-reviews/app_store/reviews.rb
+- lib/app-reviews/base_reviews.rb
+- lib/app-reviews/crawler.rb
+- lib/app-reviews/play_store/review_page.rb
+- lib/app-reviews/play_store/reviews.rb
+- lib/app-reviews/t_store/review_page.rb
+- lib/app-reviews/t_store/reviews.rb
 - lib/app-reviews/version.rb
-- lib/app_store_review_page.rb
-- lib/app_store_reviews.rb
-- lib/base_app_reviews.rb
-- lib/play_store_review_page.rb
-- lib/play_store_reviews.rb
-- lib/tstore_review_page.rb
-- lib/tstore_reviews.rb
 homepage: ''
 licenses: []
 post_install_message: 

data/lib/app-reviews-crawler.rb

@@ -1,55 +0,0 @@
-# encoding: utf-8
-require "app-reviews/version"
-require 'app_store_reviews'
-require 'play_store_reviews'
-require 'tstore_reviews'
-require 'active_support/core_ext'
-
-class AppReviewsCrawler
-  def execute(argv)
-    return print_usage unless validate argv
-    store, app_id, from_date_str = argv
-    start_page = 1
-    end_page = 10000
-    from_date = Date.parse(from_date_str)
-    puts "store: #{store}"
-    puts "app_id: #{app_id}"
-    puts "from_date: #{from_date}"
-    puts "page: #{start_page} ~ #{end_page}"
-
-    reviews = create_reviews(store, app_id)
-    return print_usage unless reviews
-    reviews.set_page start_page, end_page
-    reviews.set_from_date from_date
-    require 'yaml'
-    reviews.each do |item|
-      puts item.to_yaml
-    end
-
-    puts "Review Count: #{reviews.count}"
-    puts "Review Last Date: #{reviews.last[:date]}"
-  end
-
-  def create_reviews(store, app_id)
-    case store
-    when 'appstore'
-      AppStoreReviews.new app_id
-    when 'play'
-      PlayStoreReviews.new app_id
-    when 'tstore'
-      TstoreReviews.new app_id
-    end
-  end
-
-  def validate(argv)
-    return false if argv.size < 3
-    true
-  end
-
-  def print_usage
-    puts "USAGE: app-reviews-crawler appstore|play|tstore store_app_id from_date"
-    puts "example# app-reviews-crawler appstore 383844387 2012-03-26"
-    puts "example# app-reviews-crawler play com.thinkreals.pocketstyle2 2012-03-26"
-    puts "example# app-reviews-crawler tstore 0000033534 2012-03-26"
-  end
-end

data/lib/app_store_review_page.rb

@@ -1,62 +0,0 @@
-require 'nokogiri'
-require 'rexml/document'
-include REXML
-
-class AppStoreReviewPage
-  def initialize(text, page)
-    @doc = Nokogiri::XML(text)
-    @page = page
-  end
-
-  def items
-    path = "Document > View > ScrollView > VBoxView > View > MatrixView > VBoxView > VBoxView > VBoxView"
-    @doc.css(path).each do |link|
-      begin
-        review = parse link
-        next if review.nil?
-        break unless yield review
-      rescue Exception => e
-        print_parse_error e, link
-      end
-    end
-  end
-
-  def last_page
-    @doc.css("MatrixView > VBoxView > VBoxView > HBoxView:nth-child(2) > TextView > SetFontStyle").each do |link|
-      link.content.split(' ').last.to_i 
-    end
-  end
-
-  private
-  def parse(link)
-    node = link.css('TextView > SetFontStyle')
-    date = node[2].content.gsub("\n", '').strip
-    index = date.rindex('- ')
-    return if index.nil?
-    index += 1
-    date = date[index..-1].strip
-    name_el = node.css('GotoURL > b')
-    title_el = node[0].css('b')
-    name = name_el.first.content.strip
-    title = title_el.first.content
-    text = node[3].content.strip
-    node = link.css('HBoxView > HBoxView > HBoxView')
-    rating = node.attr('alt').value
-
-    {
-      title: title,
-      name: name,
-      text: text,
-      rating: rating,
-      date: date,
-    }
-  end
-
-  def print_parse_error(e, link)
-    puts e
-    puts e.backtrace
-    puts "page: #{@page}"
-    puts link
-  end
-end
-

data/lib/app_store_reviews.rb

@@ -1,45 +0,0 @@
-require 'base_app_reviews'
-require 'app_store_review_page'
-require 'httpclient'
-require 'open-uri'
-
-class AppStoreReviews
-  include BaseAppReviews
-
-  def each
-    unless @list.nil?
-      return @list.each do |item|
-        yield item
-      end
-    end
-
-    @list = []
-    country_codes = [143441, 143466, 143463]
-    country_codes.each do |country_code| 
-      get_reviews(country_code) do |review|
-        @list << review
-        yield review
-      end
-    end
-  end
-
-  private
-  def get_reviews(country)
-    (@start_page..@end_page).each do |page|
-      url = "http://phobos.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?sortOrdering=4&onlyLatestVersion=false&sortAscending=true&pageNumber=#{(page - 1)}&type=Purple+Software&id=#{@app_id}"
-
-      f = open(url,	"User-Agent" => "iTunes-iPhone/2.2 (2)", "X-Apple-Store-Front" => "#{country}-1") 
-#      File.new('review.xml', 'w').puts f.read
-#      exit
-      review_page = AppStoreReviewPage.new f.read, page
-      break unless review_page.items do |item|
-        return false if Date.parse(item[:date]) < @from_date
-        yield item
-        true
-      end
-
-      break if review_page.last_page < page + 1
-    end
-  end
-end
-

data/lib/base_app_reviews.rb

@@ -1,30 +0,0 @@
-module BaseAppReviews
-  include Enumerable
-
-  def initialize(app_id)
-    @app_id = app_id
-    @list = nil
-  end
-
-  def set_last_date(date)
-    @last_date = date
-  end
-
-  def last_date
-    @last_date
-  end
-
-  def set_page(start_page, end_page)
-    @start_page = start_page
-    @end_page = end_page
-  end
-
-  def set_from_date(date)
-    @from_date = date
-  end
-
-  def last
-    @list.last
-  end
-end
-

data/lib/play_store_review_page.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-require 'rexml/document'
-include REXML
-
-class PlayStoreReviewPage
-  def initialize(text, page)
-    result = JSON(text[5..-1])
-    html = result['htmlContent'].gsub('<hr>', '<hr />')
-    @xml = Document.new("<body>#{html}</body>")
-    @page = page
-  end
-
-  def items
-    @xml.root.elements.each do |item|
-      next if item.elements.size < 1
-      begin
-        name_el = item.elements['span'].elements['strong']
-        next unless name_el
-        date = item.elements[2, 'span'].text.sub('님이', '').strip
-
-        title = item.elements['div'].elements['h4'].text.strip
-        if item.elements['p']
-          text = item.elements['p'].text 
-        else
-          text = nil
-        end
-        text = text.strip if text
-        name = name_el.text.strip
-        rating = item.elements['div'].elements['div'].attribute('title').value.strip
-        review = {
-          title: title,
-          text: text,
-          name: name,
-          rating: rating,
-          date: date,
-        }
-        break unless yield review
-      rescue Exception => e
-        print_error e, item
-      end
-    end
-  end
-
-  def print_error(e, item)
-    puts e
-    puts e.backtrace
-    puts "page: #{@page}"
-    puts item
-  end
-end
-

data/lib/play_store_reviews.rb

@@ -1,38 +0,0 @@
-# encoding: utf-8
-require 'httpclient'
-require 'play_store_review_page'
-
-class PlayStoreReviews
-  include BaseAppReviews
-
-  def each
-    unless @list.nil?
-      return @list.each do |item|
-        yield item
-      end
-    end
-
-    @list = []
-    url = "https://play.google.com/store/getreviews"
-    params = {
-      id: @app_id,
-      reviewSortOrder: 0,
-      reviewType: 1,
-      pageNum: 0,
-    }
-
-    (@start_page..@end_page).each do |page|
-      params[:pageNum] = page - 1
-      client = HTTPClient.new
-      content = client.post_content(url, params.to_query)
-      review_page = PlayStoreReviewPage.new content, page
-      break unless review_page.items do |item|
-        return false if Date.strptime(item[:date], '%Y년 %m월 %d일') < @from_date
-        @list << item
-        yield item
-        true
-      end
-    end
-  end
-end
-

data/lib/tstore_review_page.rb

@@ -1,40 +0,0 @@
-require 'rexml/document'
-include REXML
-
-class TstoreReviewPage
-  def initialize(content, page)
-    content = content.gsub(/<textarea[^<]+<\/textarea>/, '')
-    content = content.gsub('gif""', 'gif"')
-    xml = Document.new(content)
-    @table = xml.root.elements['body'].elements[2, 'form'].elements[2, 'div'].elements['table']
-  end
-
-  def items
-    @table.elements.each do |item|
-      begin
-        td = item.elements['td']
-        next unless td
-        date = td.elements['p'].elements['span'].text.strip
-        name = td.elements['p'].elements[2, 'strong'].text.strip
-        text = td.elements[2, 'p'].elements['span'].elements['div'].text.gsub('&nbsp;', ' ').strip
-        review = {
-          text: text,
-          name: name,
-          date: date,
-        }
-        break unless yield review
-      rescue Exception => e
-        print_error e, item
-      end
-    end
-  end
-
-  private
-  def print_error(e, item)
-    puts e
-    puts e.backtrace
-    puts "page: #{@page}"
-    puts item
-  end
-end
-

data/lib/tstore_reviews.rb

@@ -1,37 +0,0 @@
-require 'httpclient'
-require 'tstore_review_page'
-
-class TstoreReviews
-  include BaseAppReviews
-
-  def each
-    unless @list.nil?
-      return @list.each do |item|
-        yield item
-      end
-    end
-
-    @list = []
-    url = "http://www.tstore.co.kr/userpoc/multi/popReply.omp"
-    params = {
-      prodId: @app_id,
-      currentPage: 0,
-      flag: 'L',
-      replyType: 0,
-    }
-
-    (@start_page..@end_page).each do |page|
-      params[:currentPage] = page
-      client = HTTPClient.new
-      content = client.post_content(url, params.to_query)
-      review_page = TstoreReviewPage.new content, page
-      break unless review_page.items do |item|
-        return false if Date.strptime(item[:date], '%Y-%m-%d') < @from_date
-        @list << item
-        yield item
-        true
-      end
-    end
-  end
-end
-