compactor 0.1.6 → 0.2.3

data/Gemfile.lock

@@ -1,15 +1,20 @@
 PATH
   remote: .
   specs:
-    compactor (0.1.6)
+    compactor (0.2.3)
+      jruby-openssl (= 0.7.3)
       mechanize (= 2.4)
+      nokogiri (>= 1.5.0, < 1.5.3)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    domain_name (0.5.4)
+    bouncy-castle-java (1.5.0146.1)
+    domain_name (0.5.7)
       unf (~> 0.0.3)
     fakeweb (1.3.0)
+    jruby-openssl (0.7.3)
+      bouncy-castle-java
     mechanize (2.4)
       domain_name (~> 0.5, >= 0.5.1)
       mime-types (~> 1.17, >= 1.17.2)
@@ -24,12 +29,15 @@ GEM
       metaclass (~> 0.0.1)
     net-http-digest_auth (1.2.1)
     net-http-persistent (2.8)
-    nokogiri (1.5.5)
+    nokogiri (1.5.2)
+    nokogiri (1.5.2-java)
     ntlm-http (0.1.1)
-    rake (0.9.2.2)
+    rake (10.0.2)
     rcov (0.9.11)
+    rcov (0.9.11-java)
     unf (0.0.5)
       unf_ext
+    unf (0.0.5-java)
     unf_ext (0.0.5)
     vcr (2.0.1)
     webrobots (0.0.13)

data/README.md

@@ -24,8 +24,34 @@ Or install it yourself as:
 rake test:coverage
 ```
 
-```
-more soon
+```ruby
+def scrape(email, password, from, to)
+  scraper = Compactor::Amazon::ReportScraper.new(:email => email, :password => password)
+  marketplaces = scraper.marketplaces
+
+  original_from = from
+  original_to   = to
+
+  marketplaces.each do |marketplace|
+    scraper.select_marketplace marketplace[1]
+
+    from = original_from
+    to   = original_to
+
+    puts "Marketplace: #{marketplace[1]}"
+    while from < to
+      begin
+        reports_by_type = scraper.reports(from, to)
+        puts "There are #{reports_by_type.size} reports between #{from.to_date} and #{to.to_date}"
+      rescue Exception => e
+        puts "ERROR: #{e.message} - USER: #{email}"
+      end
+      from += 1.week
+    end
+  end
+end
+
+scrape "me@there.com", "secret", DateTime.parse("1/1/2012"), DateTime.now
 ```
 
 ## Contributing
@@ -44,4 +70,4 @@ more soon
 ## To-do
 
 - Refactor
-- document
+- 100% coverage

data/compactor.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
 
   if RUBY_PLATFORM == "java"
     gem.add_runtime_dependency "jruby-openssl", '0.7.3'
-    gem.add_runtime_dependency "nokogiri", "1.5.0.beta.2"
+    gem.add_runtime_dependency "nokogiri", ">= 1.5.0", "< 1.5.3"
   end
 
   gem.add_development_dependency "rake"

data/lib/compactor/extensions.rb

@@ -2,6 +2,16 @@ class Object
   def blank?; respond_to?(:empty?) ? empty? : !self; end
 end
 
+class Date
+  def self.parse_to_us_format(date)
+    if date.is_a? String
+      date_format = date['-'] ? "%Y-%m-%d" : "%m/%d/%Y"
+      date = Date.strptime(date, date_format)
+    end
+    date.strftime("%m/%d/%y")
+  end
+end
+
 module Nokogiri
   class MissingElement < ::StandardError; end
 

data/lib/compactor/scraped_row.rb

@@ -7,7 +7,7 @@ module Compactor
       end
 
       def can_download_report?
-        !report_buttons.empty?
+        !report_buttons.blank?
       end
 
       def report_buttons
@@ -17,8 +17,10 @@ module Compactor
       end
 
       def download_report
-        report_url        = report_buttons[0].node["href"]
-        report_identifier = report_buttons[0].node.search(".button_label").text
+        buttons           = report_buttons
+        button_index      = index_of_button(buttons)
+        report_url        = buttons[button_index].node["href"]
+        report_identifier = buttons[button_index].node.search(".button_label").text
         type              = ReportScraper.report_type(report_identifier)
         response_body     = @mechanize.get(report_url).body
 
@@ -43,18 +45,25 @@ module Compactor
         })
       end
 
-      def ready?
-        div = last_cell.search("div")[-1]
-        text = div.text
+      # A settlement period (row) is considered ready to be parsed
+      # if it's not processing, open or in progress. Also the "regenerate" 
+      # button is not present. This means that all is left is 1 or more
+      # buttons to get the actual reports
+      def requestable_report?
+        !last_cell.search(".regenerateButton").empty?
+      end
+
+      def not_settled_report?
+        text = last_div.text
 
-        ignorable_periods = ["(Processing)", "(Open)", "In Progress"]
-        !ignorable_periods.any? { |ignore_text| text.include?(ignore_text) &&
-          div.search(".regenerateButton").blank? }
+        # Is the report not settled yet? (in pending-like state)
+        ["(Processing)", "(Open)", "In Progress"].any? do |report_state|
+          text.include?(report_state)
+        end
       end
 
       def deposit_amount
         @deposit_amount = fetch_deposit_amount if !@deposit_amount
-
         @deposit_amount
       end
 
@@ -64,6 +73,10 @@ module Compactor
 
       private
 
+      def last_div
+        last_cell.search("div")[-1]
+      end
+
       def fetch_deposit_amount
         deposit_cell = @node.search("td")[-2]
         deposit_cell ? deposit_cell.text.gsub(/[^0-9\.]/, '').to_f : 0.0
@@ -76,6 +89,19 @@ module Compactor
       def last_cell
         @last_cell ||= @node.search("td")[-1]
       end
+
+      def index_of_button(buttons)
+        raise MissingReportButtons if buttons.blank? # no buttons at all!
+
+        buttons.each_with_index do |button, index|
+          # XML is preferred
+          return index if button.node.search(".button_label").text == "Download XML"
+        end
+
+        # No XML, look for another type of report, use the first one, whatever
+        # the type
+        0
+      end
     end
   end
 end

data/lib/compactor/scraper.rb

@@ -1,12 +1,16 @@
+# encoding: utf-8 # make the regexes ruby 1.9 friendly
+
 module Compactor
   module Amazon
-    class AddressParseFailure < StandardError; end
-    class AuthenticationError < StandardError; end
-    class LockedAccountError  < StandardError; end
-    class MissingRow          < StandardError; end
-    class NoMarketplacesError < StandardError; end
-    class NotProAccountError  < StandardError; end
-    class UnknownReportType   < StandardError; end
+    class AddressParseFailure  < StandardError; end
+    class AuthenticationError  < StandardError; end
+    class LockedAccountError   < StandardError; end
+    class MissingRow           < StandardError; end
+    class NoMarketplacesError  < StandardError; end
+    class NotProAccountError   < StandardError; end
+    class UnknownReportType    < StandardError; end
+    class MissingXmlReport     < StandardError; end
+    class MissingReportButtons < StandardError; end
 
     ATTEMPTS_BEFORE_GIVING_UP = 15 # give up after 20 minutes
     MARKETPLACE_HOMEPAGE      = "https://sellercentral.amazon.com/gp/homepage.html"
@@ -50,7 +54,7 @@ module Compactor
           return result
         end
 
-        marketplace_name = @mechanize.page.search("#market_switch")
+        marketplace_name = @mechanize.page.search("#market_switch .merch-site-span")
         if marketplace_name
           return [ [ marketplace_name.text.strip, nil ] ]
         end
@@ -66,15 +70,14 @@ module Compactor
       def get_balance
         go_to_past_settlements('', '')
         return 0.0 if page_has_no_results?
-        open_row = report_rows.detect { |row| !row.ready? }
-        return 0.0 if open_row.nil?
-        open_row.deposit_amount
+        open_row = report_rows.detect { |row| row.not_settled_report? }
+
+        open_row.nil? ? 0.0 : open_row.deposit_amount
       end
 
       def reports(from, to)
         from, to = parse_dates(from, to)
         go_to_past_settlements(from, to)
-
         get_reports
       end
 
@@ -88,8 +91,10 @@ module Compactor
           begin
             tr = @mechanize.page.search!("//tr[@class='list-row']/td[@class='data-display-field'][text()=\"Contact Buyer:\"]").first.parent
             td = tr.search!("td[2]")
+
             order["BuyerName"] = td.text.strip
             td = @mechanize.page.search!("//tr[@class='list-row']/td[@class='data-display-field']/strong[text()='Shipping Address:']").first.parent
+
             addr_lines = td.children.map(&:text).reject { |l| l.blank? || l =~ /^Shipping Address/ }
             order["ShippingAddress"] = parse_address_lines!(addr_lines)
           rescue Exception => e
@@ -115,7 +120,7 @@ module Compactor
         results << [ 'Amazon Seller Account', AMAZON_COM_MARKETPLACE_ID ] if name
 
         name, marketplace_id = marketplaces.detect do |n, m_id|
-           n == 'Your Checkout Website' && !m_id.nil?
+          (n == 'Your Checkout Website' || n == "Checkout by Amazon (Production View)") && !m_id.nil?
         end
         results << [ 'Checkout By Amazon', marketplace_id ] if name
 
@@ -174,10 +179,8 @@ module Compactor
             reports[report_type] ||= []
             reports[report_type] << report_streams
           end
-
           page_num += 1
         end while pages_to_parse
-
         reports.each { |type, streams| streams.flatten! }
       end
 
@@ -272,13 +275,12 @@ module Compactor
       def get_reports_in_page
         reports_to_watch = []
         reports = {}
-
         return reports if page_has_no_results?
 
         report_rows.each do |row|
           if row.can_download_report?
             add_to_collection(reports, row)
-          elsif row.ready?
+          elsif row.requestable_report?
             @mechanize.transact do
               row.request_report
               reports_to_watch << row
@@ -292,10 +294,7 @@ module Compactor
       end
 
       def parse_dates(from, to)
-        from = Date.parse(from.to_s).strftime("%m/%d/%y")
-        to   = Date.parse(to.to_s).strftime("%m/%d/%y")
-
-        [from, to]
+        [ Date.parse_to_us_format(from.to_s), Date.parse_to_us_format(to.to_s) ]
       end
 
       def login_to_seller_central(email, password)

data/lib/compactor/version.rb

@@ -1,3 +1,3 @@
 module Compactor
-  VERSION = "0.1.6"
-end
+  VERSION = "0.2.3"
+end

data/test/date_extensions_test.rb

@@ -0,0 +1,15 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DateExtensionsTest < Test::Unit::TestCase
+  def test_parse_strings_separated_by_dashes
+    assert_equal "01/01/12", Date.parse_to_us_format("2012-1-1")
+  end
+
+  def test_parse_strings_separated_by_slashes
+    assert_equal "12/31/11", Date.parse_to_us_format("12/31/2011")
+  end
+
+  def test_convert_dates_to_strings
+    assert_equal "01/01/12", Date.parse_to_us_format(Date.parse("2012-1-1"))
+  end
+end

data/test/test_helper.rb

@@ -1,4 +1,5 @@
-require "rubygems"
+require "bundler/setup"
+
 require "test/unit"
 require "vcr"
 require "mechanize"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: compactor
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 17
   prerelease: 
   segments: 
   - 0
-  - 1
-  - 6
-  version: 0.1.6
+  - 2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - Julio Santos
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-05 00:00:00 Z
+date: 2013-01-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mechanize
@@ -125,13 +125,13 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- compactor-0.1.2.gem
 - compactor.gemspec
 - lib/compactor.rb
 - lib/compactor/extensions.rb
 - lib/compactor/scraped_row.rb
 - lib/compactor/scraper.rb
 - lib/compactor/version.rb
+- test/date_extensions_test.rb
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml
@@ -185,6 +185,7 @@ signing_key:
 specification_version: 3
 summary: Scrape Amazon Seller Central
 test_files: 
+- test/date_extensions_test.rb
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
 - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml