compactor 0.2.3 → 0.2.4

data/.gitignore

@@ -1,2 +1,4 @@
 coverage
+.bundle/
+vendor/
 

data/Gemfile.lock

@@ -1,20 +1,15 @@
 PATH
   remote: .
   specs:
-    compactor (0.2.3)
-      jruby-openssl (= 0.7.3)
+    compactor (0.2.4)
       mechanize (= 2.4)
-      nokogiri (>= 1.5.0, < 1.5.3)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    bouncy-castle-java (1.5.0146.1)
-    domain_name (0.5.7)
+    domain_name (0.5.4)
       unf (~> 0.0.3)
     fakeweb (1.3.0)
-    jruby-openssl (0.7.3)
-      bouncy-castle-java
     mechanize (2.4)
       domain_name (~> 0.5, >= 0.5.1)
       mime-types (~> 1.17, >= 1.17.2)
@@ -29,15 +24,13 @@ GEM
       metaclass (~> 0.0.1)
     net-http-digest_auth (1.2.1)
     net-http-persistent (2.8)
-    nokogiri (1.5.2)
-    nokogiri (1.5.2-java)
+    nokogiri (1.5.6)
     ntlm-http (0.1.1)
     rake (10.0.2)
     rcov (0.9.11)
     rcov (0.9.11-java)
     unf (0.0.5)
       unf_ext
-    unf (0.0.5-java)
     unf_ext (0.0.5)
     vcr (2.0.1)
     webrobots (0.0.13)

data/README.md

@@ -2,6 +2,8 @@
 
 [![Build Status](https://secure.travis-ci.org/julio/compactor.png)](http://travis-ci.org/julio/compactor)
 
+[![Gem Version](https://badge.fury.io/rb/compactor.png)](http://badge.fury.io/rb/compactor)
+
 Scrape Amazon Seller Central
 
 ## Installation
@@ -70,4 +72,4 @@ scrape "me@there.com", "secret", DateTime.parse("1/1/2012"), DateTime.now
 ## To-do
 
 - Refactor
-- 100% coverage
+- 100% coverage

data/lib/compactor/scraper.rb

@@ -11,6 +11,7 @@ module Compactor
     class UnknownReportType    < StandardError; end
     class MissingXmlReport     < StandardError; end
     class MissingReportButtons < StandardError; end
+    class ReportLoadingTimeout < StandardError; end
 
     ATTEMPTS_BEFORE_GIVING_UP = 15 # give up after 20 minutes
     MARKETPLACE_HOMEPAGE      = "https://sellercentral.amazon.com/gp/homepage.html"
@@ -205,6 +206,17 @@ module Compactor
         fail Compactor::Amazon::UnknownReportType
       end
 
+      # 6 attempts make it wait at most a minute, or close enough to it
+      def wait_for_element(attempts=6, &block)
+        attempts.times do |attempt|
+          element = yield
+          return element unless element.blank?
+          sleep 2**attempt # => 1 sec, 2 secs, 4, 8, 16, 32, etc
+        end
+
+        nil # no element found
+      end
+
       def rescue_empty_results(&block)
         3.times do
           yield
@@ -236,7 +248,8 @@ module Compactor
       end
 
       def get_reports_to_watch(reports_to_watch, reports, count=0)
-        return if reports_to_watch.empty? || timeout_fetching_reports(reports_to_watch, reports, count)
+        return if reports_to_watch.empty? ||
+                  timeout_fetching_reports(reports_to_watch, reports, count)
 
         rescue_empty_results { @mechanize.get @mechanize.page.uri }
         reports_to_watch.reject! do |row|
@@ -269,7 +282,12 @@ module Compactor
       end
 
       def page_has_no_results?
-        @mechanize.page.search!(".data-display").text.include? "No results found"
+        data_display_element =
+          wait_for_element { @mechanize.page.search(".data-display") }
+
+        fail ReportLoadingTimeout if data_display_element.blank?
+
+        data_display_element.text.include? "No results found"
       end
 
       def get_reports_in_page

data/lib/compactor/version.rb

@@ -1,3 +1,3 @@
 module Compactor
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/test/scraper_test.rb

@@ -2,6 +2,28 @@ require File.dirname(__FILE__) + '/test_helper'
 require File.dirname(__FILE__) + '/../lib/compactor'
 
 class ScraperTest < Test::Unit::TestCase
+  def test_should_not_find_elements_that_do_not_exist
+    VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
+      scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
+      mechanize = scraper.instance_variable_get("@mechanize")
+      element = scraper.send(:wait_for_element, 1) do
+        mechanize.page.search(".something-that-does-not-exist")
+      end
+      assert_nil element
+    end
+  end
+
+  def test_should_find_elements_that_do_exist
+    VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
+      scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
+      mechanize = scraper.instance_variable_get("@mechanize")
+      element = scraper.send(:wait_for_element, 1) do
+        mechanize.page.forms
+      end
+      assert Mechanize::Form === element[0]
+    end
+  end
+
   def test_should_raise_error_with_bad_login
     VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
       assert_raises Compactor::Amazon::AuthenticationError do
@@ -9,7 +31,7 @@ class ScraperTest < Test::Unit::TestCase
       end
     end
   end
-  
+
   def test_should_be_xml_if_button_label_is_Download_XML
     assert_equal :xml, Compactor::Amazon::ReportScraper.report_type("Download XML")
   end
@@ -27,7 +49,7 @@ class ScraperTest < Test::Unit::TestCase
       Compactor::Amazon::ReportScraper.report_type("Download PDF")
     end
   end
- 
+
   def test_should_be_able_to_get_buyer_name_and_shipping_address_for_orders
     VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders") do
       scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
@@ -46,7 +68,7 @@ class ScraperTest < Test::Unit::TestCase
       }, orders)
     end
   end
- 
+
   def test_should_support_addresses_where_the_street_address_line_does_not_start_with_a_number
     VCR.use_cassette("AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number") do
       scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
@@ -65,7 +87,7 @@ class ScraperTest < Test::Unit::TestCase
       }, orders)
     end
   end
- 
+
   def test_should_handle_large_reports
     VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders_big") do
       scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
@@ -93,7 +115,7 @@ class ScraperTest < Test::Unit::TestCase
       assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
     end
   end
- 
+
   def test_should_find_reports_in_more_than_on_page
     VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/multiple_pages") do
       scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
@@ -102,7 +124,7 @@ class ScraperTest < Test::Unit::TestCase
       assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
     end
   end
-  
+
   def test_should_find_no_reports_if_not_in_date_range
     VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports") do
       scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: compactor
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Julio Santos
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-01-12 00:00:00 Z
+date: 2013-02-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mechanize
@@ -125,6 +125,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- compactor-0.2.3.gem
 - compactor.gemspec
 - lib/compactor.rb
 - lib/compactor/extensions.rb
@@ -205,3 +206,4 @@ test_files:
 - test/scraped_row_test.rb
 - test/scraper_test.rb
 - test/test_helper.rb
+has_rdoc: