compactor 0.1.2

data/.gitignore

@@ -0,0 +1,2 @@
+coverage
+

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - jruby-18mode # JRuby in 1.8 mode
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-18mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,47 @@
+PATH
+  remote: .
+  specs:
+    compactor (0.1.2)
+      mechanize (= 2.4)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    domain_name (0.5.4)
+      unf (~> 0.0.3)
+    fakeweb (1.3.0)
+    mechanize (2.4)
+      domain_name (~> 0.5, >= 0.5.1)
+      mime-types (~> 1.17, >= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (~> 0.0, >= 0.0.9)
+    metaclass (0.0.1)
+    mime-types (1.19)
+    mocha (0.12.1)
+      metaclass (~> 0.0.1)
+    net-http-digest_auth (1.2.1)
+    net-http-persistent (2.8)
+    nokogiri (1.5.5)
+    ntlm-http (0.1.1)
+    rake (0.9.2.2)
+    rcov (0.9.11)
+    unf (0.0.5)
+      unf_ext
+    unf_ext (0.0.5)
+    vcr (2.0.1)
+    webrobots (0.0.13)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  compactor!
+  fakeweb
+  mocha (= 0.12.1)
+  rake
+  rcov (= 0.9.11)
+  vcr (~> 2.0.0)

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Julio Santos
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Compactor
+
+[![Build Status](https://secure.travis-ci.org/julio/caterpillar.png)](http://travis-ci.org/julio/caterpillar)
+
+Scrape Amazon Seller Central
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'compactor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install compactor
+
+## Usage
+
+```
+rake test:coverage
+```
+
+```
+more soon
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Contributors
+
+* Trae Robrock ( https://github.com/trobrock )
+* Julio Santos ( https://github.com/julio )
+
+## To-do
+
+- Refactor
+- document

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rake'
+require 'rake/testtask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the compactor gem'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+def run_coverage(files)
+  rm_f "coverage"
+  rm_f "coverage.data"
+
+  if files.empty?
+    puts "No files were specified for testing"
+    return
+  end
+
+  files = files.join(" ")
+
+  if RUBY_PLATFORM =~ /darwin/
+    exclude = '--exclude "gems/*" --exclude "Library/Frameworks/*"'
+  elsif RUBY_PLATFORM =~ /java/
+    exclude = '--exclude "rubygems/*,jruby/*,parser*,gemspec*,_DELEGATION*,__FORWARDABLE__,erb,eval*,recognize_optimized*,yaml,yaml/*,fcntl"'
+  else
+    exclude = '--exclude "rubygems/*"'
+  end
+
+  rcov_bin = RUBY_PLATFORM =~ /java/ ? "jruby -S bundle exec rcov" : "bundle exec rcov"
+  rcov = "#{rcov_bin} --rails -Ilib:test --sort coverage --text-report #{exclude}"
+  puts
+  puts
+  puts "Running tests..."
+  cmd = "#{rcov} #{files}"
+  puts cmd
+  sh cmd
+end
+
+namespace :test do
+  desc "Measures test coverage"
+  task :coverage do
+    run_coverage Dir["test/**/*_test.rb"]
+  end
+end

data/compactor.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "compactor/version"
+
+Gem::Specification.new do |gem|
+  gem.name        = "compactor"
+  gem.version     = Compactor::VERSION
+  gem.authors     = ["Julio Santos"]
+  gem.email       = ["julio@morgane.com"]
+  gem.homepage    = ""
+  gem.summary     = "Scrape Amazon Seller Central"
+  gem.description = "Scrape Amazon Seller Central"
+
+  gem.rubyforge_project = "compactor"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "mechanize", "2.4"
+
+  if RUBY_PLATFORM == "java"
+    gem.add_runtime_dependency "jruby-openssl", '0.7.3'
+    gem.add_runtime_dependency "nokogiri", "1.5.0.beta.2"
+  end
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "mocha", "0.12.1"
+  gem.add_development_dependency "vcr", "~>2.0.0"
+  gem.add_development_dependency "fakeweb"
+  gem.add_development_dependency "rcov", "0.9.11"
+end

data/lib/compactor/extensions.rb

@@ -0,0 +1,23 @@
+class Object
+  def blank?; respond_to?(:empty?) ? empty? : !self; end
+end
+
+module Nokogiri
+  class MissingElement < ::StandardError; end
+
+  module XML
+    class Node
+      def search!(selector)
+        result = search(selector)
+        if result.blank?
+          fail MissingElement.new("No elements for [#{selector}]")
+        end
+        result
+      end
+    end
+  end
+end
+
+class Mechanize::Page
+  def_delegator :parser, :search!, :search!
+end

data/lib/compactor/scraped_row.rb

@@ -0,0 +1,81 @@
+module Compactor
+  module Amazon
+    class ScrapedRow
+      def initialize(node, mechanize)
+        @node = node
+        @mechanize = mechanize
+      end
+
+      def can_download_report?
+        !report_buttons.empty?
+      end
+
+      def report_buttons
+        last_cell.search(".secondarySmallButton").map do |ele|
+          Mechanize::Page::Link.new(ele.parent, @mechanize, @mechanize.page)
+        end
+      end
+
+      def download_report
+        report_url        = report_buttons[0].node["href"]
+        report_identifier = report_buttons[0].node.search(".button_label").text
+        type              = ReportScraper.report_type(report_identifier)
+        response_body     = @mechanize.get(report_url).body
+
+        [type, response_body]
+      end
+
+      def reload
+        table_rows.each do |row|
+          row = ScrapedRow.new(row, @mechanize)
+          return row if row.date_range == date_range
+        end
+
+        nil
+      end
+
+      def request_report
+        button    = last_cell.search(".regenerateButton")[0]
+        button_id = button['id']
+
+        @mechanize.post("/gp/payments-account/redrive.html", {
+          "groupId" => button_id
+        })
+      end
+
+      def ready?
+        div = last_cell.search("div")[-1]
+        text = div.text
+
+        ignorable_periods = ["(Processing)", "(Open)", "In Progress"]
+        !ignorable_periods.any? { |ignore_text| text.include?(ignore_text) &&
+          div.search(".regenerateButton").blank? }
+      end
+
+      def deposit_amount
+        @deposit_amount = fetch_deposit_amount if !@deposit_amount
+
+        @deposit_amount
+      end
+
+      def date_range
+        @date_range ||= @node.search("td:first-child a").text
+      end
+
+      private
+
+      def fetch_deposit_amount
+        deposit_cell = @node.search("td")[-2]
+        deposit_cell ? deposit_cell.text.gsub(/[^0-9\.]/, '').to_f : 0.0
+      end
+
+      def table_rows
+        @mechanize.page.search("tr")
+      end
+
+      def last_cell
+        @last_cell ||= @node.search("td")[-1]
+      end
+    end
+  end
+end

data/lib/compactor/scraper.rb

@@ -0,0 +1,369 @@
+module Compactor
+  module Amazon
+    class AddressParseFailure < StandardError; end
+    class AuthenticationError < StandardError; end
+    class LockedAccountError  < StandardError; end
+    class MissingRow          < StandardError; end
+    class NoMarketplacesError < StandardError; end
+    class NotProAccountError  < StandardError; end
+    class UnknownReportType   < StandardError; end
+
+    ATTEMPTS_BEFORE_GIVING_UP = 15 # give up after 20 minutes
+    MARKETPLACE_HOMEPAGE      = "https://sellercentral.amazon.com/gp/homepage.html"
+    AMAZON_COM_MARKETPLACE_ID = 'ATVPDKIKX0DER'
+
+    class ReportScraper
+      def initialize(email, password, merchant_id)
+        @merchant_id = merchant_id
+
+        @mechanize = Mechanize.new
+        @mechanize.max_file_buffer               = 4 * 1024 * 1024
+        @mechanize.max_history                   = 2
+        @mechanize.agent.http.verify_mode        = OpenSSL::SSL::VERIFY_NONE
+        @mechanize.agent.http.reuse_ssl_sessions = false
+
+        randomize_user_agent!
+        login_to_seller_central email, password
+      end
+
+      def self.submit_form!(form)
+        form.submit
+      rescue Mechanize::ResponseCodeError => e
+        raise ::Compactor::Amazon::NotProAccountError if e.message.include?("403 => Net::HTTPForbidden")
+        raise # any other error just re-raise it
+      end
+
+      def self.authorized_user?
+        message_box_error.empty? ||
+        !message_box_error.text.include?('There was an error with your email/password combination.')
+      end
+
+      def self.merchant_identification(credentials={})
+        @agent = Mechanize.new
+        @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        @agent.agent.http.reuse_ssl_sessions = false
+        @agent.get 'https://sellercentral.amazon.com/gp/mws/registration/register.html'
+        form = @agent.page.forms.first
+        form.email    = credentials[:email]
+        form.password = credentials[:password]
+        submit_form! form
+
+        raise Compactor::Amazon::AuthenticationError unless authorized_user?
+
+        form = @agent.page.forms.first
+        form.developerName   = credentials[:developer_name]
+        form.devMWSAccountId = credentials[:dev_account_id]
+        form.radiobutton_with(:value => 'devAuthorization').checked=true
+        form.submit
+
+        form = @agent.page.forms.first
+        form.checkbox_with(:name => 'agreeCheckBox').checked=true
+        form.checkbox_with(:name => 'understandCheckBox').checked=true
+        form.submit
+
+        @agent.page.forms.first.submit
+        merchant_id    = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[2]/td[1]').text
+        marketplace_id = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[3]/td[1]').text
+
+        [merchant_id, marketplace_id]
+      end
+
+      def marketplaces
+        marketplaces = filter_marketplaces(get_marketplaces)
+        raise NoMarketplacesError if marketplaces.empty?
+
+        marketplaces.map do |account_name, marketplace_id|
+          select_marketplace(marketplace_id)
+          balance = get_balance
+
+          [ account_name, marketplace_id, balance ]
+        end
+      end
+
+      def get_marketplaces
+        @mechanize.get MARKETPLACE_HOMEPAGE
+
+        marketplace_selector = @mechanize.page.search("#marketplaceSelect").first
+        if marketplace_selector
+          result = []
+          marketplace_selector.search("option").each do |ele|
+            name = ele.text
+            marketplace_id = ele["value"]
+            result << [ name, marketplace_id ]
+          end
+          return result
+        end
+
+        marketplace_name = @mechanize.page.search("#market_switch")
+        if marketplace_name
+          return [ [ marketplace_name.text.strip, nil ] ]
+        end
+
+        return []
+      end
+
+      def select_marketplace(marketplace_id)
+        marketplace_id = CGI.escape(marketplace_id)
+        @mechanize.get "https://sellercentral.amazon.com/gp/utilities/set-rainier-prefs.html?ie=UTF8&&marketplaceID=#{marketplace_id}"
+      end
+
+      def get_balance
+        go_to_past_settlements('', '')
+        return 0.0 if page_has_no_results?
+        open_row = report_rows.detect { |row| !row.ready? }
+        return 0.0 if open_row.nil?
+        open_row.deposit_amount
+      end
+
+      def reports(from, to)
+        from, to = parse_dates(from, to)
+        go_to_past_settlements(from, to)
+
+        get_reports
+      end
+
+      def get_orders(order_ids)
+        orders_hash = {}
+        order_ids.each do |order_id|
+          order = {}
+          @mechanize.get order_detail_url(order_id)
+
+          # Get the buyer name
+          begin
+            tr = @mechanize.page.search!("//tr[@class='list-row']/td[@class='data-display-field'][text()=\"Contact Buyer:\"]").first.parent
+            td = tr.search!("td[2]")
+            order["BuyerName"] = td.text.strip
+            td = @mechanize.page.search!("//tr[@class='list-row']/td[@class='data-display-field']/strong[text()='Shipping Address:']").first.parent
+            addr_lines = td.children.map(&:text).reject { |l| l.blank? || l =~ /^Shipping Address/ }
+            order["ShippingAddress"] = parse_address_lines!(addr_lines)
+          rescue Exception => e
+          end
+
+          orders_hash[order_id] = order
+        end
+        orders_hash
+      end
+
+      private
+
+      def self.message_box_error
+        @agent.page.parser.css(".messageboxerror")
+      end
+
+      def slowdown_like_a_human(count)
+        sleep count ** 2
+      end
+
+      def filter_marketplaces(marketplaces)
+        results = []
+
+        name, marketplace_id = marketplaces.detect do |n, m_id|
+          n == 'www.amazon.com' && ( m_id.nil? || m_id == AMAZON_COM_MARKETPLACE_ID )
+        end
+        results << [ 'Amazon Seller Account', AMAZON_COM_MARKETPLACE_ID ] if name
+
+        name, marketplace_id = marketplaces.detect do |n, m_id|
+           n == 'Your Checkout Website' && !m_id.nil?
+        end
+        results << [ 'Checkout By Amazon', marketplace_id ] if name
+
+        results
+      end
+
+      def order_detail_url(order_id)
+        "https://sellercentral.amazon.com/gp/orders-v2/details?ie=UTF8&orderID=#{order_id}"
+      end
+
+      def parse_address_lines!(addr_lines)
+        nbsp = "\302\240"
+        addr_lines = addr_lines.map { |line| line.gsub(nbsp, " ") }
+        # Assume the first line is the name of the buyer, so skip it
+        addr_lines = addr_lines[1..-1].reject { |l| l =~ /^Phone:/ }
+
+        raise AddressParseFailure if addr_lines.empty?
+
+        citystate_line = addr_lines.pop
+        city, remainder = citystate_line.split(/,\s*/)
+
+        raise AddressParseFailure if remainder.nil?
+
+        state, postalcode = remainder.split(/\s+/)
+
+        {
+          'street'     => addr_lines.join('\n'),
+          'city'       => city,
+          'state'      => state,
+          'postalcode' => postalcode
+        }
+      end
+
+      # Pick a random user agent that isn't Mechanize
+      def randomize_user_agent!
+        agents = Mechanize::AGENT_ALIASES.keys.reject{ |k| k == "Mechanize" }
+
+        @mechanize.user_agent =
+          agents.respond_to?(:choice) ? agents.choice : agents.sample
+      end
+
+      def go_to_past_settlements(from, to)
+        from = CGI.escape(from)
+        to   = CGI.escape(to)
+
+        @mechanize.get "https://sellercentral.amazon.com/gp/payments-account/past-settlements.html?endDate=#{to}&startDate=#{from}&pageSize=Ten"
+      end
+
+      def get_reports
+        reports = {}
+        page_num = 0
+        begin
+          get_reports_in_page.each do |report_type, report_streams|
+            reports[report_type] ||= []
+            reports[report_type] << report_streams
+          end
+
+          page_num += 1
+        end while pages_to_parse
+
+        reports.each { |type, streams| streams.flatten! }
+      end
+
+      def self.xml_report?(report_identifier)
+        report_identifier == "Download XML"
+      end
+
+      def self.text_v1_report?(report_identifier)
+        report_identifier == "Download Flat File"
+      end
+
+      def self.text_v2_report?(report_identifier)
+        report_identifier == "Download Flat File V2"
+      end
+
+      # Make this into a hash instead
+      def self.report_type(report_identifier)
+        return :xml  if xml_report?(report_identifier)
+        return :tsv  if text_v1_report?(report_identifier)
+        return :tsv2 if text_v2_report?(report_identifier)
+
+        fail Compactor::Amazon::UnknownReportType
+      end
+
+      def rescue_empty_results(&block)
+        3.times do
+          yield
+          break unless page_has_no_results?
+        end
+      end
+
+      def timeout_fetching_reports(reports_to_watch, reports, count)
+        if count > ATTEMPTS_BEFORE_GIVING_UP
+          reports_downloaded     = reports.map { |type, reports| reports.size }.inject(:+)
+          reports_not_downloaded = reports_to_watch.size
+          total_reports          = reports_not_downloaded + reports_downloaded
+
+          true
+        else
+          false
+        end
+      end
+
+      # Find the report to download from a row, and add it
+      # to a collection of reports. Do this while ensuring
+      # that the current page stays the current page.
+      def add_to_collection(reports, row)
+        @mechanize.transact do
+          report_type, report = row.download_report
+          reports[report_type] ||= []
+          reports[report_type] << report
+        end
+      end
+
+      def get_reports_to_watch(reports_to_watch, reports, count=0)
+        return if reports_to_watch.empty? || timeout_fetching_reports(reports_to_watch, reports, count)
+
+        rescue_empty_results { @mechanize.get @mechanize.page.uri }
+        reports_to_watch.reject! do |row|
+          row = row.reload
+          if row.nil?
+            true
+          elsif row.can_download_report?
+            add_to_collection(reports, row)
+          end
+        end
+
+        slowdown_like_a_human(count)
+        get_reports_to_watch(reports_to_watch, reports, count+1)
+      end
+
+      def pages_to_parse
+        next_button = @mechanize.page.links_with(:text => "Next")[0]
+        return false if next_button.nil?
+
+        next_button.click
+      end
+
+      def report_rows
+        tables = @mechanize.page.search!("#content-main-entities > table")
+        rows = tables[1].search("tr[class]").select do |ele|
+          ["list-row-even","list-row-odd"].include? ele["class"]
+        end
+
+        rows.map { |raw_row| ScrapedRow.new(raw_row, @mechanize) }
+      end
+
+      def page_has_no_results?
+        @mechanize.page.search!(".data-display").text.include? "No results found"
+      end
+
+      def get_reports_in_page
+        reports_to_watch = []
+        reports = {}
+
+        return reports if page_has_no_results?
+
+        report_rows.each do |row|
+          if row.can_download_report?
+            add_to_collection(reports, row)
+          elsif row.ready?
+            @mechanize.transact do
+              row.request_report
+              reports_to_watch << row
+            end
+          end
+        end
+
+        get_reports_to_watch(reports_to_watch, reports)
+
+        reports
+      end
+
+      def parse_dates(from, to)
+        from = Date.parse(from.to_s).strftime("%m/%d/%y")
+        to   = Date.parse(to.to_s).strftime("%m/%d/%y")
+
+        [from, to]
+      end
+
+      def login_to_seller_central(email, password)
+        @mechanize.get MARKETPLACE_HOMEPAGE
+        form = @mechanize.page.forms.first
+        form.email    = email
+        form.password = password
+        form.submit
+
+        raise Compactor::Amazon::AuthenticationError if bad_login?
+        raise Compactor::Amazon::LockedAccountError  if locked_account?
+      end
+
+      def bad_login?
+        !@mechanize.page.parser.css(".messageboxerror").blank? ||
+          @mechanize.page.parser.css('.tiny').text.include?('Sorry, you are not an authorized Seller Central user')
+      end
+
+      def locked_account?
+        alert_box = @mechanize.page.search(".messageboxalert")
+        alert_box && alert_box.text.include?("limited access to your seller account")
+      end
+    end
+  end
+end

data/lib/compactor/version.rb

@@ -0,0 +1,3 @@
+module Compactor
+  VERSION = "0.1.2"
+end