compactor 0.1.2 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -3
- data/Gemfile.lock +1 -1
- data/README.md +1 -1
- data/compactor-0.1.2.gem +0 -0
- data/lib/compactor/scraper.rb +5 -51
- data/lib/compactor/version.rb +1 -1
- data/test/scraper_test.rb +18 -21
- metadata +5 -4
data/.travis.yml
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Compactor
|
2
2
|
|
3
|
-
[![Build Status](https://secure.travis-ci.org/julio/
|
3
|
+
[![Build Status](https://secure.travis-ci.org/julio/compactor.png)](http://travis-ci.org/julio/compactor)
|
4
4
|
|
5
5
|
Scrape Amazon Seller Central
|
6
6
|
|
data/compactor-0.1.2.gem
ADDED
Binary file
|
data/lib/compactor/scraper.rb
CHANGED
@@ -13,9 +13,7 @@ module Compactor
|
|
13
13
|
AMAZON_COM_MARKETPLACE_ID = 'ATVPDKIKX0DER'
|
14
14
|
|
15
15
|
class ReportScraper
|
16
|
-
def initialize(
|
17
|
-
@merchant_id = merchant_id
|
18
|
-
|
16
|
+
def initialize(user_credentials={})
|
19
17
|
@mechanize = Mechanize.new
|
20
18
|
@mechanize.max_file_buffer = 4 * 1024 * 1024
|
21
19
|
@mechanize.max_history = 2
|
@@ -23,49 +21,7 @@ module Compactor
|
|
23
21
|
@mechanize.agent.http.reuse_ssl_sessions = false
|
24
22
|
|
25
23
|
randomize_user_agent!
|
26
|
-
login_to_seller_central email, password
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.submit_form!(form)
|
30
|
-
form.submit
|
31
|
-
rescue Mechanize::ResponseCodeError => e
|
32
|
-
raise ::Compactor::Amazon::NotProAccountError if e.message.include?("403 => Net::HTTPForbidden")
|
33
|
-
raise # any other error just re-raise it
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.authorized_user?
|
37
|
-
message_box_error.empty? ||
|
38
|
-
!message_box_error.text.include?('There was an error with your email/password combination.')
|
39
|
-
end
|
40
|
-
|
41
|
-
def self.merchant_identification(credentials={})
|
42
|
-
@agent = Mechanize.new
|
43
|
-
@agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
44
|
-
@agent.agent.http.reuse_ssl_sessions = false
|
45
|
-
@agent.get 'https://sellercentral.amazon.com/gp/mws/registration/register.html'
|
46
|
-
form = @agent.page.forms.first
|
47
|
-
form.email = credentials[:email]
|
48
|
-
form.password = credentials[:password]
|
49
|
-
submit_form! form
|
50
|
-
|
51
|
-
raise Compactor::Amazon::AuthenticationError unless authorized_user?
|
52
|
-
|
53
|
-
form = @agent.page.forms.first
|
54
|
-
form.developerName = credentials[:developer_name]
|
55
|
-
form.devMWSAccountId = credentials[:dev_account_id]
|
56
|
-
form.radiobutton_with(:value => 'devAuthorization').checked=true
|
57
|
-
form.submit
|
58
|
-
|
59
|
-
form = @agent.page.forms.first
|
60
|
-
form.checkbox_with(:name => 'agreeCheckBox').checked=true
|
61
|
-
form.checkbox_with(:name => 'understandCheckBox').checked=true
|
62
|
-
form.submit
|
63
|
-
|
64
|
-
@agent.page.forms.first.submit
|
65
|
-
merchant_id = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[2]/td[1]').text
|
66
|
-
marketplace_id = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[3]/td[1]').text
|
67
|
-
|
68
|
-
[merchant_id, marketplace_id]
|
24
|
+
login_to_seller_central user_credentials[:email], user_credentials[:password]
|
69
25
|
end
|
70
26
|
|
71
27
|
def marketplaces
|
@@ -146,10 +102,6 @@ module Compactor
|
|
146
102
|
|
147
103
|
private
|
148
104
|
|
149
|
-
def self.message_box_error
|
150
|
-
@agent.page.parser.css(".messageboxerror")
|
151
|
-
end
|
152
|
-
|
153
105
|
def slowdown_like_a_human(count)
|
154
106
|
sleep count ** 2
|
155
107
|
end
|
@@ -208,8 +160,10 @@ module Compactor
|
|
208
160
|
def go_to_past_settlements(from, to)
|
209
161
|
from = CGI.escape(from)
|
210
162
|
to = CGI.escape(to)
|
211
|
-
|
212
163
|
@mechanize.get "https://sellercentral.amazon.com/gp/payments-account/past-settlements.html?endDate=#{to}&startDate=#{from}&pageSize=Ten"
|
164
|
+
rescue Mechanize::ResponseCodeError => e
|
165
|
+
raise ::Compactor::Amazon::NotProAccountError if e.message["403 => Net::HTTPForbidden"]
|
166
|
+
raise # any other error just re-raise it as is
|
213
167
|
end
|
214
168
|
|
215
169
|
def get_reports
|
data/lib/compactor/version.rb
CHANGED
data/test/scraper_test.rb
CHANGED
@@ -3,12 +3,9 @@ require File.dirname(__FILE__) + '/../lib/compactor'
|
|
3
3
|
|
4
4
|
class ScraperTest < Test::Unit::TestCase
|
5
5
|
def test_should_raise_error_with_bad_login
|
6
|
-
email = "bad@email.here"
|
7
|
-
password = "invalid"
|
8
|
-
|
9
6
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
10
7
|
assert_raises Compactor::Amazon::AuthenticationError do
|
11
|
-
Compactor::Amazon::ReportScraper.new(email, password
|
8
|
+
Compactor::Amazon::ReportScraper.new(:email => "bad@email.here", :password => "invalid")
|
12
9
|
end
|
13
10
|
end
|
14
11
|
end
|
@@ -33,7 +30,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
33
30
|
|
34
31
|
def test_should_be_able_to_get_buyer_name_and_shipping_address_for_orders
|
35
32
|
VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders") do
|
36
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
33
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
37
34
|
orders = scraper.get_orders(["103-4328675-4697061"])
|
38
35
|
|
39
36
|
assert_equal({
|
@@ -52,7 +49,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
52
49
|
|
53
50
|
def test_should_support_addresses_where_the_street_address_line_does_not_start_with_a_number
|
54
51
|
VCR.use_cassette("AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number") do
|
55
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
52
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
56
53
|
orders = scraper.get_orders(["105-1753716-0471420"])
|
57
54
|
|
58
55
|
assert_equal({
|
@@ -71,7 +68,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
71
68
|
|
72
69
|
def test_should_handle_large_reports
|
73
70
|
VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders_big") do
|
74
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
71
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
75
72
|
scraper.select_marketplace("ATVPDKIKX0DER")
|
76
73
|
assert_nothing_raised do
|
77
74
|
scraper.reports('2012-05-01', '2012-05-08')
|
@@ -81,7 +78,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
81
78
|
|
82
79
|
def test_should_find_no_reports_if_none_exist
|
83
80
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
|
84
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
81
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
85
82
|
reports = scraper.reports("1/1/2012", "3/20/2012")
|
86
83
|
|
87
84
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -90,7 +87,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
90
87
|
|
91
88
|
def test_should_find_reports_with_good_login
|
92
89
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
|
93
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
90
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
94
91
|
reports = scraper.reports("12/28/2011", "12/30/2011")
|
95
92
|
|
96
93
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -99,7 +96,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
99
96
|
|
100
97
|
def test_should_find_reports_in_more_than_on_page
|
101
98
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/multiple_pages") do
|
102
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
99
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
103
100
|
reports = scraper.reports("3/1/2012", "3/21/2012")
|
104
101
|
|
105
102
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -108,7 +105,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
108
105
|
|
109
106
|
def test_should_find_no_reports_if_not_in_date_range
|
110
107
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports") do
|
111
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
108
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
112
109
|
reports = scraper.reports("1/1/2011", "1/8/2011")
|
113
110
|
|
114
111
|
assert_equal( true, reports.all? { |type, reports| reports.empty? } )
|
@@ -117,7 +114,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
117
114
|
|
118
115
|
def test_should_raise_error_if_nothing_to_request
|
119
116
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
|
120
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
117
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
121
118
|
Compactor::Amazon::ReportScraper.stubs(:report_type).raises(Compactor::Amazon::UnknownReportType)
|
122
119
|
|
123
120
|
assert_raises Compactor::Amazon::UnknownReportType do
|
@@ -127,15 +124,15 @@ class ScraperTest < Test::Unit::TestCase
|
|
127
124
|
end
|
128
125
|
|
129
126
|
def test_should_return_balance
|
130
|
-
VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance"
|
131
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
127
|
+
VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance") do
|
128
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
132
129
|
assert_equal(26.14, scraper.get_balance)
|
133
130
|
end
|
134
131
|
end
|
135
132
|
|
136
133
|
def test_should_list_marketplaces_if_single
|
137
134
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces") do
|
138
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
135
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
139
136
|
expected_marketplaces = [["www.amazon.com", nil]]
|
140
137
|
assert_equal expected_marketplaces, scraper.get_marketplaces.sort
|
141
138
|
end
|
@@ -143,7 +140,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
143
140
|
|
144
141
|
def test_should_list_marketplaces_if_several
|
145
142
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces") do
|
146
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
143
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
147
144
|
expected_marketplaces = [["Your Checkout Website", "AZ4B0ZS3LGLX"], ["Your Checkout Website (Sandbox)", "A2SMC08ZTYKXKX"], ["www.amazon.com", "ATVPDKIKX0DER"]]
|
148
145
|
assert_equal expected_marketplaces, scraper.get_marketplaces.sort
|
149
146
|
end
|
@@ -151,7 +148,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
151
148
|
|
152
149
|
def test_should_find_reports_for_current_marketplace
|
153
150
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request") do
|
154
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
151
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
155
152
|
scraper.select_marketplace("AZ4B0ZS3LGLX")
|
156
153
|
reports_1 = scraper.reports("4/1/2012", "4/5/2012")
|
157
154
|
assert_equal(719, ( reports_1[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
|
@@ -166,7 +163,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
166
163
|
def test_should_raise_error_with_bad_login
|
167
164
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
168
165
|
assert_raises Compactor::Amazon::AuthenticationError do
|
169
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
166
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
170
167
|
end
|
171
168
|
end
|
172
169
|
end
|
@@ -174,15 +171,15 @@ class ScraperTest < Test::Unit::TestCase
|
|
174
171
|
def test_should_raise_error_with_no_email_or_password
|
175
172
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
176
173
|
assert_raises Compactor::Amazon::AuthenticationError do
|
177
|
-
scraper = Compactor::Amazon::ReportScraper.new(
|
174
|
+
scraper = Compactor::Amazon::ReportScraper.new({})
|
178
175
|
end
|
179
176
|
end
|
180
177
|
end
|
181
178
|
|
182
179
|
def test_should_raise_error_with_locked_account
|
183
|
-
VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error"
|
180
|
+
VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error") do
|
184
181
|
assert_raises Compactor::Amazon::LockedAccountError do
|
185
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
182
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
186
183
|
end
|
187
184
|
end
|
188
185
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: compactor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 6
|
10
|
+
version: 0.1.6
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Julio Santos
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-11-
|
18
|
+
date: 2012-11-05 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: mechanize
|
@@ -125,6 +125,7 @@ files:
|
|
125
125
|
- LICENSE
|
126
126
|
- README.md
|
127
127
|
- Rakefile
|
128
|
+
- compactor-0.1.2.gem
|
128
129
|
- compactor.gemspec
|
129
130
|
- lib/compactor.rb
|
130
131
|
- lib/compactor/extensions.rb
|