compactor 0.1.2 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +1 -3
- data/Gemfile.lock +1 -1
- data/README.md +1 -1
- data/compactor-0.1.2.gem +0 -0
- data/lib/compactor/scraper.rb +5 -51
- data/lib/compactor/version.rb +1 -1
- data/test/scraper_test.rb +18 -21
- metadata +5 -4
data/.travis.yml
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Compactor
|
2
2
|
|
3
|
-
[](http://travis-ci.org/julio/compactor)
|
4
4
|
|
5
5
|
Scrape Amazon Seller Central
|
6
6
|
|
data/compactor-0.1.2.gem
ADDED
Binary file
|
data/lib/compactor/scraper.rb
CHANGED
@@ -13,9 +13,7 @@ module Compactor
|
|
13
13
|
AMAZON_COM_MARKETPLACE_ID = 'ATVPDKIKX0DER'
|
14
14
|
|
15
15
|
class ReportScraper
|
16
|
-
def initialize(
|
17
|
-
@merchant_id = merchant_id
|
18
|
-
|
16
|
+
def initialize(user_credentials={})
|
19
17
|
@mechanize = Mechanize.new
|
20
18
|
@mechanize.max_file_buffer = 4 * 1024 * 1024
|
21
19
|
@mechanize.max_history = 2
|
@@ -23,49 +21,7 @@ module Compactor
|
|
23
21
|
@mechanize.agent.http.reuse_ssl_sessions = false
|
24
22
|
|
25
23
|
randomize_user_agent!
|
26
|
-
login_to_seller_central email, password
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.submit_form!(form)
|
30
|
-
form.submit
|
31
|
-
rescue Mechanize::ResponseCodeError => e
|
32
|
-
raise ::Compactor::Amazon::NotProAccountError if e.message.include?("403 => Net::HTTPForbidden")
|
33
|
-
raise # any other error just re-raise it
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.authorized_user?
|
37
|
-
message_box_error.empty? ||
|
38
|
-
!message_box_error.text.include?('There was an error with your email/password combination.')
|
39
|
-
end
|
40
|
-
|
41
|
-
def self.merchant_identification(credentials={})
|
42
|
-
@agent = Mechanize.new
|
43
|
-
@agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
44
|
-
@agent.agent.http.reuse_ssl_sessions = false
|
45
|
-
@agent.get 'https://sellercentral.amazon.com/gp/mws/registration/register.html'
|
46
|
-
form = @agent.page.forms.first
|
47
|
-
form.email = credentials[:email]
|
48
|
-
form.password = credentials[:password]
|
49
|
-
submit_form! form
|
50
|
-
|
51
|
-
raise Compactor::Amazon::AuthenticationError unless authorized_user?
|
52
|
-
|
53
|
-
form = @agent.page.forms.first
|
54
|
-
form.developerName = credentials[:developer_name]
|
55
|
-
form.devMWSAccountId = credentials[:dev_account_id]
|
56
|
-
form.radiobutton_with(:value => 'devAuthorization').checked=true
|
57
|
-
form.submit
|
58
|
-
|
59
|
-
form = @agent.page.forms.first
|
60
|
-
form.checkbox_with(:name => 'agreeCheckBox').checked=true
|
61
|
-
form.checkbox_with(:name => 'understandCheckBox').checked=true
|
62
|
-
form.submit
|
63
|
-
|
64
|
-
@agent.page.forms.first.submit
|
65
|
-
merchant_id = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[2]/td[1]').text
|
66
|
-
marketplace_id = @agent.page.parser.xpath('//table[@class="registration-key-table"]/tr[3]/td[1]').text
|
67
|
-
|
68
|
-
[merchant_id, marketplace_id]
|
24
|
+
login_to_seller_central user_credentials[:email], user_credentials[:password]
|
69
25
|
end
|
70
26
|
|
71
27
|
def marketplaces
|
@@ -146,10 +102,6 @@ module Compactor
|
|
146
102
|
|
147
103
|
private
|
148
104
|
|
149
|
-
def self.message_box_error
|
150
|
-
@agent.page.parser.css(".messageboxerror")
|
151
|
-
end
|
152
|
-
|
153
105
|
def slowdown_like_a_human(count)
|
154
106
|
sleep count ** 2
|
155
107
|
end
|
@@ -208,8 +160,10 @@ module Compactor
|
|
208
160
|
def go_to_past_settlements(from, to)
|
209
161
|
from = CGI.escape(from)
|
210
162
|
to = CGI.escape(to)
|
211
|
-
|
212
163
|
@mechanize.get "https://sellercentral.amazon.com/gp/payments-account/past-settlements.html?endDate=#{to}&startDate=#{from}&pageSize=Ten"
|
164
|
+
rescue Mechanize::ResponseCodeError => e
|
165
|
+
raise ::Compactor::Amazon::NotProAccountError if e.message["403 => Net::HTTPForbidden"]
|
166
|
+
raise # any other error just re-raise it as is
|
213
167
|
end
|
214
168
|
|
215
169
|
def get_reports
|
data/lib/compactor/version.rb
CHANGED
data/test/scraper_test.rb
CHANGED
@@ -3,12 +3,9 @@ require File.dirname(__FILE__) + '/../lib/compactor'
|
|
3
3
|
|
4
4
|
class ScraperTest < Test::Unit::TestCase
|
5
5
|
def test_should_raise_error_with_bad_login
|
6
|
-
email = "bad@email.here"
|
7
|
-
password = "invalid"
|
8
|
-
|
9
6
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
10
7
|
assert_raises Compactor::Amazon::AuthenticationError do
|
11
|
-
Compactor::Amazon::ReportScraper.new(email, password
|
8
|
+
Compactor::Amazon::ReportScraper.new(:email => "bad@email.here", :password => "invalid")
|
12
9
|
end
|
13
10
|
end
|
14
11
|
end
|
@@ -33,7 +30,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
33
30
|
|
34
31
|
def test_should_be_able_to_get_buyer_name_and_shipping_address_for_orders
|
35
32
|
VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders") do
|
36
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
33
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
37
34
|
orders = scraper.get_orders(["103-4328675-4697061"])
|
38
35
|
|
39
36
|
assert_equal({
|
@@ -52,7 +49,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
52
49
|
|
53
50
|
def test_should_support_addresses_where_the_street_address_line_does_not_start_with_a_number
|
54
51
|
VCR.use_cassette("AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number") do
|
55
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
52
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
56
53
|
orders = scraper.get_orders(["105-1753716-0471420"])
|
57
54
|
|
58
55
|
assert_equal({
|
@@ -71,7 +68,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
71
68
|
|
72
69
|
def test_should_handle_large_reports
|
73
70
|
VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders_big") do
|
74
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
71
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
75
72
|
scraper.select_marketplace("ATVPDKIKX0DER")
|
76
73
|
assert_nothing_raised do
|
77
74
|
scraper.reports('2012-05-01', '2012-05-08')
|
@@ -81,7 +78,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
81
78
|
|
82
79
|
def test_should_find_no_reports_if_none_exist
|
83
80
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
|
84
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
81
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
85
82
|
reports = scraper.reports("1/1/2012", "3/20/2012")
|
86
83
|
|
87
84
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -90,7 +87,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
90
87
|
|
91
88
|
def test_should_find_reports_with_good_login
|
92
89
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
|
93
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
90
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
94
91
|
reports = scraper.reports("12/28/2011", "12/30/2011")
|
95
92
|
|
96
93
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -99,7 +96,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
99
96
|
|
100
97
|
def test_should_find_reports_in_more_than_on_page
|
101
98
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/multiple_pages") do
|
102
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
99
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
103
100
|
reports = scraper.reports("3/1/2012", "3/21/2012")
|
104
101
|
|
105
102
|
assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
|
@@ -108,7 +105,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
108
105
|
|
109
106
|
def test_should_find_no_reports_if_not_in_date_range
|
110
107
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports") do
|
111
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
108
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
112
109
|
reports = scraper.reports("1/1/2011", "1/8/2011")
|
113
110
|
|
114
111
|
assert_equal( true, reports.all? { |type, reports| reports.empty? } )
|
@@ -117,7 +114,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
117
114
|
|
118
115
|
def test_should_raise_error_if_nothing_to_request
|
119
116
|
VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
|
120
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
117
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
121
118
|
Compactor::Amazon::ReportScraper.stubs(:report_type).raises(Compactor::Amazon::UnknownReportType)
|
122
119
|
|
123
120
|
assert_raises Compactor::Amazon::UnknownReportType do
|
@@ -127,15 +124,15 @@ class ScraperTest < Test::Unit::TestCase
|
|
127
124
|
end
|
128
125
|
|
129
126
|
def test_should_return_balance
|
130
|
-
VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance"
|
131
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
127
|
+
VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance") do
|
128
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
132
129
|
assert_equal(26.14, scraper.get_balance)
|
133
130
|
end
|
134
131
|
end
|
135
132
|
|
136
133
|
def test_should_list_marketplaces_if_single
|
137
134
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces") do
|
138
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
135
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
139
136
|
expected_marketplaces = [["www.amazon.com", nil]]
|
140
137
|
assert_equal expected_marketplaces, scraper.get_marketplaces.sort
|
141
138
|
end
|
@@ -143,7 +140,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
143
140
|
|
144
141
|
def test_should_list_marketplaces_if_several
|
145
142
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces") do
|
146
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
143
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
147
144
|
expected_marketplaces = [["Your Checkout Website", "AZ4B0ZS3LGLX"], ["Your Checkout Website (Sandbox)", "A2SMC08ZTYKXKX"], ["www.amazon.com", "ATVPDKIKX0DER"]]
|
148
145
|
assert_equal expected_marketplaces, scraper.get_marketplaces.sort
|
149
146
|
end
|
@@ -151,7 +148,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
151
148
|
|
152
149
|
def test_should_find_reports_for_current_marketplace
|
153
150
|
VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request") do
|
154
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
151
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
155
152
|
scraper.select_marketplace("AZ4B0ZS3LGLX")
|
156
153
|
reports_1 = scraper.reports("4/1/2012", "4/5/2012")
|
157
154
|
assert_equal(719, ( reports_1[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
|
@@ -166,7 +163,7 @@ class ScraperTest < Test::Unit::TestCase
|
|
166
163
|
def test_should_raise_error_with_bad_login
|
167
164
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
168
165
|
assert_raises Compactor::Amazon::AuthenticationError do
|
169
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away",
|
166
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
170
167
|
end
|
171
168
|
end
|
172
169
|
end
|
@@ -174,15 +171,15 @@ class ScraperTest < Test::Unit::TestCase
|
|
174
171
|
def test_should_raise_error_with_no_email_or_password
|
175
172
|
VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
|
176
173
|
assert_raises Compactor::Amazon::AuthenticationError do
|
177
|
-
scraper = Compactor::Amazon::ReportScraper.new(
|
174
|
+
scraper = Compactor::Amazon::ReportScraper.new({})
|
178
175
|
end
|
179
176
|
end
|
180
177
|
end
|
181
178
|
|
182
179
|
def test_should_raise_error_with_locked_account
|
183
|
-
VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error"
|
180
|
+
VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error") do
|
184
181
|
assert_raises Compactor::Amazon::LockedAccountError do
|
185
|
-
scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test"
|
182
|
+
scraper = Compactor::Amazon::ReportScraper.new(:email => "far@far.away", :password => "test")
|
186
183
|
end
|
187
184
|
end
|
188
185
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: compactor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 6
|
10
|
+
version: 0.1.6
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Julio Santos
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-11-
|
18
|
+
date: 2012-11-05 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: mechanize
|
@@ -125,6 +125,7 @@ files:
|
|
125
125
|
- LICENSE
|
126
126
|
- README.md
|
127
127
|
- Rakefile
|
128
|
+
- compactor-0.1.2.gem
|
128
129
|
- compactor.gemspec
|
129
130
|
- lib/compactor.rb
|
130
131
|
- lib/compactor/extensions.rb
|