compactor 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. data/.gitignore +2 -0
  2. data/.travis.yml +9 -0
  3. data/Gemfile +3 -0
  4. data/Gemfile.lock +47 -0
  5. data/LICENSE +22 -0
  6. data/README.md +47 -0
  7. data/Rakefile +48 -0
  8. data/compactor.gemspec +33 -0
  9. data/lib/compactor/extensions.rb +23 -0
  10. data/lib/compactor/scraped_row.rb +81 -0
  11. data/lib/compactor/scraper.rb +369 -0
  12. data/lib/compactor/version.rb +3 -0
  13. data/lib/compactor.rb +5 -0
  14. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml +535 -0
  15. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml +11382 -0
  16. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml +777 -0
  17. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml +1804 -0
  18. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml +13482 -0
  19. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml +1050 -0
  20. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml +822 -0
  21. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml +4223 -0
  22. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml +820 -0
  23. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml +793 -0
  24. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml +800 -0
  25. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml +2948 -0
  26. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml +842 -0
  27. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml +877 -0
  28. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml +1033 -0
  29. data/test/mechanize_extensions_test.rb +16 -0
  30. data/test/scraped_row_test.rb +9 -0
  31. data/test/scraper_test.rb +189 -0
  32. data/test/test_helper.rb +18 -0
  33. metadata +205 -0
data/test/mechanize_extensions_test.rb ADDED
@@ -0,0 +1,16 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class MechanizeExtensionsTest < Test::Unit::TestCase
4
+ def test_should_raise_exception_if_selector_cannot_be_found
5
+ email = "far@far.away"
6
+ password = "password"
7
+
8
+ mechanize = Mechanize.new
9
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
10
+ mechanize.get "https://sellercentral.amazon.com/gp/homepage.html"
11
+ assert_raises Nokogiri::MissingElement do
12
+ mechanize.page.search! "#foo"
13
+ end
14
+ end
15
+ end
16
+ end
data/test/scraped_row_test.rb ADDED
@@ -0,0 +1,9 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+ require File.dirname(__FILE__) + '/../lib/compactor'
3
+
4
+ class ScrapedRowTest < Test::Unit::TestCase
5
+ def test_should_be_nil_on_reload_if_no_more_table_rows_present
6
+ Compactor::Amazon::ScrapedRow.any_instance.stubs(:table_rows).returns([])
7
+ assert_nil Compactor::Amazon::ScrapedRow.new("node", "mechanize").reload
8
+ end
9
+ end
data/test/scraper_test.rb ADDED
@@ -0,0 +1,189 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+ require File.dirname(__FILE__) + '/../lib/compactor'
3
+
4
+ class ScraperTest < Test::Unit::TestCase
5
+ def test_should_raise_error_with_bad_login
6
+ email = "bad@email.here"
7
+ password = "invalid"
8
+
9
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
10
+ assert_raises Compactor::Amazon::AuthenticationError do
11
+ Compactor::Amazon::ReportScraper.new(email, password, "123")
12
+ end
13
+ end
14
+ end
15
+
16
+ def test_should_be_xml_if_button_label_is_Download_XML
17
+ assert_equal :xml, Compactor::Amazon::ReportScraper.report_type("Download XML")
18
+ end
19
+
20
+ def test_should_be_xml_if_button_label_is_Flat_File
21
+ assert_equal :tsv, Compactor::Amazon::ReportScraper.report_type("Download Flat File")
22
+ end
23
+
24
+ def test_should_be_xml_if_button_label_is_Flat_File_V2
25
+ assert_equal :tsv2, Compactor::Amazon::ReportScraper.report_type("Download Flat File V2")
26
+ end
27
+
28
+ def test_should_raise_error_if_type_is_not_identifiable_from_the_button_label
29
+ assert_raises Compactor::Amazon::UnknownReportType do
30
+ Compactor::Amazon::ReportScraper.report_type("Download PDF")
31
+ end
32
+ end
33
+
34
+ def test_should_be_able_to_get_buyer_name_and_shipping_address_for_orders
35
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders") do
36
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
37
+ orders = scraper.get_orders(["103-4328675-4697061"])
38
+
39
+ assert_equal({
40
+ "103-4328675-4697061" => {
41
+ "BuyerName" => "Jared Smith",
42
+ "ShippingAddress" => {
43
+ "street" => "813 FARLEY ST",
44
+ "city" => "MOUNTAIN VIEW",
45
+ "state" => "CA",
46
+ "postalcode" => "94043-3013"
47
+ }
48
+ }
49
+ }, orders)
50
+ end
51
+ end
52
+
53
+ def test_should_support_addresses_where_the_street_address_line_does_not_start_with_a_number
54
+ VCR.use_cassette("AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number") do
55
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
56
+ orders = scraper.get_orders(["105-1753716-0471420"])
57
+
58
+ assert_equal({
59
+ "105-1753716-0471420" => {
60
+ "BuyerName" => "Lisa M Strand",
61
+ "ShippingAddress" => {
62
+ "street" => "W190S6321 Preston Ln",
63
+ "city" => "Muskego",
64
+ "state" => "WI",
65
+ "postalcode" => "53150-8512"
66
+ }
67
+ }
68
+ }, orders)
69
+ end
70
+ end
71
+
72
+ def test_should_handle_large_reports
73
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders_big") do
74
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "FOO")
75
+ scraper.select_marketplace("ATVPDKIKX0DER")
76
+ assert_nothing_raised do
77
+ scraper.reports('2012-05-01', '2012-05-08')
78
+ end
79
+ end
80
+ end
81
+
82
+ def test_should_find_no_reports_if_none_exist
83
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
84
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
85
+ reports = scraper.reports("1/1/2012", "3/20/2012")
86
+
87
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
88
+ end
89
+ end
90
+
91
+ def test_should_find_reports_with_good_login
92
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
93
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
94
+ reports = scraper.reports("12/28/2011", "12/30/2011")
95
+
96
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
97
+ end
98
+ end
99
+
100
+ def test_should_find_reports_in_more_than_on_page
101
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/multiple_pages") do
102
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
103
+ reports = scraper.reports("3/1/2012", "3/21/2012")
104
+
105
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
106
+ end
107
+ end
108
+
109
+ def test_should_find_no_reports_if_not_in_date_range
110
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports") do
111
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
112
+ reports = scraper.reports("1/1/2011", "1/8/2011")
113
+
114
+ assert_equal( true, reports.all? { |type, reports| reports.empty? } )
115
+ end
116
+ end
117
+
118
+ def test_should_raise_error_if_nothing_to_request
119
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
120
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
121
+ Compactor::Amazon::ReportScraper.stubs(:report_type).raises(Compactor::Amazon::UnknownReportType)
122
+
123
+ assert_raises Compactor::Amazon::UnknownReportType do
124
+ scraper.reports("1/1/2012", "3/20/2012")
125
+ end
126
+ end
127
+ end
128
+
129
+ def test_should_return_balance
130
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance", :record => :once) do
131
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
132
+ assert_equal(26.14, scraper.get_balance)
133
+ end
134
+ end
135
+
136
+ def test_should_list_marketplaces_if_single
137
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces") do
138
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
139
+ expected_marketplaces = [["www.amazon.com", nil]]
140
+ assert_equal expected_marketplaces, scraper.get_marketplaces.sort
141
+ end
142
+ end
143
+
144
+ def test_should_list_marketplaces_if_several
145
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces") do
146
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
147
+ expected_marketplaces = [["Your Checkout Website", "AZ4B0ZS3LGLX"], ["Your Checkout Website (Sandbox)", "A2SMC08ZTYKXKX"], ["www.amazon.com", "ATVPDKIKX0DER"]]
148
+ assert_equal expected_marketplaces, scraper.get_marketplaces.sort
149
+ end
150
+ end
151
+
152
+ def test_should_find_reports_for_current_marketplace
153
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request") do
154
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
155
+ scraper.select_marketplace("AZ4B0ZS3LGLX")
156
+ reports_1 = scraper.reports("4/1/2012", "4/5/2012")
157
+ assert_equal(719, ( reports_1[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
158
+ assert_equal("<AmazonOrderID>105-3439340-2677033</AmazonOrderID>", reports_1[:xml].first[719,50])
159
+ scraper.select_marketplace("ATVPDKIKX0DER")
160
+ reports_2 = scraper.reports("4/1/2012", "4/5/2012")
161
+ assert_equal(720, ( reports_2[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
162
+ assert_equal("<AmazonOrderID>105-3231361-4893023</AmazonOrderID>", reports_2[:xml].first[720,50])
163
+ end
164
+ end
165
+
166
+ def test_should_raise_error_with_bad_login
167
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
168
+ assert_raises Compactor::Amazon::AuthenticationError do
169
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "invalid", "123")
170
+ end
171
+ end
172
+ end
173
+
174
+ def test_should_raise_error_with_no_email_or_password
175
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
176
+ assert_raises Compactor::Amazon::AuthenticationError do
177
+ scraper = Compactor::Amazon::ReportScraper.new(nil, nil, "123")
178
+ end
179
+ end
180
+ end
181
+
182
+ def test_should_raise_error_with_locked_account
183
+ VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error", :record => :once) do
184
+ assert_raises Compactor::Amazon::LockedAccountError do
185
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
186
+ end
187
+ end
188
+ end
189
+ end
data/test/test_helper.rb ADDED
@@ -0,0 +1,18 @@
1
+ require "rubygems"
2
+ require "test/unit"
3
+ require "vcr"
4
+ require "mechanize"
5
+ require "compactor"
6
+ require "mocha"
7
+
8
+ VCR.configure do |vcr|
9
+ vcr.cassette_library_dir = 'test/fixtures/vcr_cassettes'
10
+ vcr.hook_into :fakeweb
11
+ end
12
+ FakeWeb.allow_net_connect = false
13
+
14
+ class Compactor::Amazon::ReportScraper
15
+ def slowdown_like_a_human(count)
16
+ # do not slowdown
17
+ end
18
+ end
metadata ADDED
@@ -0,0 +1,205 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: compactor
3
+ version: !ruby/object:Gem::Version
4
+ hash: 31
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 1
9
+ - 2
10
+ version: 0.1.2
11
+ platform: ruby
12
+ authors:
13
+ - Julio Santos
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-11-03 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: mechanize
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - "="
27
+ - !ruby/object:Gem::Version
28
+ hash: 11
29
+ segments:
30
+ - 2
31
+ - 4
32
+ version: "2.4"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: rake
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :development
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: mocha
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - "="
56
+ - !ruby/object:Gem::Version
57
+ hash: 45
58
+ segments:
59
+ - 0
60
+ - 12
61
+ - 1
62
+ version: 0.12.1
63
+ type: :development
64
+ version_requirements: *id003
65
+ - !ruby/object:Gem::Dependency
66
+ name: vcr
67
+ prerelease: false
68
+ requirement: &id004 !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ~>
72
+ - !ruby/object:Gem::Version
73
+ hash: 15
74
+ segments:
75
+ - 2
76
+ - 0
77
+ - 0
78
+ version: 2.0.0
79
+ type: :development
80
+ version_requirements: *id004
81
+ - !ruby/object:Gem::Dependency
82
+ name: fakeweb
83
+ prerelease: false
84
+ requirement: &id005 !ruby/object:Gem::Requirement
85
+ none: false
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ hash: 3
90
+ segments:
91
+ - 0
92
+ version: "0"
93
+ type: :development
94
+ version_requirements: *id005
95
+ - !ruby/object:Gem::Dependency
96
+ name: rcov
97
+ prerelease: false
98
+ requirement: &id006 !ruby/object:Gem::Requirement
99
+ none: false
100
+ requirements:
101
+ - - "="
102
+ - !ruby/object:Gem::Version
103
+ hash: 45
104
+ segments:
105
+ - 0
106
+ - 9
107
+ - 11
108
+ version: 0.9.11
109
+ type: :development
110
+ version_requirements: *id006
111
+ description: Scrape Amazon Seller Central
112
+ email:
113
+ - julio@morgane.com
114
+ executables: []
115
+
116
+ extensions: []
117
+
118
+ extra_rdoc_files: []
119
+
120
+ files:
121
+ - .gitignore
122
+ - .travis.yml
123
+ - Gemfile
124
+ - Gemfile.lock
125
+ - LICENSE
126
+ - README.md
127
+ - Rakefile
128
+ - compactor.gemspec
129
+ - lib/compactor.rb
130
+ - lib/compactor/extensions.rb
131
+ - lib/compactor/scraped_row.rb
132
+ - lib/compactor/scraper.rb
133
+ - lib/compactor/version.rb
134
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
135
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
136
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml
137
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml
138
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml
139
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml
140
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml
141
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml
142
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml
143
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml
144
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml
145
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml
146
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml
147
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml
148
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml
149
+ - test/mechanize_extensions_test.rb
150
+ - test/scraped_row_test.rb
151
+ - test/scraper_test.rb
152
+ - test/test_helper.rb
153
+ homepage: ""
154
+ licenses: []
155
+
156
+ post_install_message:
157
+ rdoc_options: []
158
+
159
+ require_paths:
160
+ - lib
161
+ required_ruby_version: !ruby/object:Gem::Requirement
162
+ none: false
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ hash: 3
167
+ segments:
168
+ - 0
169
+ version: "0"
170
+ required_rubygems_version: !ruby/object:Gem::Requirement
171
+ none: false
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ hash: 3
176
+ segments:
177
+ - 0
178
+ version: "0"
179
+ requirements: []
180
+
181
+ rubyforge_project: compactor
182
+ rubygems_version: 1.8.24
183
+ signing_key:
184
+ specification_version: 3
185
+ summary: Scrape Amazon Seller Central
186
+ test_files:
187
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
188
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
189
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml
190
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml
191
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml
192
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml
193
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml
194
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml
195
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml
196
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml
197
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml
198
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml
199
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml
200
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml
201
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml
202
+ - test/mechanize_extensions_test.rb
203
+ - test/scraped_row_test.rb
204
+ - test/scraper_test.rb
205
+ - test/test_helper.rb