compactor 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. data/.gitignore +2 -0
  2. data/.travis.yml +9 -0
  3. data/Gemfile +3 -0
  4. data/Gemfile.lock +47 -0
  5. data/LICENSE +22 -0
  6. data/README.md +47 -0
  7. data/Rakefile +48 -0
  8. data/compactor.gemspec +33 -0
  9. data/lib/compactor/extensions.rb +23 -0
  10. data/lib/compactor/scraped_row.rb +81 -0
  11. data/lib/compactor/scraper.rb +369 -0
  12. data/lib/compactor/version.rb +3 -0
  13. data/lib/compactor.rb +5 -0
  14. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml +535 -0
  15. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml +11382 -0
  16. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml +777 -0
  17. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml +1804 -0
  18. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml +13482 -0
  19. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml +1050 -0
  20. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml +822 -0
  21. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml +4223 -0
  22. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml +820 -0
  23. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml +793 -0
  24. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml +800 -0
  25. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml +2948 -0
  26. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml +842 -0
  27. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml +877 -0
  28. data/test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml +1033 -0
  29. data/test/mechanize_extensions_test.rb +16 -0
  30. data/test/scraped_row_test.rb +9 -0
  31. data/test/scraper_test.rb +189 -0
  32. data/test/test_helper.rb +18 -0
  33. metadata +205 -0
data/test/mechanize_extensions_test.rb ADDED
@@ -0,0 +1,16 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class MechanizeExtensionsTest < Test::Unit::TestCase
4
+ def test_should_raise_exception_if_selector_cannot_be_found
5
+ email = "far@far.away"
6
+ password = "password"
7
+
8
+ mechanize = Mechanize.new
9
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
10
+ mechanize.get "https://sellercentral.amazon.com/gp/homepage.html"
11
+ assert_raises Nokogiri::MissingElement do
12
+ mechanize.page.search! "#foo"
13
+ end
14
+ end
15
+ end
16
+ end
data/test/scraped_row_test.rb ADDED
@@ -0,0 +1,9 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+ require File.dirname(__FILE__) + '/../lib/compactor'
3
+
4
+ class ScrapedRowTest < Test::Unit::TestCase
5
+ def test_should_be_nil_on_reload_if_no_more_table_rows_present
6
+ Compactor::Amazon::ScrapedRow.any_instance.stubs(:table_rows).returns([])
7
+ assert_nil Compactor::Amazon::ScrapedRow.new("node", "mechanize").reload
8
+ end
9
+ end
data/test/scraper_test.rb ADDED
@@ -0,0 +1,189 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+ require File.dirname(__FILE__) + '/../lib/compactor'
3
+
4
+ class ScraperTest < Test::Unit::TestCase
5
+ def test_should_raise_error_with_bad_login
6
+ email = "bad@email.here"
7
+ password = "invalid"
8
+
9
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
10
+ assert_raises Compactor::Amazon::AuthenticationError do
11
+ Compactor::Amazon::ReportScraper.new(email, password, "123")
12
+ end
13
+ end
14
+ end
15
+
16
+ def test_should_be_xml_if_button_label_is_Download_XML
17
+ assert_equal :xml, Compactor::Amazon::ReportScraper.report_type("Download XML")
18
+ end
19
+
20
+ def test_should_be_xml_if_button_label_is_Flat_File
21
+ assert_equal :tsv, Compactor::Amazon::ReportScraper.report_type("Download Flat File")
22
+ end
23
+
24
+ def test_should_be_xml_if_button_label_is_Flat_File_V2
25
+ assert_equal :tsv2, Compactor::Amazon::ReportScraper.report_type("Download Flat File V2")
26
+ end
27
+
28
+ def test_should_raise_error_if_type_is_not_identifiable_from_the_button_label
29
+ assert_raises Compactor::Amazon::UnknownReportType do
30
+ Compactor::Amazon::ReportScraper.report_type("Download PDF")
31
+ end
32
+ end
33
+
34
+ def test_should_be_able_to_get_buyer_name_and_shipping_address_for_orders
35
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders") do
36
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
37
+ orders = scraper.get_orders(["103-4328675-4697061"])
38
+
39
+ assert_equal({
40
+ "103-4328675-4697061" => {
41
+ "BuyerName" => "Jared Smith",
42
+ "ShippingAddress" => {
43
+ "street" => "813 FARLEY ST",
44
+ "city" => "MOUNTAIN VIEW",
45
+ "state" => "CA",
46
+ "postalcode" => "94043-3013"
47
+ }
48
+ }
49
+ }, orders)
50
+ end
51
+ end
52
+
53
+ def test_should_support_addresses_where_the_street_address_line_does_not_start_with_a_number
54
+ VCR.use_cassette("AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number") do
55
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
56
+ orders = scraper.get_orders(["105-1753716-0471420"])
57
+
58
+ assert_equal({
59
+ "105-1753716-0471420" => {
60
+ "BuyerName" => "Lisa M Strand",
61
+ "ShippingAddress" => {
62
+ "street" => "W190S6321 Preston Ln",
63
+ "city" => "Muskego",
64
+ "state" => "WI",
65
+ "postalcode" => "53150-8512"
66
+ }
67
+ }
68
+ }, orders)
69
+ end
70
+ end
71
+
72
+ def test_should_handle_large_reports
73
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_orders_big") do
74
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "FOO")
75
+ scraper.select_marketplace("ATVPDKIKX0DER")
76
+ assert_nothing_raised do
77
+ scraper.reports('2012-05-01', '2012-05-08')
78
+ end
79
+ end
80
+ end
81
+
82
+ def test_should_find_no_reports_if_none_exist
83
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
84
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
85
+ reports = scraper.reports("1/1/2012", "3/20/2012")
86
+
87
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
88
+ end
89
+ end
90
+
91
+ def test_should_find_reports_with_good_login
92
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/reports_to_request") do
93
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
94
+ reports = scraper.reports("12/28/2011", "12/30/2011")
95
+
96
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
97
+ end
98
+ end
99
+
100
+ def test_should_find_reports_in_more_than_on_page
101
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/multiple_pages") do
102
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
103
+ reports = scraper.reports("3/1/2012", "3/21/2012")
104
+
105
+ assert_equal( true, reports.any? { |type, reports| !reports.empty? } )
106
+ end
107
+ end
108
+
109
+ def test_should_find_no_reports_if_not_in_date_range
110
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports") do
111
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
112
+ reports = scraper.reports("1/1/2011", "1/8/2011")
113
+
114
+ assert_equal( true, reports.all? { |type, reports| reports.empty? } )
115
+ end
116
+ end
117
+
118
+ def test_should_raise_error_if_nothing_to_request
119
+ VCR.use_cassette("AmazonReportScraper/with_good_login/find_reports/no_reports_to_request") do
120
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password17", "123")
121
+ Compactor::Amazon::ReportScraper.stubs(:report_type).raises(Compactor::Amazon::UnknownReportType)
122
+
123
+ assert_raises Compactor::Amazon::UnknownReportType do
124
+ scraper.reports("1/1/2012", "3/20/2012")
125
+ end
126
+ end
127
+ end
128
+
129
+ def test_should_return_balance
130
+ VCR.use_cassette("AmazonReportScraper/with_good_login/get_balance", :record => :once) do
131
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
132
+ assert_equal(26.14, scraper.get_balance)
133
+ end
134
+ end
135
+
136
+ def test_should_list_marketplaces_if_single
137
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces") do
138
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
139
+ expected_marketplaces = [["www.amazon.com", nil]]
140
+ assert_equal expected_marketplaces, scraper.get_marketplaces.sort
141
+ end
142
+ end
143
+
144
+ def test_should_list_marketplaces_if_several
145
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces") do
146
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "FOO")
147
+ expected_marketplaces = [["Your Checkout Website", "AZ4B0ZS3LGLX"], ["Your Checkout Website (Sandbox)", "A2SMC08ZTYKXKX"], ["www.amazon.com", "ATVPDKIKX0DER"]]
148
+ assert_equal expected_marketplaces, scraper.get_marketplaces.sort
149
+ end
150
+ end
151
+
152
+ def test_should_find_reports_for_current_marketplace
153
+ VCR.use_cassette("AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request") do
154
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "password", "123")
155
+ scraper.select_marketplace("AZ4B0ZS3LGLX")
156
+ reports_1 = scraper.reports("4/1/2012", "4/5/2012")
157
+ assert_equal(719, ( reports_1[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
158
+ assert_equal("<AmazonOrderID>105-3439340-2677033</AmazonOrderID>", reports_1[:xml].first[719,50])
159
+ scraper.select_marketplace("ATVPDKIKX0DER")
160
+ reports_2 = scraper.reports("4/1/2012", "4/5/2012")
161
+ assert_equal(720, ( reports_2[:xml].first =~ /<AmazonOrderID>.*<\/AmazonOrderID>/ ) )
162
+ assert_equal("<AmazonOrderID>105-3231361-4893023</AmazonOrderID>", reports_2[:xml].first[720,50])
163
+ end
164
+ end
165
+
166
+ def test_should_raise_error_with_bad_login
167
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
168
+ assert_raises Compactor::Amazon::AuthenticationError do
169
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "invalid", "123")
170
+ end
171
+ end
172
+ end
173
+
174
+ def test_should_raise_error_with_no_email_or_password
175
+ VCR.use_cassette("AmazonReportScraper/with_bad_login/raise_error") do
176
+ assert_raises Compactor::Amazon::AuthenticationError do
177
+ scraper = Compactor::Amazon::ReportScraper.new(nil, nil, "123")
178
+ end
179
+ end
180
+ end
181
+
182
+ def test_should_raise_error_with_locked_account
183
+ VCR.use_cassette("AmazonReportScraper/with_locked_account/raise_error", :record => :once) do
184
+ assert_raises Compactor::Amazon::LockedAccountError do
185
+ scraper = Compactor::Amazon::ReportScraper.new("far@far.away", "test", "123")
186
+ end
187
+ end
188
+ end
189
+ end
data/test/test_helper.rb ADDED
@@ -0,0 +1,18 @@
1
+ require "rubygems"
2
+ require "test/unit"
3
+ require "vcr"
4
+ require "mechanize"
5
+ require "compactor"
6
+ require "mocha"
7
+
8
+ VCR.configure do |vcr|
9
+ vcr.cassette_library_dir = 'test/fixtures/vcr_cassettes'
10
+ vcr.hook_into :fakeweb
11
+ end
12
+ FakeWeb.allow_net_connect = false
13
+
14
+ class Compactor::Amazon::ReportScraper
15
+ def slowdown_like_a_human(count)
16
+ # do not slowdown
17
+ end
18
+ end
metadata ADDED
@@ -0,0 +1,205 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: compactor
3
+ version: !ruby/object:Gem::Version
4
+ hash: 31
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 1
9
+ - 2
10
+ version: 0.1.2
11
+ platform: ruby
12
+ authors:
13
+ - Julio Santos
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-11-03 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: mechanize
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - "="
27
+ - !ruby/object:Gem::Version
28
+ hash: 11
29
+ segments:
30
+ - 2
31
+ - 4
32
+ version: "2.4"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: rake
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :development
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: mocha
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - "="
56
+ - !ruby/object:Gem::Version
57
+ hash: 45
58
+ segments:
59
+ - 0
60
+ - 12
61
+ - 1
62
+ version: 0.12.1
63
+ type: :development
64
+ version_requirements: *id003
65
+ - !ruby/object:Gem::Dependency
66
+ name: vcr
67
+ prerelease: false
68
+ requirement: &id004 !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ~>
72
+ - !ruby/object:Gem::Version
73
+ hash: 15
74
+ segments:
75
+ - 2
76
+ - 0
77
+ - 0
78
+ version: 2.0.0
79
+ type: :development
80
+ version_requirements: *id004
81
+ - !ruby/object:Gem::Dependency
82
+ name: fakeweb
83
+ prerelease: false
84
+ requirement: &id005 !ruby/object:Gem::Requirement
85
+ none: false
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ hash: 3
90
+ segments:
91
+ - 0
92
+ version: "0"
93
+ type: :development
94
+ version_requirements: *id005
95
+ - !ruby/object:Gem::Dependency
96
+ name: rcov
97
+ prerelease: false
98
+ requirement: &id006 !ruby/object:Gem::Requirement
99
+ none: false
100
+ requirements:
101
+ - - "="
102
+ - !ruby/object:Gem::Version
103
+ hash: 45
104
+ segments:
105
+ - 0
106
+ - 9
107
+ - 11
108
+ version: 0.9.11
109
+ type: :development
110
+ version_requirements: *id006
111
+ description: Scrape Amazon Seller Central
112
+ email:
113
+ - julio@morgane.com
114
+ executables: []
115
+
116
+ extensions: []
117
+
118
+ extra_rdoc_files: []
119
+
120
+ files:
121
+ - .gitignore
122
+ - .travis.yml
123
+ - Gemfile
124
+ - Gemfile.lock
125
+ - LICENSE
126
+ - README.md
127
+ - Rakefile
128
+ - compactor.gemspec
129
+ - lib/compactor.rb
130
+ - lib/compactor/extensions.rb
131
+ - lib/compactor/scraped_row.rb
132
+ - lib/compactor/scraper.rb
133
+ - lib/compactor/version.rb
134
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
135
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
136
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml
137
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml
138
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml
139
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml
140
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml
141
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml
142
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml
143
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml
144
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml
145
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml
146
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml
147
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml
148
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml
149
+ - test/mechanize_extensions_test.rb
150
+ - test/scraped_row_test.rb
151
+ - test/scraper_test.rb
152
+ - test/test_helper.rb
153
+ homepage: ""
154
+ licenses: []
155
+
156
+ post_install_message:
157
+ rdoc_options: []
158
+
159
+ require_paths:
160
+ - lib
161
+ required_ruby_version: !ruby/object:Gem::Requirement
162
+ none: false
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ hash: 3
167
+ segments:
168
+ - 0
169
+ version: "0"
170
+ required_rubygems_version: !ruby/object:Gem::Requirement
171
+ none: false
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ hash: 3
176
+ segments:
177
+ - 0
178
+ version: "0"
179
+ requirements: []
180
+
181
+ rubyforge_project: compactor
182
+ rubygems_version: 1.8.24
183
+ signing_key:
184
+ specification_version: 3
185
+ summary: Scrape Amazon Seller Central
186
+ test_files:
187
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_bad_login/raise_error.yml
188
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/multiple_pages.yml
189
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports.yml
190
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/no_reports_to_request.yml
191
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/find_reports/reports_to_request.yml
192
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_balance.yml
193
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders.yml
194
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_big.yml
195
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_logging.yml
196
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/get_orders_with_po_box.yml
197
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/shipping_address_not_starting_with_number.yml
198
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/find_reports/reports_to_request.yml
199
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_multiple_marketplaces/get_marketplaces.yml
200
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_good_login/with_single_marketplaces/get_marketplaces.yml
201
+ - test/fixtures/vcr_cassettes/AmazonReportScraper/with_locked_account/raise_error.yml
202
+ - test/mechanize_extensions_test.rb
203
+ - test/scraped_row_test.rb
204
+ - test/scraper_test.rb
205
+ - test/test_helper.rb