stockboy 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +9 -11
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +1 -1
  5. data/lib/stockboy/configuration.rb +1 -1
  6. data/lib/stockboy/configurator.rb +0 -1
  7. data/lib/stockboy/exceptions.rb +14 -10
  8. data/lib/stockboy/job.rb +4 -4
  9. data/lib/stockboy/mapped_record.rb +0 -7
  10. data/lib/stockboy/provider.rb +3 -3
  11. data/lib/stockboy/provider_repeater.rb +0 -1
  12. data/lib/stockboy/providers/ftp.rb +24 -9
  13. data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
  14. data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
  15. data/lib/stockboy/providers/http.rb +0 -8
  16. data/lib/stockboy/providers/imap.rb +11 -10
  17. data/lib/stockboy/providers/soap.rb +3 -2
  18. data/lib/stockboy/readers/csv.rb +3 -3
  19. data/lib/stockboy/readers/fixed_width.rb +28 -21
  20. data/lib/stockboy/readers/spreadsheet.rb +30 -18
  21. data/lib/stockboy/translations/default_zero.rb +1 -1
  22. data/lib/stockboy/translations/integer.rb +2 -2
  23. data/lib/stockboy/version.rb +1 -1
  24. data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
  25. data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
  26. data/spec/spec_helper.rb +2 -6
  27. data/spec/stockboy/candidate_record_spec.rb +20 -11
  28. data/spec/stockboy/configurator_spec.rb +2 -2
  29. data/spec/stockboy/provider_repeater_spec.rb +16 -0
  30. data/spec/stockboy/providers/file_spec.rb +16 -1
  31. data/spec/stockboy/providers/ftp_spec.rb +18 -27
  32. data/spec/stockboy/providers/http_spec.rb +11 -3
  33. data/spec/stockboy/providers/imap_spec.rb +3 -3
  34. data/spec/stockboy/providers/soap_spec.rb +17 -1
  35. data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
  36. data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
  37. data/stockboy.gemspec +1 -0
  38. metadata +23 -4
@@ -42,17 +42,6 @@ module Stockboy
42
42
  end
43
43
 
44
44
  describe "#client" do
45
- it "should open connection to host with username and password" do
46
- expect_connection
47
-
48
- connection = false
49
- provider.client { |f| connection = f }
50
-
51
- expect(connection).to be_a Net::FTP
52
- expect(connection.binary).to be true
53
- expect(connection.passive).to be true
54
- end
55
-
56
45
  it "should return yielded result" do
57
46
  expect_connection
58
47
 
@@ -67,20 +56,20 @@ module Stockboy
67
56
  provider.host = nil
68
57
  provider.data
69
58
 
70
- expect(provider.errors.first).to match /host/
59
+ expect(provider.errors.first).to include "host"
71
60
  end
72
61
 
73
62
  it "adds an error on missing file_name" do
74
63
  provider.file_name = nil
75
64
  provider.data
76
65
 
77
- expect(provider.errors.first).to match /file_name/
66
+ expect(provider.errors.first).to include "file_name"
78
67
  end
79
68
 
80
69
  it "downloads the last matching file" do
81
70
  net_ftp = expect_connection
82
- expect(net_ftp).to receive(:nlst).and_return ['20120101.csv', '20120102.csv']
83
- expect(net_ftp).to receive(:get).with('20120102.csv', nil).and_return "DATA"
71
+ expect(net_ftp).to receive(:list_files).and_return ['20120101.csv', '20120102.csv']
72
+ expect(net_ftp).to receive(:download).with('20120102.csv').and_return "DATA"
84
73
  expect(provider).to receive(:validate_file).and_return true
85
74
 
86
75
  expect(provider.data).to eq "DATA"
@@ -88,12 +77,12 @@ module Stockboy
88
77
 
89
78
  it "skips old files" do
90
79
  net_ftp = expect_connection
91
- expect(net_ftp).to receive(:nlst).and_return ['20120101.csv', '20120102.csv']
92
- expect(net_ftp).to receive(:mtime).with('20120102.csv').and_return(Time.new(2009,01,01))
93
- expect(net_ftp).to_not receive(:get)
80
+ expect(net_ftp).to receive(:list_files).and_return ['20120101.csv', '20120102.csv']
81
+ expect(net_ftp).to receive(:modification_time).with('20120102.csv').and_return(Time.new(2009,01,01))
82
+ expect(net_ftp).to receive(:size).with('20120102.csv').and_return(Time.new(2009,01,01))
83
+ expect(net_ftp).to_not receive(:download)
94
84
 
95
85
  provider.file_newer = Time.new(2010,1,1)
96
-
97
86
  expect(provider.data).to be nil
98
87
  end
99
88
  end
@@ -101,12 +90,12 @@ module Stockboy
101
90
  describe "#matching_file" do
102
91
  it "does not change until cleared" do
103
92
  net_ftp = expect_connection
104
- expect(net_ftp).to receive(:nlst).and_return ["1.csv", "2.csv"]
93
+ expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv"]
105
94
 
106
95
  expect(provider.matching_file).to eq "2.csv"
107
96
 
108
97
  net_ftp = expect_connection
109
- expect(net_ftp).to receive(:nlst).and_return ["1.csv", "2.csv", "3.csv"]
98
+ expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv", "3.csv"]
110
99
 
111
100
  expect(provider.matching_file).to eq "2.csv"
112
101
  provider.clear
@@ -116,13 +105,13 @@ module Stockboy
116
105
 
117
106
  describe "#delete_data" do
118
107
  it "should raise an error when called blindly" do
119
- expect_any_instance_of(Net::FTP).to_not receive(:delete)
108
+ expect_any_instance_of(provider.adapter_class).to_not receive(:delete)
120
109
  expect { provider.delete_data }.to raise_error Stockboy::OutOfSequence
121
110
  end
122
111
 
123
112
  it "should delete the matching file" do
124
113
  net_ftp = expect_connection
125
- expect(net_ftp).to receive(:nlst).and_return ["1.csv", "2.csv"]
114
+ expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv"]
126
115
 
127
116
  expect(provider.matching_file).to eq "2.csv"
128
117
 
@@ -133,10 +122,12 @@ module Stockboy
133
122
  end
134
123
  end
135
124
 
136
- def expect_connection(host="localhost.test", user="a", pass="b")
137
- net_ftp = Net::FTP.new
138
- expect(Net::FTP).to receive(:open).with(host, user, pass).and_yield(net_ftp)
139
- net_ftp
125
+ def expect_connection
126
+ adapter = instance_double(provider.adapter_class)
127
+ expect(adapter).to receive(:open).and_yield(adapter)
128
+
129
+ expect(provider.adapter_class).to receive(:new).with(provider).and_return adapter
130
+ adapter
140
131
  end
141
132
 
142
133
  end
@@ -86,21 +86,21 @@ module Stockboy
86
86
  provider.uri = "http://example.com"
87
87
  provider.method = nil
88
88
  expect(provider).not_to be_valid
89
- expect(provider.errors.first).to match /method/
89
+ expect(provider.errors.first).to include "method"
90
90
  end
91
91
 
92
92
  it "should not be valid without a uri" do
93
93
  provider.uri = ""
94
94
  provider.method = :get
95
95
  expect(provider).not_to be_valid
96
- expect(provider.errors.first).to match /uri/
96
+ expect(provider.errors.first).to include "uri"
97
97
  end
98
98
 
99
99
  it "should require a body for post" do
100
100
  provider.uri = "http://example.com"
101
101
  provider.method = :post
102
102
  expect(provider).not_to be_valid
103
- expect(provider.errors.first).to match /body/
103
+ expect(provider.errors.first).to include "body"
104
104
  end
105
105
  end
106
106
 
@@ -120,6 +120,7 @@ module Stockboy
120
120
 
121
121
  describe "#data" do
122
122
  let(:response) { HTTPI::Response.new(200, {}, '{"success":true}') }
123
+ let(:not_found) { HTTPI::Response.new(404, {}, '{"success":false}') }
123
124
 
124
125
  subject(:provider) do
125
126
  Providers::HTTP.new do |s|
@@ -135,6 +136,13 @@ module Stockboy
135
136
  expect(provider.data).to eq '{"success":true}'
136
137
  end
137
138
 
139
+ it "sets an error on failure" do
140
+ expect(HTTPI).to receive(:request) { not_found }
141
+
142
+ expect(provider.data).to be_nil
143
+ expect(provider.errors.first).to include "HTTP response error: 404"
144
+ end
145
+
138
146
  it "should setup basic auth if a username and password are supplied" do
139
147
  provider.username = "username"
140
148
  provider.password = "password"
@@ -154,7 +154,7 @@ module Stockboy
154
154
 
155
155
  it "should call delete on the message key" do
156
156
  allow(provider).to receive(:client).and_yield(imap)
157
- allow(provider).to receive(:search) { [5] }
157
+ allow(provider).to receive(:find_messages).and_return([5])
158
158
 
159
159
  provider.message_key
160
160
 
@@ -184,9 +184,9 @@ module Stockboy
184
184
  end
185
185
 
186
186
  it "closes connections when catching exceptions" do
187
- net_imap = expect_connection("hhh", "uuu", "ppp", "UNBOX")
187
+ expect_connection("hhh", "uuu", "ppp", "UNBOX")
188
188
  provider.client { |i| raise Net::IMAP::Error }
189
- expect(provider.errors.first).to match /IMAP connection error/
189
+ expect(provider.errors.first).to include "IMAP connection error"
190
190
  end
191
191
 
192
192
  end
@@ -21,6 +21,22 @@ module Stockboy
21
21
  expect(provider.headers).to eq({key: 'k'})
22
22
  end
23
23
 
24
+ describe "logging" do
25
+ before do
26
+ provider.wsdl = "http://api.example.com/?wsdl"
27
+ end
28
+
29
+ it "should be active on debug level" do
30
+ provider.logger.level = Logger::DEBUG
31
+ expect(provider.client.globals[:log]).to be true
32
+ end
33
+
34
+ it "should not be active on info level" do
35
+ provider.logger.level = Logger::INFO
36
+ expect(provider.client.globals[:log]).to be false
37
+ end
38
+ end
39
+
24
40
  describe ".new" do
25
41
  its(:errors) { should be_empty }
26
42
 
@@ -56,7 +72,7 @@ module Stockboy
56
72
  context "without a WSDL document" do
57
73
  it "has error for blank endpoint & WSDL namespace" do
58
74
  provider.valid?
59
- expect(provider.errors.first).to match /endpoint/
75
+ expect(provider.errors.first).to include "endpoint"
60
76
  end
61
77
  end
62
78
  end
@@ -48,5 +48,13 @@ module Stockboy
48
48
  expect(records.last[:age]).to eq '44'
49
49
  end
50
50
 
51
+ context "without specified headers" do
52
+ it "raises an error" do
53
+ reader = described_class.new
54
+
55
+ expect { reader.parse(data) }.to raise_error ArgumentError
56
+ end
57
+ end
58
+
51
59
  end
52
60
  end
@@ -18,13 +18,15 @@ module Stockboy
18
18
  format: :xlsx,
19
19
  sheet: 'Sheet 42',
20
20
  header_row: 5,
21
- headers: %w(X Y Z)
21
+ headers: %w(X Y Z),
22
+ options: {packed: :zip}
22
23
  )
23
24
 
24
25
  expect(reader.format).to eq :xlsx
25
26
  expect(reader.sheet).to eq 'Sheet 42'
26
27
  expect(reader.header_row).to eq 5
27
28
  expect(reader.headers).to eq %w(X Y Z)
29
+ expect(reader.options).to eq({packed: :zip})
28
30
  end
29
31
 
30
32
  it "configures with a block" do
@@ -33,12 +35,14 @@ module Stockboy
33
35
  sheet 'Sheet 42'
34
36
  header_row 5
35
37
  headers %w(X Y Z)
38
+ options({packed: :zip})
36
39
  end
37
40
 
38
41
  expect(reader.format).to eq :xlsx
39
42
  expect(reader.sheet).to eq 'Sheet 42'
40
43
  expect(reader.header_row).to eq 5
41
44
  expect(reader.headers).to eq %w(X Y Z)
45
+ expect(reader.options).to eq({packed: :zip})
42
46
  end
43
47
  end
44
48
 
@@ -48,12 +52,31 @@ module Stockboy
48
52
  context "with an XLS file" do
49
53
  let(:fixture_file) { 'spreadsheets/test_data.xls' }
50
54
 
51
- it "returns an array of hashes" do
55
+ it "uses line 1 for header names and line 2 for first row by default" do
52
56
  reader = described_class.new(format: :xls)
53
57
  data = reader.parse(content)
54
58
 
55
- expect(data).not_to be_empty
56
- data.each { |i| expect(i).to be_a Hash }
59
+ expect(data.first).to eq({"Name" => "Arthur Dent", "Age" => 42})
60
+ expect(data.last).to eq({"Name" => "Marvin", "Age" => 9999999})
61
+ end
62
+
63
+ it "Uses line 1 for first data row if headers are given" do
64
+ reader = described_class.new(format: :xls, headers: ["id", "years"])
65
+ data = reader.parse(content)
66
+
67
+ expect(data.first).to eq({"id" => "Name", "years" => "Age"})
68
+ expect(data.last).to eq({"id" => "Marvin", "years" => 9999999})
69
+ end
70
+ end
71
+
72
+ context "underlying gem other options" do
73
+ let(:fixture_file) { 'spreadsheets/test_data.xls.zip' }
74
+
75
+ it "are passed to underlying library" do
76
+ reader = described_class.new(format: :xls, options: {packed: :zip, file_warning: :ignore})
77
+
78
+ data = reader.parse(content)
79
+ expect(data).to be_an Array
57
80
  end
58
81
  end
59
82
 
@@ -81,39 +104,50 @@ module Stockboy
81
104
  expect(data.last.values).to eq ["Ford", 40]
82
105
  end
83
106
  end
84
- end
85
107
 
86
- describe "#sheet", skip: "Hard to test this due to roo. Needs a test case with fixtures" do
87
- let(:sheets) { ['Towels', 'Lemons'] }
88
- let(:be_selected) { receive(:default_sheet=) }
89
- let(:spreadsheet) { double(:spreadsheet, sheets: sheets) }
90
- before { allow(subject).to receive(:with_spreadsheet).and_yield(spreadsheet) }
108
+ context "with non-first header row" do
109
+ let(:fixture_file) { 'spreadsheets/test_row_options.xls' }
91
110
 
92
- context "with :first" do
93
- before { expect(spreadsheet).to be_selected.with('Towels') }
111
+ it "can use a different header_row" do
112
+ reader = described_class.new(format: :xls, header_row: 4)
113
+ data = reader.parse(content)
94
114
 
95
- it "calls on first sheet by name" do
96
- subject.sheet = :first
97
- subject.parse ""
115
+ expect(data.first).to eq({"Name" => nil, "Age" => nil})
98
116
  end
99
- end
100
117
 
101
- context "with a string" do
102
- before { expect(spreadsheet).to be_selected.with('Towels') }
118
+ it "can set both header_row and first_row" do
119
+ reader = described_class.new(format: :xls, header_row: 4, first_row: 6)
120
+ data = reader.parse(content)
103
121
 
104
- it "passes unchanged" do
105
- subject.sheet = 'Towels'
106
- subject.parse ""
122
+ expect(data.first).to eq({"Name" => "Arthur Dent", "Age" => 42})
107
123
  end
108
124
  end
125
+ end
109
126
 
110
- context "with an integer" do
111
- before { expect(spreadsheet).to be_selected.with('Towels') }
127
+ describe "#sheet" do
128
+ let(:reader) { described_class.new(format: :xls) }
129
+ let(:fixture_file) { 'spreadsheets/test_data_sheets.xls' }
130
+ let(:content) { File.read(fixture_path fixture_file) }
112
131
 
113
- it "gets sheet name" do
114
- subject.sheet = 1
115
- subject.parse ""
116
- end
132
+ it "can specify :last sheet" do
133
+ reader = described_class.new(format: :xls, sheet: :last)
134
+ data = reader.parse content
135
+
136
+ expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
137
+ end
138
+
139
+ it "can specify sheet by name" do
140
+ reader = described_class.new(format: :xls, sheet: 'Planets')
141
+ data = reader.parse content
142
+
143
+ expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
144
+ end
145
+
146
+ it "can specify sheet by number" do
147
+ reader = described_class.new(format: :xls, sheet: 2)
148
+ data = reader.parse content
149
+
150
+ expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
117
151
  end
118
152
  end
119
153
 
@@ -32,4 +32,5 @@ Gem::Specification.new do |s|
32
32
  s.add_runtime_dependency "httpi"
33
33
  s.add_runtime_dependency "mail"
34
34
  s.add_runtime_dependency "activesupport", ">= 3.0"
35
+ s.add_runtime_dependency "net-sftp"
35
36
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: stockboy
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Vit
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-08-18 00:00:00.000000000 Z
11
+ date: 2017-10-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -136,6 +136,20 @@ dependencies:
136
136
  - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '3.0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: net-sftp
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
139
153
  description: Supports importing data over various transports with key-value remapping
140
154
  & normalization
141
155
  email:
@@ -173,6 +187,8 @@ files:
173
187
  - lib/stockboy/providers.rb
174
188
  - lib/stockboy/providers/file.rb
175
189
  - lib/stockboy/providers/ftp.rb
190
+ - lib/stockboy/providers/ftp/ftp_adapter.rb
191
+ - lib/stockboy/providers/ftp/sftp_adapter.rb
176
192
  - lib/stockboy/providers/http.rb
177
193
  - lib/stockboy/providers/imap.rb
178
194
  - lib/stockboy/providers/imap/search_options.rb
@@ -215,6 +231,8 @@ files:
215
231
  - spec/fixtures/soap/get_list/fault.xml
216
232
  - spec/fixtures/soap/get_list/success.xml
217
233
  - spec/fixtures/spreadsheets/test_data.xls
234
+ - spec/fixtures/spreadsheets/test_data.xls.zip
235
+ - spec/fixtures/spreadsheets/test_data_sheets.xls
218
236
  - spec/fixtures/spreadsheets/test_row_options.xls
219
237
  - spec/fixtures/xml/body.xml
220
238
  - spec/spec_helper.rb
@@ -284,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
284
302
  version: '0'
285
303
  requirements: []
286
304
  rubyforge_project: stockboy
287
- rubygems_version: 2.5.1
305
+ rubygems_version: 2.4.5.2
288
306
  signing_key:
289
307
  specification_version: 4
290
308
  summary: Multi-source data normalization library
@@ -299,6 +317,8 @@ test_files:
299
317
  - spec/fixtures/soap/get_list/fault.xml
300
318
  - spec/fixtures/soap/get_list/success.xml
301
319
  - spec/fixtures/spreadsheets/test_data.xls
320
+ - spec/fixtures/spreadsheets/test_data.xls.zip
321
+ - spec/fixtures/spreadsheets/test_data_sheets.xls
302
322
  - spec/fixtures/spreadsheets/test_row_options.xls
303
323
  - spec/fixtures/xml/body.xml
304
324
  - spec/spec_helper.rb
@@ -347,4 +367,3 @@ test_files:
347
367
  - spec/stockboy/translations/us_date_spec.rb
348
368
  - spec/stockboy/translations_spec.rb
349
369
  - spec/stockboy/translator_spec.rb
350
- has_rdoc: false