stockboy 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -11
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/lib/stockboy/configuration.rb +1 -1
- data/lib/stockboy/configurator.rb +0 -1
- data/lib/stockboy/exceptions.rb +14 -10
- data/lib/stockboy/job.rb +4 -4
- data/lib/stockboy/mapped_record.rb +0 -7
- data/lib/stockboy/provider.rb +3 -3
- data/lib/stockboy/provider_repeater.rb +0 -1
- data/lib/stockboy/providers/ftp.rb +24 -9
- data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
- data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
- data/lib/stockboy/providers/http.rb +0 -8
- data/lib/stockboy/providers/imap.rb +11 -10
- data/lib/stockboy/providers/soap.rb +3 -2
- data/lib/stockboy/readers/csv.rb +3 -3
- data/lib/stockboy/readers/fixed_width.rb +28 -21
- data/lib/stockboy/readers/spreadsheet.rb +30 -18
- data/lib/stockboy/translations/default_zero.rb +1 -1
- data/lib/stockboy/translations/integer.rb +2 -2
- data/lib/stockboy/version.rb +1 -1
- data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
- data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
- data/spec/spec_helper.rb +2 -6
- data/spec/stockboy/candidate_record_spec.rb +20 -11
- data/spec/stockboy/configurator_spec.rb +2 -2
- data/spec/stockboy/provider_repeater_spec.rb +16 -0
- data/spec/stockboy/providers/file_spec.rb +16 -1
- data/spec/stockboy/providers/ftp_spec.rb +18 -27
- data/spec/stockboy/providers/http_spec.rb +11 -3
- data/spec/stockboy/providers/imap_spec.rb +3 -3
- data/spec/stockboy/providers/soap_spec.rb +17 -1
- data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
- data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
- data/stockboy.gemspec +1 -0
- metadata +23 -4
@@ -42,17 +42,6 @@ module Stockboy
|
|
42
42
|
end
|
43
43
|
|
44
44
|
describe "#client" do
|
45
|
-
it "should open connection to host with username and password" do
|
46
|
-
expect_connection
|
47
|
-
|
48
|
-
connection = false
|
49
|
-
provider.client { |f| connection = f }
|
50
|
-
|
51
|
-
expect(connection).to be_a Net::FTP
|
52
|
-
expect(connection.binary).to be true
|
53
|
-
expect(connection.passive).to be true
|
54
|
-
end
|
55
|
-
|
56
45
|
it "should return yielded result" do
|
57
46
|
expect_connection
|
58
47
|
|
@@ -67,20 +56,20 @@ module Stockboy
|
|
67
56
|
provider.host = nil
|
68
57
|
provider.data
|
69
58
|
|
70
|
-
expect(provider.errors.first).to
|
59
|
+
expect(provider.errors.first).to include "host"
|
71
60
|
end
|
72
61
|
|
73
62
|
it "adds an error on missing file_name" do
|
74
63
|
provider.file_name = nil
|
75
64
|
provider.data
|
76
65
|
|
77
|
-
expect(provider.errors.first).to
|
66
|
+
expect(provider.errors.first).to include "file_name"
|
78
67
|
end
|
79
68
|
|
80
69
|
it "downloads the last matching file" do
|
81
70
|
net_ftp = expect_connection
|
82
|
-
expect(net_ftp).to receive(:
|
83
|
-
expect(net_ftp).to receive(:
|
71
|
+
expect(net_ftp).to receive(:list_files).and_return ['20120101.csv', '20120102.csv']
|
72
|
+
expect(net_ftp).to receive(:download).with('20120102.csv').and_return "DATA"
|
84
73
|
expect(provider).to receive(:validate_file).and_return true
|
85
74
|
|
86
75
|
expect(provider.data).to eq "DATA"
|
@@ -88,12 +77,12 @@ module Stockboy
|
|
88
77
|
|
89
78
|
it "skips old files" do
|
90
79
|
net_ftp = expect_connection
|
91
|
-
expect(net_ftp).to receive(:
|
92
|
-
expect(net_ftp).to receive(:
|
93
|
-
expect(net_ftp).
|
80
|
+
expect(net_ftp).to receive(:list_files).and_return ['20120101.csv', '20120102.csv']
|
81
|
+
expect(net_ftp).to receive(:modification_time).with('20120102.csv').and_return(Time.new(2009,01,01))
|
82
|
+
expect(net_ftp).to receive(:size).with('20120102.csv').and_return(Time.new(2009,01,01))
|
83
|
+
expect(net_ftp).to_not receive(:download)
|
94
84
|
|
95
85
|
provider.file_newer = Time.new(2010,1,1)
|
96
|
-
|
97
86
|
expect(provider.data).to be nil
|
98
87
|
end
|
99
88
|
end
|
@@ -101,12 +90,12 @@ module Stockboy
|
|
101
90
|
describe "#matching_file" do
|
102
91
|
it "does not change until cleared" do
|
103
92
|
net_ftp = expect_connection
|
104
|
-
expect(net_ftp).to receive(:
|
93
|
+
expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv"]
|
105
94
|
|
106
95
|
expect(provider.matching_file).to eq "2.csv"
|
107
96
|
|
108
97
|
net_ftp = expect_connection
|
109
|
-
expect(net_ftp).to receive(:
|
98
|
+
expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv", "3.csv"]
|
110
99
|
|
111
100
|
expect(provider.matching_file).to eq "2.csv"
|
112
101
|
provider.clear
|
@@ -116,13 +105,13 @@ module Stockboy
|
|
116
105
|
|
117
106
|
describe "#delete_data" do
|
118
107
|
it "should raise an error when called blindly" do
|
119
|
-
expect_any_instance_of(
|
108
|
+
expect_any_instance_of(provider.adapter_class).to_not receive(:delete)
|
120
109
|
expect { provider.delete_data }.to raise_error Stockboy::OutOfSequence
|
121
110
|
end
|
122
111
|
|
123
112
|
it "should delete the matching file" do
|
124
113
|
net_ftp = expect_connection
|
125
|
-
expect(net_ftp).to receive(:
|
114
|
+
expect(net_ftp).to receive(:list_files).and_return ["1.csv", "2.csv"]
|
126
115
|
|
127
116
|
expect(provider.matching_file).to eq "2.csv"
|
128
117
|
|
@@ -133,10 +122,12 @@ module Stockboy
|
|
133
122
|
end
|
134
123
|
end
|
135
124
|
|
136
|
-
def expect_connection
|
137
|
-
|
138
|
-
expect(
|
139
|
-
|
125
|
+
def expect_connection
|
126
|
+
adapter = instance_double(provider.adapter_class)
|
127
|
+
expect(adapter).to receive(:open).and_yield(adapter)
|
128
|
+
|
129
|
+
expect(provider.adapter_class).to receive(:new).with(provider).and_return adapter
|
130
|
+
adapter
|
140
131
|
end
|
141
132
|
|
142
133
|
end
|
@@ -86,21 +86,21 @@ module Stockboy
|
|
86
86
|
provider.uri = "http://example.com"
|
87
87
|
provider.method = nil
|
88
88
|
expect(provider).not_to be_valid
|
89
|
-
expect(provider.errors.first).to
|
89
|
+
expect(provider.errors.first).to include "method"
|
90
90
|
end
|
91
91
|
|
92
92
|
it "should not be valid without a uri" do
|
93
93
|
provider.uri = ""
|
94
94
|
provider.method = :get
|
95
95
|
expect(provider).not_to be_valid
|
96
|
-
expect(provider.errors.first).to
|
96
|
+
expect(provider.errors.first).to include "uri"
|
97
97
|
end
|
98
98
|
|
99
99
|
it "should require a body for post" do
|
100
100
|
provider.uri = "http://example.com"
|
101
101
|
provider.method = :post
|
102
102
|
expect(provider).not_to be_valid
|
103
|
-
expect(provider.errors.first).to
|
103
|
+
expect(provider.errors.first).to include "body"
|
104
104
|
end
|
105
105
|
end
|
106
106
|
|
@@ -120,6 +120,7 @@ module Stockboy
|
|
120
120
|
|
121
121
|
describe "#data" do
|
122
122
|
let(:response) { HTTPI::Response.new(200, {}, '{"success":true}') }
|
123
|
+
let(:not_found) { HTTPI::Response.new(404, {}, '{"success":false}') }
|
123
124
|
|
124
125
|
subject(:provider) do
|
125
126
|
Providers::HTTP.new do |s|
|
@@ -135,6 +136,13 @@ module Stockboy
|
|
135
136
|
expect(provider.data).to eq '{"success":true}'
|
136
137
|
end
|
137
138
|
|
139
|
+
it "sets an error on failure" do
|
140
|
+
expect(HTTPI).to receive(:request) { not_found }
|
141
|
+
|
142
|
+
expect(provider.data).to be_nil
|
143
|
+
expect(provider.errors.first).to include "HTTP response error: 404"
|
144
|
+
end
|
145
|
+
|
138
146
|
it "should setup basic auth if a username and password are supplied" do
|
139
147
|
provider.username = "username"
|
140
148
|
provider.password = "password"
|
@@ -154,7 +154,7 @@ module Stockboy
|
|
154
154
|
|
155
155
|
it "should call delete on the message key" do
|
156
156
|
allow(provider).to receive(:client).and_yield(imap)
|
157
|
-
allow(provider).to receive(:
|
157
|
+
allow(provider).to receive(:find_messages).and_return([5])
|
158
158
|
|
159
159
|
provider.message_key
|
160
160
|
|
@@ -184,9 +184,9 @@ module Stockboy
|
|
184
184
|
end
|
185
185
|
|
186
186
|
it "closes connections when catching exceptions" do
|
187
|
-
|
187
|
+
expect_connection("hhh", "uuu", "ppp", "UNBOX")
|
188
188
|
provider.client { |i| raise Net::IMAP::Error }
|
189
|
-
expect(provider.errors.first).to
|
189
|
+
expect(provider.errors.first).to include "IMAP connection error"
|
190
190
|
end
|
191
191
|
|
192
192
|
end
|
@@ -21,6 +21,22 @@ module Stockboy
|
|
21
21
|
expect(provider.headers).to eq({key: 'k'})
|
22
22
|
end
|
23
23
|
|
24
|
+
describe "logging" do
|
25
|
+
before do
|
26
|
+
provider.wsdl = "http://api.example.com/?wsdl"
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should be active on debug level" do
|
30
|
+
provider.logger.level = Logger::DEBUG
|
31
|
+
expect(provider.client.globals[:log]).to be true
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should not be active on info level" do
|
35
|
+
provider.logger.level = Logger::INFO
|
36
|
+
expect(provider.client.globals[:log]).to be false
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
24
40
|
describe ".new" do
|
25
41
|
its(:errors) { should be_empty }
|
26
42
|
|
@@ -56,7 +72,7 @@ module Stockboy
|
|
56
72
|
context "without a WSDL document" do
|
57
73
|
it "has error for blank endpoint & WSDL namespace" do
|
58
74
|
provider.valid?
|
59
|
-
expect(provider.errors.first).to
|
75
|
+
expect(provider.errors.first).to include "endpoint"
|
60
76
|
end
|
61
77
|
end
|
62
78
|
end
|
@@ -48,5 +48,13 @@ module Stockboy
|
|
48
48
|
expect(records.last[:age]).to eq '44'
|
49
49
|
end
|
50
50
|
|
51
|
+
context "without specified headers" do
|
52
|
+
it "raises an error" do
|
53
|
+
reader = described_class.new
|
54
|
+
|
55
|
+
expect { reader.parse(data) }.to raise_error ArgumentError
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
51
59
|
end
|
52
60
|
end
|
@@ -18,13 +18,15 @@ module Stockboy
|
|
18
18
|
format: :xlsx,
|
19
19
|
sheet: 'Sheet 42',
|
20
20
|
header_row: 5,
|
21
|
-
headers: %w(X Y Z)
|
21
|
+
headers: %w(X Y Z),
|
22
|
+
options: {packed: :zip}
|
22
23
|
)
|
23
24
|
|
24
25
|
expect(reader.format).to eq :xlsx
|
25
26
|
expect(reader.sheet).to eq 'Sheet 42'
|
26
27
|
expect(reader.header_row).to eq 5
|
27
28
|
expect(reader.headers).to eq %w(X Y Z)
|
29
|
+
expect(reader.options).to eq({packed: :zip})
|
28
30
|
end
|
29
31
|
|
30
32
|
it "configures with a block" do
|
@@ -33,12 +35,14 @@ module Stockboy
|
|
33
35
|
sheet 'Sheet 42'
|
34
36
|
header_row 5
|
35
37
|
headers %w(X Y Z)
|
38
|
+
options({packed: :zip})
|
36
39
|
end
|
37
40
|
|
38
41
|
expect(reader.format).to eq :xlsx
|
39
42
|
expect(reader.sheet).to eq 'Sheet 42'
|
40
43
|
expect(reader.header_row).to eq 5
|
41
44
|
expect(reader.headers).to eq %w(X Y Z)
|
45
|
+
expect(reader.options).to eq({packed: :zip})
|
42
46
|
end
|
43
47
|
end
|
44
48
|
|
@@ -48,12 +52,31 @@ module Stockboy
|
|
48
52
|
context "with an XLS file" do
|
49
53
|
let(:fixture_file) { 'spreadsheets/test_data.xls' }
|
50
54
|
|
51
|
-
it "
|
55
|
+
it "uses line 1 for header names and line 2 for first row by default" do
|
52
56
|
reader = described_class.new(format: :xls)
|
53
57
|
data = reader.parse(content)
|
54
58
|
|
55
|
-
expect(data).
|
56
|
-
data.
|
59
|
+
expect(data.first).to eq({"Name" => "Arthur Dent", "Age" => 42})
|
60
|
+
expect(data.last).to eq({"Name" => "Marvin", "Age" => 9999999})
|
61
|
+
end
|
62
|
+
|
63
|
+
it "Uses line 1 for first data row if headers are given" do
|
64
|
+
reader = described_class.new(format: :xls, headers: ["id", "years"])
|
65
|
+
data = reader.parse(content)
|
66
|
+
|
67
|
+
expect(data.first).to eq({"id" => "Name", "years" => "Age"})
|
68
|
+
expect(data.last).to eq({"id" => "Marvin", "years" => 9999999})
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context "underlying gem other options" do
|
73
|
+
let(:fixture_file) { 'spreadsheets/test_data.xls.zip' }
|
74
|
+
|
75
|
+
it "are passed to underlying library" do
|
76
|
+
reader = described_class.new(format: :xls, options: {packed: :zip, file_warning: :ignore})
|
77
|
+
|
78
|
+
data = reader.parse(content)
|
79
|
+
expect(data).to be_an Array
|
57
80
|
end
|
58
81
|
end
|
59
82
|
|
@@ -81,39 +104,50 @@ module Stockboy
|
|
81
104
|
expect(data.last.values).to eq ["Ford", 40]
|
82
105
|
end
|
83
106
|
end
|
84
|
-
end
|
85
107
|
|
86
|
-
|
87
|
-
|
88
|
-
let(:be_selected) { receive(:default_sheet=) }
|
89
|
-
let(:spreadsheet) { double(:spreadsheet, sheets: sheets) }
|
90
|
-
before { allow(subject).to receive(:with_spreadsheet).and_yield(spreadsheet) }
|
108
|
+
context "with non-first header row" do
|
109
|
+
let(:fixture_file) { 'spreadsheets/test_row_options.xls' }
|
91
110
|
|
92
|
-
|
93
|
-
|
111
|
+
it "can use a different header_row" do
|
112
|
+
reader = described_class.new(format: :xls, header_row: 4)
|
113
|
+
data = reader.parse(content)
|
94
114
|
|
95
|
-
|
96
|
-
subject.sheet = :first
|
97
|
-
subject.parse ""
|
115
|
+
expect(data.first).to eq({"Name" => nil, "Age" => nil})
|
98
116
|
end
|
99
|
-
end
|
100
117
|
|
101
|
-
|
102
|
-
|
118
|
+
it "can set both header_row and first_row" do
|
119
|
+
reader = described_class.new(format: :xls, header_row: 4, first_row: 6)
|
120
|
+
data = reader.parse(content)
|
103
121
|
|
104
|
-
|
105
|
-
subject.sheet = 'Towels'
|
106
|
-
subject.parse ""
|
122
|
+
expect(data.first).to eq({"Name" => "Arthur Dent", "Age" => 42})
|
107
123
|
end
|
108
124
|
end
|
125
|
+
end
|
109
126
|
|
110
|
-
|
111
|
-
|
127
|
+
describe "#sheet" do
|
128
|
+
let(:reader) { described_class.new(format: :xls) }
|
129
|
+
let(:fixture_file) { 'spreadsheets/test_data_sheets.xls' }
|
130
|
+
let(:content) { File.read(fixture_path fixture_file) }
|
112
131
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
132
|
+
it "can specify :last sheet" do
|
133
|
+
reader = described_class.new(format: :xls, sheet: :last)
|
134
|
+
data = reader.parse content
|
135
|
+
|
136
|
+
expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
|
137
|
+
end
|
138
|
+
|
139
|
+
it "can specify sheet by name" do
|
140
|
+
reader = described_class.new(format: :xls, sheet: 'Planets')
|
141
|
+
data = reader.parse content
|
142
|
+
|
143
|
+
expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
|
144
|
+
end
|
145
|
+
|
146
|
+
it "can specify sheet by number" do
|
147
|
+
reader = described_class.new(format: :xls, sheet: 2)
|
148
|
+
data = reader.parse content
|
149
|
+
|
150
|
+
expect(data.first.values).to eq(["Earth", "Mostly Harmless"])
|
117
151
|
end
|
118
152
|
end
|
119
153
|
|
data/stockboy.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: stockboy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Vit
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -136,6 +136,20 @@ dependencies:
|
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '3.0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: net-sftp
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
139
153
|
description: Supports importing data over various transports with key-value remapping
|
140
154
|
& normalization
|
141
155
|
email:
|
@@ -173,6 +187,8 @@ files:
|
|
173
187
|
- lib/stockboy/providers.rb
|
174
188
|
- lib/stockboy/providers/file.rb
|
175
189
|
- lib/stockboy/providers/ftp.rb
|
190
|
+
- lib/stockboy/providers/ftp/ftp_adapter.rb
|
191
|
+
- lib/stockboy/providers/ftp/sftp_adapter.rb
|
176
192
|
- lib/stockboy/providers/http.rb
|
177
193
|
- lib/stockboy/providers/imap.rb
|
178
194
|
- lib/stockboy/providers/imap/search_options.rb
|
@@ -215,6 +231,8 @@ files:
|
|
215
231
|
- spec/fixtures/soap/get_list/fault.xml
|
216
232
|
- spec/fixtures/soap/get_list/success.xml
|
217
233
|
- spec/fixtures/spreadsheets/test_data.xls
|
234
|
+
- spec/fixtures/spreadsheets/test_data.xls.zip
|
235
|
+
- spec/fixtures/spreadsheets/test_data_sheets.xls
|
218
236
|
- spec/fixtures/spreadsheets/test_row_options.xls
|
219
237
|
- spec/fixtures/xml/body.xml
|
220
238
|
- spec/spec_helper.rb
|
@@ -284,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
284
302
|
version: '0'
|
285
303
|
requirements: []
|
286
304
|
rubyforge_project: stockboy
|
287
|
-
rubygems_version: 2.5.
|
305
|
+
rubygems_version: 2.4.5.2
|
288
306
|
signing_key:
|
289
307
|
specification_version: 4
|
290
308
|
summary: Multi-source data normalization library
|
@@ -299,6 +317,8 @@ test_files:
|
|
299
317
|
- spec/fixtures/soap/get_list/fault.xml
|
300
318
|
- spec/fixtures/soap/get_list/success.xml
|
301
319
|
- spec/fixtures/spreadsheets/test_data.xls
|
320
|
+
- spec/fixtures/spreadsheets/test_data.xls.zip
|
321
|
+
- spec/fixtures/spreadsheets/test_data_sheets.xls
|
302
322
|
- spec/fixtures/spreadsheets/test_row_options.xls
|
303
323
|
- spec/fixtures/xml/body.xml
|
304
324
|
- spec/spec_helper.rb
|
@@ -347,4 +367,3 @@ test_files:
|
|
347
367
|
- spec/stockboy/translations/us_date_spec.rb
|
348
368
|
- spec/stockboy/translations_spec.rb
|
349
369
|
- spec/stockboy/translator_spec.rb
|
350
|
-
has_rdoc: false
|