stockboy 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -11
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/lib/stockboy/configuration.rb +1 -1
- data/lib/stockboy/configurator.rb +0 -1
- data/lib/stockboy/exceptions.rb +14 -10
- data/lib/stockboy/job.rb +4 -4
- data/lib/stockboy/mapped_record.rb +0 -7
- data/lib/stockboy/provider.rb +3 -3
- data/lib/stockboy/provider_repeater.rb +0 -1
- data/lib/stockboy/providers/ftp.rb +24 -9
- data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
- data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
- data/lib/stockboy/providers/http.rb +0 -8
- data/lib/stockboy/providers/imap.rb +11 -10
- data/lib/stockboy/providers/soap.rb +3 -2
- data/lib/stockboy/readers/csv.rb +3 -3
- data/lib/stockboy/readers/fixed_width.rb +28 -21
- data/lib/stockboy/readers/spreadsheet.rb +30 -18
- data/lib/stockboy/translations/default_zero.rb +1 -1
- data/lib/stockboy/translations/integer.rb +2 -2
- data/lib/stockboy/version.rb +1 -1
- data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
- data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
- data/spec/spec_helper.rb +2 -6
- data/spec/stockboy/candidate_record_spec.rb +20 -11
- data/spec/stockboy/configurator_spec.rb +2 -2
- data/spec/stockboy/provider_repeater_spec.rb +16 -0
- data/spec/stockboy/providers/file_spec.rb +16 -1
- data/spec/stockboy/providers/ftp_spec.rb +18 -27
- data/spec/stockboy/providers/http_spec.rb +11 -3
- data/spec/stockboy/providers/imap_spec.rb +3 -3
- data/spec/stockboy/providers/soap_spec.rb +17 -1
- data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
- data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
- data/stockboy.gemspec +1 -0
- metadata +23 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b36c08f870f1306e9c36cb125438bf8e01b3e6dc
|
4
|
+
data.tar.gz: e997f70d91635b9456ed6c0758e981ea7f2e16ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0a129da6b911fa8812706140192419782ce1e09e5ea26bc8a161dcf1006a918ff7ab6cbd94a94e1b6f4c279d9d092146e87386b5dd64a837980a03ba2682e48a
|
7
|
+
data.tar.gz: e5fd24fb0698cfb2d8488f35587ea49b0ac2d7bc0d39422a17fd5218a0974429fb1baf214e5f00f5295ec4ba309dade51e5a9a1d2b9048e5a9b0db6a51e19773
|
data/.travis.yml
CHANGED
@@ -3,22 +3,20 @@ bundler_args: --without debug --without doc
|
|
3
3
|
env:
|
4
4
|
- CI=true
|
5
5
|
rvm:
|
6
|
-
-
|
7
|
-
-
|
8
|
-
-
|
9
|
-
-
|
10
|
-
-
|
11
|
-
- "jruby-head"
|
6
|
+
- 2.2
|
7
|
+
- 2.3
|
8
|
+
- 2.4
|
9
|
+
- ruby-head
|
10
|
+
- jruby-head
|
12
11
|
before_install:
|
13
12
|
- gem install bundler
|
14
|
-
script:
|
13
|
+
script:
|
14
|
+
- bundle exec rspec spec
|
15
|
+
- bundle exec codeclimate-test-reporter
|
15
16
|
matrix:
|
16
17
|
fast_finish: true
|
17
18
|
allow_failures:
|
18
|
-
- rvm:
|
19
|
-
- rvm: "2.0.0"
|
20
|
-
- rvm: "rbx"
|
21
|
-
- rvm: "jruby-head"
|
19
|
+
- rvm: jruby-head
|
22
20
|
addons:
|
23
21
|
code_climate:
|
24
22
|
repo_token: 7591072ad04d7a8ee788ce9259baa284724740cab1ae9ca94c162a583acd8f10
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 1.3.0 / 2017-10-26
|
4
|
+
|
5
|
+
* [FEATURE] Add `secure` option for FTP to use SFTP
|
6
|
+
* [FEATURE] Update test matrix for recent ruby versions
|
7
|
+
* [ENHANCEMENT] Log SOAP provider output on debug level
|
8
|
+
* [ENHANCEMENT] Let pick accept proc with single arg for list
|
9
|
+
* [BUGFIX] Fix defaults for spreadsheet row options
|
10
|
+
* [BUGFIX] Fix spreadsheet option initialization
|
11
|
+
|
3
12
|
## 1.2.1 / 2016-08-02
|
4
13
|
|
5
14
|
* [BUGFIX] Repeater would error with zero yielded data iterations
|
data/README.md
CHANGED
@@ -236,7 +236,7 @@ with the "as" option.
|
|
236
236
|
* [`:decimal`][deci]
|
237
237
|
Numeric strings to `BigDecimal` (e.g. prices)
|
238
238
|
* [`:integer`][intg]
|
239
|
-
Numeric strings to `
|
239
|
+
Numeric strings to `Integer` integers
|
240
240
|
* [`:string`][stri]
|
241
241
|
Clean strings with leading/trailing whitespace trimmed
|
242
242
|
* [`:or_empty`][dest]
|
data/lib/stockboy/exceptions.rb
CHANGED
@@ -1,20 +1,24 @@
|
|
1
1
|
module Stockboy
|
2
2
|
class OutOfSequence < StandardError; end
|
3
3
|
|
4
|
-
|
5
|
-
class TranslationError < StandardError
|
4
|
+
class TranslationError < StandardError
|
6
5
|
|
7
|
-
|
6
|
+
def initialize (key, record)
|
8
7
|
@key = key
|
9
8
|
@record = record
|
10
|
-
|
11
|
-
|
12
|
-
def message
|
13
|
-
"Attribute [#{key}] caused #{cause.message}"
|
14
|
-
end
|
9
|
+
@cause = $!
|
10
|
+
end
|
15
11
|
|
16
|
-
|
17
|
-
|
12
|
+
def message
|
13
|
+
reason = @cause && @cause.message || super
|
14
|
+
"Attribute [#{key}] caused #{reason}"
|
15
|
+
end
|
18
16
|
|
17
|
+
def backtrace
|
18
|
+
@cause && @cause.backtrace || super
|
19
19
|
end
|
20
|
+
|
21
|
+
attr_reader :key
|
22
|
+
attr_reader :record
|
23
|
+
end
|
20
24
|
end
|
data/lib/stockboy/job.rb
CHANGED
@@ -127,7 +127,7 @@ module Stockboy
|
|
127
127
|
# Count of all processed records
|
128
128
|
#
|
129
129
|
# @!attribute [r] total_records
|
130
|
-
# @return [
|
130
|
+
# @return [Integer]
|
131
131
|
#
|
132
132
|
def total_records
|
133
133
|
@all_records.size
|
@@ -135,7 +135,7 @@ module Stockboy
|
|
135
135
|
|
136
136
|
# Counts of processed records grouped by filter key
|
137
137
|
#
|
138
|
-
# @return [Hash{Symbol=>
|
138
|
+
# @return [Hash{Symbol=>Integer}]
|
139
139
|
#
|
140
140
|
def record_counts
|
141
141
|
@records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
|
@@ -267,9 +267,9 @@ module Stockboy
|
|
267
267
|
end
|
268
268
|
end
|
269
269
|
|
270
|
-
def with_query_caching
|
270
|
+
def with_query_caching
|
271
271
|
if defined? ActiveRecord
|
272
|
-
ActiveRecord::Base.cache
|
272
|
+
ActiveRecord::Base.cache { yield }
|
273
273
|
else
|
274
274
|
yield
|
275
275
|
end
|
data/lib/stockboy/provider.rb
CHANGED
@@ -162,7 +162,7 @@ module Stockboy
|
|
162
162
|
when Symbol
|
163
163
|
list.public_send @pick
|
164
164
|
when Proc
|
165
|
-
list.reduce
|
165
|
+
@pick.arity == 1 ? @pick.call(list) : list.reduce(&@pick)
|
166
166
|
end
|
167
167
|
end
|
168
168
|
|
@@ -216,14 +216,14 @@ module Stockboy
|
|
216
216
|
#
|
217
217
|
# @!attribute [rw] file_smaller
|
218
218
|
# Validates the maximum data size for the matched file, in bytes
|
219
|
-
# @return [
|
219
|
+
# @return [Integer]
|
220
220
|
# @macro provider.pick_validation
|
221
221
|
# @example
|
222
222
|
# file_smaller 1024^3
|
223
223
|
#
|
224
224
|
# @!attribute [rw] file_larger
|
225
225
|
# Validates the minimum file size for the matched file, in bytes. This can # help guard against processing zero-byte or truncated files.
|
226
|
-
# @return [
|
226
|
+
# @return [Integer]
|
227
227
|
# @macro provider.pick_validation
|
228
228
|
# @example
|
229
229
|
# file_larger 1024
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'stockboy/provider'
|
2
|
-
require 'net/ftp'
|
3
2
|
|
4
3
|
module Stockboy::Providers
|
5
4
|
|
@@ -23,6 +22,8 @@ module Stockboy::Providers
|
|
23
22
|
# end
|
24
23
|
#
|
25
24
|
class FTP < Stockboy::Provider
|
25
|
+
require_relative 'ftp/ftp_adapter'
|
26
|
+
require_relative 'ftp/sftp_adapter'
|
26
27
|
|
27
28
|
# @!group Options
|
28
29
|
|
@@ -71,6 +72,15 @@ module Stockboy::Providers
|
|
71
72
|
#
|
72
73
|
dsl_attr :binary
|
73
74
|
|
75
|
+
# Use SFTP protocol for file transfers
|
76
|
+
#
|
77
|
+
# @!attribute [rw] secure
|
78
|
+
# @return [Boolean]
|
79
|
+
# @example
|
80
|
+
# secure true
|
81
|
+
#
|
82
|
+
dsl_attr :secure
|
83
|
+
|
74
84
|
# @macro provider.file_options
|
75
85
|
dsl_attr :file_name
|
76
86
|
dsl_attr :file_dir
|
@@ -91,6 +101,7 @@ module Stockboy::Providers
|
|
91
101
|
@passive = opts[:passive]
|
92
102
|
@username = opts[:username]
|
93
103
|
@password = opts[:password]
|
104
|
+
@secure = opts[:secure]
|
94
105
|
@binary = opts[:binary]
|
95
106
|
@file_dir = opts[:file_dir]
|
96
107
|
@file_name = opts[:file_name]
|
@@ -99,21 +110,25 @@ module Stockboy::Providers
|
|
99
110
|
@file_larger = opts[:file_larger]
|
100
111
|
@pick = opts[:pick] || :last
|
101
112
|
DSL.new(self).instance_eval(&block) if block_given?
|
113
|
+
@open_client = nil
|
114
|
+
end
|
115
|
+
|
116
|
+
def adapter_class
|
117
|
+
secure ? SFTPAdapter : FTPAdapter
|
102
118
|
end
|
103
119
|
|
104
120
|
def client
|
105
121
|
return yield @open_client if @open_client
|
106
122
|
|
107
|
-
|
108
|
-
ftp.binary = binary
|
109
|
-
ftp.passive = passive
|
110
|
-
ftp.chdir file_dir if file_dir
|
123
|
+
adapter_class.new(self).open do |ftp|
|
111
124
|
@open_client = ftp
|
125
|
+
ftp.chdir file_dir if file_dir
|
112
126
|
response = yield ftp
|
113
127
|
@open_client = nil
|
114
128
|
response
|
115
129
|
end
|
116
|
-
|
130
|
+
|
131
|
+
rescue adapter_class.exception_class => e
|
117
132
|
errors << e.message
|
118
133
|
logger.warn e.message
|
119
134
|
nil
|
@@ -122,7 +137,7 @@ module Stockboy::Providers
|
|
122
137
|
def matching_file
|
123
138
|
return @matching_file if @matching_file
|
124
139
|
client do |ftp|
|
125
|
-
file_listing = ftp.
|
140
|
+
file_listing = ftp.list_files
|
126
141
|
@matching_file = pick_from file_listing.select(&file_name_matcher)
|
127
142
|
end
|
128
143
|
end
|
@@ -150,7 +165,7 @@ module Stockboy::Providers
|
|
150
165
|
validate_file(matching_file)
|
151
166
|
if valid?
|
152
167
|
logger.debug "FTP getting file #{inspect_matching_file}"
|
153
|
-
@data = ftp.
|
168
|
+
@data = ftp.download(matching_file)
|
154
169
|
logger.debug "FTP got file #{inspect_matching_file} (#{data_size} bytes)"
|
155
170
|
end
|
156
171
|
end
|
@@ -184,7 +199,7 @@ module Stockboy::Providers
|
|
184
199
|
end
|
185
200
|
|
186
201
|
def validate_file_newer(data_file)
|
187
|
-
@data_time ||= client { |ftp| ftp.
|
202
|
+
@data_time ||= client { |ftp| ftp.modification_time(data_file) }
|
188
203
|
if file_newer and @data_time < file_newer
|
189
204
|
errors << "No new files since #{file_newer}"
|
190
205
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'net/ftp'
|
2
|
+
|
3
|
+
module Stockboy::Providers
|
4
|
+
class FTP::FTPAdapter
|
5
|
+
attr_reader :client
|
6
|
+
|
7
|
+
def initialize(provider)
|
8
|
+
@provider = provider
|
9
|
+
end
|
10
|
+
|
11
|
+
def open
|
12
|
+
result = nil
|
13
|
+
Net::FTP.open(@provider.host, @provider.username, @provider.password) do |ftp|
|
14
|
+
@client = ftp
|
15
|
+
client.binary = @provider.binary
|
16
|
+
client.passive = @provider.passive
|
17
|
+
result = yield self
|
18
|
+
end
|
19
|
+
result
|
20
|
+
end
|
21
|
+
|
22
|
+
def chdir(directory)
|
23
|
+
client.chdir directory
|
24
|
+
end
|
25
|
+
|
26
|
+
def list_files
|
27
|
+
client.nlst.sort
|
28
|
+
end
|
29
|
+
|
30
|
+
def delete(file_name)
|
31
|
+
client.delete file_name
|
32
|
+
end
|
33
|
+
|
34
|
+
def download(file_name)
|
35
|
+
client.get(file_name, nil)
|
36
|
+
end
|
37
|
+
|
38
|
+
def modification_time(file_name)
|
39
|
+
client.mtime file_name
|
40
|
+
end
|
41
|
+
|
42
|
+
def size(file_name)
|
43
|
+
client.size file_name
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.exception_class
|
47
|
+
Net::FTPError
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'net/sftp'
|
2
|
+
|
3
|
+
module Stockboy::Providers
|
4
|
+
class FTP::SFTPAdapter
|
5
|
+
attr_reader :client
|
6
|
+
|
7
|
+
def initialize(provider)
|
8
|
+
@provider = provider
|
9
|
+
@file_dir = "."
|
10
|
+
end
|
11
|
+
|
12
|
+
def open
|
13
|
+
result = nil
|
14
|
+
Net::SFTP.start(@provider.host, @provider.username, password: @provider.password) do |sftp|
|
15
|
+
@client = sftp
|
16
|
+
result = yield self
|
17
|
+
end
|
18
|
+
result
|
19
|
+
end
|
20
|
+
|
21
|
+
def chdir(directory)
|
22
|
+
@file_dir = ::File.join(directory, '')
|
23
|
+
end
|
24
|
+
|
25
|
+
def list_files
|
26
|
+
client.dir.entries(@file_dir).map(&:name).sort
|
27
|
+
end
|
28
|
+
|
29
|
+
def delete(file_name)
|
30
|
+
client.remove!(full_path(file_name))
|
31
|
+
end
|
32
|
+
|
33
|
+
def download(file_name)
|
34
|
+
client.download!(full_path(file_name))
|
35
|
+
end
|
36
|
+
|
37
|
+
def full_path(file_name)
|
38
|
+
::File.join(@file_dir, file_name)
|
39
|
+
end
|
40
|
+
|
41
|
+
def modification_time(file_name)
|
42
|
+
stat(file_name).mtime
|
43
|
+
end
|
44
|
+
|
45
|
+
def size(file_name)
|
46
|
+
stat(file_name).size
|
47
|
+
end
|
48
|
+
|
49
|
+
def stat(file_name)
|
50
|
+
client.file.open(full_path(file_name)).stat
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.exception_class
|
54
|
+
Net::SFTP::Exception
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -143,6 +143,7 @@ module Stockboy::Providers
|
|
143
143
|
@file_larger = opts[:file_larger]
|
144
144
|
@pick = opts[:pick] || :last
|
145
145
|
DSL.new(self).instance_eval(&block) if block_given?
|
146
|
+
@open_client = nil
|
146
147
|
end
|
147
148
|
|
148
149
|
# Direct access to the configured +Net::IMAP+ connection
|
@@ -161,7 +162,7 @@ module Stockboy::Providers
|
|
161
162
|
@open_client.examine(mailbox)
|
162
163
|
end
|
163
164
|
yield @open_client
|
164
|
-
rescue ::Net::IMAP::Error
|
165
|
+
rescue ::Net::IMAP::Error
|
165
166
|
errors << "IMAP connection error"
|
166
167
|
ensure
|
167
168
|
if first_connection && @open_client
|
@@ -190,7 +191,7 @@ module Stockboy::Providers
|
|
190
191
|
#
|
191
192
|
def message_key
|
192
193
|
return @message_key if @message_key
|
193
|
-
message_ids =
|
194
|
+
message_ids = find_messages(default_search_options)
|
194
195
|
@message_key = pick_from(message_ids) unless message_ids.empty?
|
195
196
|
end
|
196
197
|
|
@@ -210,11 +211,11 @@ module Stockboy::Providers
|
|
210
211
|
# Override default configured search options
|
211
212
|
#
|
212
213
|
# @example
|
213
|
-
# provider.
|
214
|
-
# provider.
|
215
|
-
# provider.
|
214
|
+
# provider.find_messages(subject: "Daily Report", before: Date.today)
|
215
|
+
# provider.find_messages(["SUBJECT", "Daily Report", "BEFORE", "21-DEC-12"])
|
216
|
+
# provider.find_messages("FLAGGED BEFORE 21-DEC-12")
|
216
217
|
#
|
217
|
-
def
|
218
|
+
def find_messages(options=nil)
|
218
219
|
client { |imap| imap.sort(['DATE'], search_keys(options), 'UTF-8') }
|
219
220
|
end
|
220
221
|
|
@@ -261,10 +262,10 @@ module Stockboy::Providers
|
|
261
262
|
end
|
262
263
|
|
263
264
|
def open_attachment(mail)
|
264
|
-
|
265
|
-
validate_file(
|
266
|
-
yield
|
267
|
-
|
265
|
+
file = mail.attachments.detect { |part| validate_attachment(part) }
|
266
|
+
validate_file(file) if file or return
|
267
|
+
yield file.decoded if valid?
|
268
|
+
file
|
268
269
|
end
|
269
270
|
|
270
271
|
def validate
|