stockboy 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +9 -11
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +1 -1
  5. data/lib/stockboy/configuration.rb +1 -1
  6. data/lib/stockboy/configurator.rb +0 -1
  7. data/lib/stockboy/exceptions.rb +14 -10
  8. data/lib/stockboy/job.rb +4 -4
  9. data/lib/stockboy/mapped_record.rb +0 -7
  10. data/lib/stockboy/provider.rb +3 -3
  11. data/lib/stockboy/provider_repeater.rb +0 -1
  12. data/lib/stockboy/providers/ftp.rb +24 -9
  13. data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
  14. data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
  15. data/lib/stockboy/providers/http.rb +0 -8
  16. data/lib/stockboy/providers/imap.rb +11 -10
  17. data/lib/stockboy/providers/soap.rb +3 -2
  18. data/lib/stockboy/readers/csv.rb +3 -3
  19. data/lib/stockboy/readers/fixed_width.rb +28 -21
  20. data/lib/stockboy/readers/spreadsheet.rb +30 -18
  21. data/lib/stockboy/translations/default_zero.rb +1 -1
  22. data/lib/stockboy/translations/integer.rb +2 -2
  23. data/lib/stockboy/version.rb +1 -1
  24. data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
  25. data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
  26. data/spec/spec_helper.rb +2 -6
  27. data/spec/stockboy/candidate_record_spec.rb +20 -11
  28. data/spec/stockboy/configurator_spec.rb +2 -2
  29. data/spec/stockboy/provider_repeater_spec.rb +16 -0
  30. data/spec/stockboy/providers/file_spec.rb +16 -1
  31. data/spec/stockboy/providers/ftp_spec.rb +18 -27
  32. data/spec/stockboy/providers/http_spec.rb +11 -3
  33. data/spec/stockboy/providers/imap_spec.rb +3 -3
  34. data/spec/stockboy/providers/soap_spec.rb +17 -1
  35. data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
  36. data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
  37. data/stockboy.gemspec +1 -0
  38. metadata +23 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5fcae8abd12ed8674c56d476860be0ed4469ee13
4
- data.tar.gz: b0f49fa1a87c62dfd39184161cea8afcc7185b6c
3
+ metadata.gz: b36c08f870f1306e9c36cb125438bf8e01b3e6dc
4
+ data.tar.gz: e997f70d91635b9456ed6c0758e981ea7f2e16ed
5
5
  SHA512:
6
- metadata.gz: cf10196314b5e62dc785316a95741d3034772ca197defec68e032e570fece15a0de0f87bbc2888a9e35c2696f07d21f0a5dde7398e4ba5d24007029ea1040190
7
- data.tar.gz: fba95677551b2a57e5915c79b0cc13432168954772369e36ec54fd30e377aa5b6f728393586d9b190d0624f4e2b1da27af01729064b039bda6ae432caf05aed0
6
+ metadata.gz: 0a129da6b911fa8812706140192419782ce1e09e5ea26bc8a161dcf1006a918ff7ab6cbd94a94e1b6f4c279d9d092146e87386b5dd64a837980a03ba2682e48a
7
+ data.tar.gz: e5fd24fb0698cfb2d8488f35587ea49b0ac2d7bc0d39422a17fd5218a0974429fb1baf214e5f00f5295ec4ba309dade51e5a9a1d2b9048e5a9b0db6a51e19773
@@ -3,22 +3,20 @@ bundler_args: --without debug --without doc
3
3
  env:
4
4
  - CI=true
5
5
  rvm:
6
- - "1.9.3"
7
- - "2.0.0"
8
- - "2.1"
9
- - "2.2"
10
- - "rbx"
11
- - "jruby-head"
6
+ - 2.2
7
+ - 2.3
8
+ - 2.4
9
+ - ruby-head
10
+ - jruby-head
12
11
  before_install:
13
12
  - gem install bundler
14
- script: bundle exec rspec spec
13
+ script:
14
+ - bundle exec rspec spec
15
+ - bundle exec codeclimate-test-reporter
15
16
  matrix:
16
17
  fast_finish: true
17
18
  allow_failures:
18
- - rvm: "1.9.3"
19
- - rvm: "2.0.0"
20
- - rvm: "rbx"
21
- - rvm: "jruby-head"
19
+ - rvm: jruby-head
22
20
  addons:
23
21
  code_climate:
24
22
  repo_token: 7591072ad04d7a8ee788ce9259baa284724740cab1ae9ca94c162a583acd8f10
@@ -1,5 +1,14 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.3.0 / 2017-10-26
4
+
5
+ * [FEATURE] Add `secure` option for FTP to use SFTP
6
+ * [FEATURE] Update test matrix for recent ruby versions
7
+ * [ENHANCEMENT] Log SOAP provider output on debug level
8
+ * [ENHANCEMENT] Let pick accept proc with single arg for list
9
+ * [BUGFIX] Fix defaults for spreadsheet row options
10
+ * [BUGFIX] Fix spreadsheet option initialization
11
+
3
12
  ## 1.2.1 / 2016-08-02
4
13
 
5
14
  * [BUGFIX] Repeater would error with zero yielded data iterations
data/README.md CHANGED
@@ -236,7 +236,7 @@ with the "as" option.
236
236
  * [`:decimal`][deci]
237
237
  Numeric strings to `BigDecimal` (e.g. prices)
238
238
  * [`:integer`][intg]
239
- Numeric strings to `Fixnum` integers
239
+ Numeric strings to `Integer` integers
240
240
  * [`:string`][stri]
241
241
  Clean strings with leading/trailing whitespace trimmed
242
242
  * [`:or_empty`][dest]
@@ -42,7 +42,7 @@ module Stockboy
42
42
  @template_load_paths = []
43
43
  @logger = Logger.new(STDOUT)
44
44
  @tmp_dir = Dir.tmpdir
45
- @translation_error_handler = -> (error) { nil }
45
+ @translation_error_handler = ->(error) { nil }
46
46
  yield self if block_given?
47
47
  end
48
48
  end
@@ -1,4 +1,3 @@
1
- require 'stockboy/job'
2
1
  require 'stockboy/providers'
3
2
  require 'stockboy/readers'
4
3
  require 'stockboy/filters'
@@ -1,20 +1,24 @@
1
1
  module Stockboy
2
2
  class OutOfSequence < StandardError; end
3
3
 
4
- # TranslationError is a wrapper to store the standard error as well as the key and record which caused it
5
- class TranslationError < StandardError
4
+ class TranslationError < StandardError
6
5
 
7
- def initialize (key, record)
6
+ def initialize (key, record)
8
7
  @key = key
9
8
  @record = record
10
- end
11
-
12
- def message
13
- "Attribute [#{key}] caused #{cause.message}"
14
- end
9
+ @cause = $!
10
+ end
15
11
 
16
- attr_reader :key
17
- attr_reader :record
12
+ def message
13
+ reason = @cause && @cause.message || super
14
+ "Attribute [#{key}] caused #{reason}"
15
+ end
18
16
 
17
+ def backtrace
18
+ @cause && @cause.backtrace || super
19
19
  end
20
+
21
+ attr_reader :key
22
+ attr_reader :record
23
+ end
20
24
  end
@@ -127,7 +127,7 @@ module Stockboy
127
127
  # Count of all processed records
128
128
  #
129
129
  # @!attribute [r] total_records
130
- # @return [Fixnum]
130
+ # @return [Integer]
131
131
  #
132
132
  def total_records
133
133
  @all_records.size
@@ -135,7 +135,7 @@ module Stockboy
135
135
 
136
136
  # Counts of processed records grouped by filter key
137
137
  #
138
- # @return [Hash{Symbol=>Fixnum}]
138
+ # @return [Hash{Symbol=>Integer}]
139
139
  #
140
140
  def record_counts
141
141
  @records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
@@ -267,9 +267,9 @@ module Stockboy
267
267
  end
268
268
  end
269
269
 
270
- def with_query_caching(&block)
270
+ def with_query_caching
271
271
  if defined? ActiveRecord
272
- ActiveRecord::Base.cache(&block)
272
+ ActiveRecord::Base.cache { yield }
273
273
  else
274
274
  yield
275
275
  end
@@ -48,12 +48,5 @@ module Stockboy
48
48
  @fields = fields
49
49
  freeze
50
50
  end
51
-
52
- # @return [String]
53
- #
54
- def to_s
55
- @fields.to_s
56
- end
57
-
58
51
  end
59
52
  end
@@ -162,7 +162,7 @@ module Stockboy
162
162
  when Symbol
163
163
  list.public_send @pick
164
164
  when Proc
165
- list.reduce &@pick
165
+ @pick.arity == 1 ? @pick.call(list) : list.reduce(&@pick)
166
166
  end
167
167
  end
168
168
 
@@ -216,14 +216,14 @@ module Stockboy
216
216
  #
217
217
  # @!attribute [rw] file_smaller
218
218
  # Validates the maximum data size for the matched file, in bytes
219
- # @return [Fixnum]
219
+ # @return [Integer]
220
220
  # @macro provider.pick_validation
221
221
  # @example
222
222
  # file_smaller 1024^3
223
223
  #
224
224
  # @!attribute [rw] file_larger
225
225
  # Validates the minimum file size for the matched file, in bytes. This can # help guard against processing zero-byte or truncated files.
226
- # @return [Fixnum]
226
+ # @return [Integer]
227
227
  # @macro provider.pick_validation
228
228
  # @example
229
229
  # file_larger 1024
@@ -64,7 +64,6 @@ module Stockboy
64
64
  def clear
65
65
  @base_provider = @orig_provider.dup
66
66
  @iterations.clear
67
- super
68
67
  end
69
68
 
70
69
  def each
@@ -1,5 +1,4 @@
1
1
  require 'stockboy/provider'
2
- require 'net/ftp'
3
2
 
4
3
  module Stockboy::Providers
5
4
 
@@ -23,6 +22,8 @@ module Stockboy::Providers
23
22
  # end
24
23
  #
25
24
  class FTP < Stockboy::Provider
25
+ require_relative 'ftp/ftp_adapter'
26
+ require_relative 'ftp/sftp_adapter'
26
27
 
27
28
  # @!group Options
28
29
 
@@ -71,6 +72,15 @@ module Stockboy::Providers
71
72
  #
72
73
  dsl_attr :binary
73
74
 
75
+ # Use SFTP protocol for file transfers
76
+ #
77
+ # @!attribute [rw] secure
78
+ # @return [Boolean]
79
+ # @example
80
+ # secure true
81
+ #
82
+ dsl_attr :secure
83
+
74
84
  # @macro provider.file_options
75
85
  dsl_attr :file_name
76
86
  dsl_attr :file_dir
@@ -91,6 +101,7 @@ module Stockboy::Providers
91
101
  @passive = opts[:passive]
92
102
  @username = opts[:username]
93
103
  @password = opts[:password]
104
+ @secure = opts[:secure]
94
105
  @binary = opts[:binary]
95
106
  @file_dir = opts[:file_dir]
96
107
  @file_name = opts[:file_name]
@@ -99,21 +110,25 @@ module Stockboy::Providers
99
110
  @file_larger = opts[:file_larger]
100
111
  @pick = opts[:pick] || :last
101
112
  DSL.new(self).instance_eval(&block) if block_given?
113
+ @open_client = nil
114
+ end
115
+
116
+ def adapter_class
117
+ secure ? SFTPAdapter : FTPAdapter
102
118
  end
103
119
 
104
120
  def client
105
121
  return yield @open_client if @open_client
106
122
 
107
- Net::FTP.open(host, username, password) do |ftp|
108
- ftp.binary = binary
109
- ftp.passive = passive
110
- ftp.chdir file_dir if file_dir
123
+ adapter_class.new(self).open do |ftp|
111
124
  @open_client = ftp
125
+ ftp.chdir file_dir if file_dir
112
126
  response = yield ftp
113
127
  @open_client = nil
114
128
  response
115
129
  end
116
- rescue Net::FTPError => e
130
+
131
+ rescue adapter_class.exception_class => e
117
132
  errors << e.message
118
133
  logger.warn e.message
119
134
  nil
@@ -122,7 +137,7 @@ module Stockboy::Providers
122
137
  def matching_file
123
138
  return @matching_file if @matching_file
124
139
  client do |ftp|
125
- file_listing = ftp.nlst.sort
140
+ file_listing = ftp.list_files
126
141
  @matching_file = pick_from file_listing.select(&file_name_matcher)
127
142
  end
128
143
  end
@@ -150,7 +165,7 @@ module Stockboy::Providers
150
165
  validate_file(matching_file)
151
166
  if valid?
152
167
  logger.debug "FTP getting file #{inspect_matching_file}"
153
- @data = ftp.get(matching_file, nil)
168
+ @data = ftp.download(matching_file)
154
169
  logger.debug "FTP got file #{inspect_matching_file} (#{data_size} bytes)"
155
170
  end
156
171
  end
@@ -184,7 +199,7 @@ module Stockboy::Providers
184
199
  end
185
200
 
186
201
  def validate_file_newer(data_file)
187
- @data_time ||= client { |ftp| ftp.mtime(data_file) }
202
+ @data_time ||= client { |ftp| ftp.modification_time(data_file) }
188
203
  if file_newer and @data_time < file_newer
189
204
  errors << "No new files since #{file_newer}"
190
205
  end
@@ -0,0 +1,50 @@
1
+ require 'net/ftp'
2
+
3
+ module Stockboy::Providers
4
+ class FTP::FTPAdapter
5
+ attr_reader :client
6
+
7
+ def initialize(provider)
8
+ @provider = provider
9
+ end
10
+
11
+ def open
12
+ result = nil
13
+ Net::FTP.open(@provider.host, @provider.username, @provider.password) do |ftp|
14
+ @client = ftp
15
+ client.binary = @provider.binary
16
+ client.passive = @provider.passive
17
+ result = yield self
18
+ end
19
+ result
20
+ end
21
+
22
+ def chdir(directory)
23
+ client.chdir directory
24
+ end
25
+
26
+ def list_files
27
+ client.nlst.sort
28
+ end
29
+
30
+ def delete(file_name)
31
+ client.delete file_name
32
+ end
33
+
34
+ def download(file_name)
35
+ client.get(file_name, nil)
36
+ end
37
+
38
+ def modification_time(file_name)
39
+ client.mtime file_name
40
+ end
41
+
42
+ def size(file_name)
43
+ client.size file_name
44
+ end
45
+
46
+ def self.exception_class
47
+ Net::FTPError
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,57 @@
1
+ require 'net/sftp'
2
+
3
+ module Stockboy::Providers
4
+ class FTP::SFTPAdapter
5
+ attr_reader :client
6
+
7
+ def initialize(provider)
8
+ @provider = provider
9
+ @file_dir = "."
10
+ end
11
+
12
+ def open
13
+ result = nil
14
+ Net::SFTP.start(@provider.host, @provider.username, password: @provider.password) do |sftp|
15
+ @client = sftp
16
+ result = yield self
17
+ end
18
+ result
19
+ end
20
+
21
+ def chdir(directory)
22
+ @file_dir = ::File.join(directory, '')
23
+ end
24
+
25
+ def list_files
26
+ client.dir.entries(@file_dir).map(&:name).sort
27
+ end
28
+
29
+ def delete(file_name)
30
+ client.remove!(full_path(file_name))
31
+ end
32
+
33
+ def download(file_name)
34
+ client.download!(full_path(file_name))
35
+ end
36
+
37
+ def full_path(file_name)
38
+ ::File.join(@file_dir, file_name)
39
+ end
40
+
41
+ def modification_time(file_name)
42
+ stat(file_name).mtime
43
+ end
44
+
45
+ def size(file_name)
46
+ stat(file_name).size
47
+ end
48
+
49
+ def stat(file_name)
50
+ client.file.open(full_path(file_name)).stat
51
+ end
52
+
53
+ def self.exception_class
54
+ Net::SFTP::Exception
55
+ end
56
+ end
57
+ end
@@ -120,14 +120,6 @@ module Stockboy::Providers
120
120
  @uri = uri
121
121
  end
122
122
 
123
- def username=(username)
124
- @username = username
125
- end
126
-
127
- def password=(password)
128
- @password = password
129
- end
130
-
131
123
  # @!endgroup
132
124
 
133
125
  # Initialize an HTTP provider
@@ -143,6 +143,7 @@ module Stockboy::Providers
143
143
  @file_larger = opts[:file_larger]
144
144
  @pick = opts[:pick] || :last
145
145
  DSL.new(self).instance_eval(&block) if block_given?
146
+ @open_client = nil
146
147
  end
147
148
 
148
149
  # Direct access to the configured +Net::IMAP+ connection
@@ -161,7 +162,7 @@ module Stockboy::Providers
161
162
  @open_client.examine(mailbox)
162
163
  end
163
164
  yield @open_client
164
- rescue ::Net::IMAP::Error => e
165
+ rescue ::Net::IMAP::Error
165
166
  errors << "IMAP connection error"
166
167
  ensure
167
168
  if first_connection && @open_client
@@ -190,7 +191,7 @@ module Stockboy::Providers
190
191
  #
191
192
  def message_key
192
193
  return @message_key if @message_key
193
- message_ids = search(default_search_options)
194
+ message_ids = find_messages(default_search_options)
194
195
  @message_key = pick_from(message_ids) unless message_ids.empty?
195
196
  end
196
197
 
@@ -210,11 +211,11 @@ module Stockboy::Providers
210
211
  # Override default configured search options
211
212
  #
212
213
  # @example
213
- # provider.search(subject: "Daily Report", before: Date.today)
214
- # provider.search(["SUBJECT", "Daily Report", "BEFORE", "21-DEC-12"])
215
- # provider.search("FLAGGED BEFORE 21-DEC-12")
214
+ # provider.find_messages(subject: "Daily Report", before: Date.today)
215
+ # provider.find_messages(["SUBJECT", "Daily Report", "BEFORE", "21-DEC-12"])
216
+ # provider.find_messages("FLAGGED BEFORE 21-DEC-12")
216
217
  #
217
- def search(options=nil)
218
+ def find_messages(options=nil)
218
219
  client { |imap| imap.sort(['DATE'], search_keys(options), 'UTF-8') }
219
220
  end
220
221
 
@@ -261,10 +262,10 @@ module Stockboy::Providers
261
262
  end
262
263
 
263
264
  def open_attachment(mail)
264
- part = mail.attachments.detect { |part| validate_attachment(part) }
265
- validate_file(part) if part or return
266
- yield part.decoded if valid?
267
- part
265
+ file = mail.attachments.detect { |part| validate_attachment(part) }
266
+ validate_file(file) if file or return
267
+ yield file.decoded if valid?
268
+ file
268
269
  end
269
270
 
270
271
  def validate