stockboy 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +9 -11
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +1 -1
  5. data/lib/stockboy/configuration.rb +1 -1
  6. data/lib/stockboy/configurator.rb +0 -1
  7. data/lib/stockboy/exceptions.rb +14 -10
  8. data/lib/stockboy/job.rb +4 -4
  9. data/lib/stockboy/mapped_record.rb +0 -7
  10. data/lib/stockboy/provider.rb +3 -3
  11. data/lib/stockboy/provider_repeater.rb +0 -1
  12. data/lib/stockboy/providers/ftp.rb +24 -9
  13. data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
  14. data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
  15. data/lib/stockboy/providers/http.rb +0 -8
  16. data/lib/stockboy/providers/imap.rb +11 -10
  17. data/lib/stockboy/providers/soap.rb +3 -2
  18. data/lib/stockboy/readers/csv.rb +3 -3
  19. data/lib/stockboy/readers/fixed_width.rb +28 -21
  20. data/lib/stockboy/readers/spreadsheet.rb +30 -18
  21. data/lib/stockboy/translations/default_zero.rb +1 -1
  22. data/lib/stockboy/translations/integer.rb +2 -2
  23. data/lib/stockboy/version.rb +1 -1
  24. data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
  25. data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
  26. data/spec/spec_helper.rb +2 -6
  27. data/spec/stockboy/candidate_record_spec.rb +20 -11
  28. data/spec/stockboy/configurator_spec.rb +2 -2
  29. data/spec/stockboy/provider_repeater_spec.rb +16 -0
  30. data/spec/stockboy/providers/file_spec.rb +16 -1
  31. data/spec/stockboy/providers/ftp_spec.rb +18 -27
  32. data/spec/stockboy/providers/http_spec.rb +11 -3
  33. data/spec/stockboy/providers/imap_spec.rb +3 -3
  34. data/spec/stockboy/providers/soap_spec.rb +17 -1
  35. data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
  36. data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
  37. data/stockboy.gemspec +1 -0
  38. metadata +23 -4
@@ -28,7 +28,7 @@ module Stockboy::Providers
28
28
  # Maximum time to establish a connection
29
29
  #
30
30
  # @!attribute [rw] open_timeout
31
- # @return [Fixnum]
31
+ # @return [Integer]
32
32
  # @example
33
33
  # open_timeout 10
34
34
  #
@@ -37,7 +37,7 @@ module Stockboy::Providers
37
37
  # Maximum time to read data from connection
38
38
  #
39
39
  # @!attribute [rw] read_timeout
40
- # @return [Fixnum]
40
+ # @return [Integer]
41
41
  # @example
42
42
  # read_timeout 10
43
43
  #
@@ -182,6 +182,7 @@ module Stockboy::Providers
182
182
  opts[:open_timeout] = open_timeout if open_timeout
183
183
  opts[:read_timeout] = read_timeout if read_timeout
184
184
  opts[:logger] = logger
185
+ opts[:log] = logger.debug?
185
186
  opts[:convert_response_tags_to] = ->(tag) { string_pool(tag) }
186
187
  opts[:namespace] = namespace if namespace
187
188
  opts[:namespaces] = namespaces if namespaces
@@ -25,14 +25,14 @@ module Stockboy::Readers
25
25
  # Skip number of rows at start of file before data starts
26
26
  #
27
27
  # @!attribute [rw] skip_header_rows
28
- # @return [Fixnum]
28
+ # @return [Integer]
29
29
  #
30
30
  dsl_attr :skip_header_rows
31
31
 
32
32
  # Skip number of rows at end of file after data ends
33
33
  #
34
34
  # @!attribute [rw] skip_footer_rows
35
- # @return [Fixnum]
35
+ # @return [Integer]
36
36
  #
37
37
  dsl_attr :skip_footer_rows
38
38
 
@@ -80,7 +80,7 @@ module Stockboy::Readers
80
80
  chain = options[:header_converters] || []
81
81
  chain << proc{ |h| h.freeze }
82
82
  opts = options.merge(header_converters: chain)
83
- ::CSV.parse(sanitize(data), opts).map &:to_hash
83
+ ::CSV.parse(sanitize(data), opts).map(&:to_hash)
84
84
  end
85
85
 
86
86
  # Hash of all CSV-specific options
@@ -14,7 +14,7 @@ module Stockboy::Readers
14
14
  # Array format will use numeric indexes for field keys. Hash will use the
15
15
  # keys for naming the fields.
16
16
  #
17
- # @return [Array<Fixnum>, Hash{Object=>Fixnum}]
17
+ # @return [Array<Integer>, Hash{Object=>Integer}]
18
18
  # @example
19
19
  # reader.headers = [10, 5, 10, 42]
20
20
  # reader.parse(data)
@@ -29,27 +29,31 @@ module Stockboy::Readers
29
29
  # String format used for unpacking rows
30
30
  #
31
31
  # This is read from the {#headers} attribute by default but can be
32
- # overridden
32
+ # overridden. Uses implementation from +String#unpack+ to set field widths
33
+ # and types.
33
34
  #
34
35
  # @return [String]
36
+ # @see http://ruby-doc.org/core/String.html#method-i-unpack
37
+ # @example
38
+ # row_format "U16U32" # column A: 16 unicode, column B: 32 unicode
35
39
  #
36
- dsl_attr :skip_header_rows
40
+ dsl_attr :row_format, attr_reader: false
37
41
 
38
42
  # Number of file rows to skip from start of file
39
43
  #
40
44
  # Useful if the file starts with a preamble or header metadata
41
45
  #
42
- # @return [Fixnum]
46
+ # @return [Integer]
43
47
  #
44
- dsl_attr :skip_footer_rows
48
+ dsl_attr :skip_header_rows
45
49
 
46
50
  # Number of file rows to skip at end of file
47
51
  #
48
52
  # Useful if the file ends with a summary or notice
49
53
  #
50
- # @return [Fixnum]
54
+ # @return [Integer]
51
55
  #
52
- dsl_attr :row_format
56
+ dsl_attr :skip_footer_rows
53
57
 
54
58
  # Override original file encoding
55
59
  #
@@ -62,9 +66,9 @@ module Stockboy::Readers
62
66
  # Initialize a new fixed-width reader
63
67
  #
64
68
  # @param [Hash] opts
65
- # @option opts [Array<Fixnum>, Hash<Fixnum>] headers
66
- # @option opts [Fixnum] skip_header_rows
67
- # @option opts [Fixnum] skip_footer_rows
69
+ # @option opts [Array<Integer>, Hash<Integer>] headers
70
+ # @option opts [Integer] skip_header_rows
71
+ # @option opts [Integer] skip_footer_rows
68
72
  # @option opts [String] encoding
69
73
  #
70
74
  def initialize(opts={}, &block)
@@ -76,12 +80,12 @@ module Stockboy::Readers
76
80
  end
77
81
 
78
82
  def parse(data)
79
- @column_widths, @column_keys = nil, nil
83
+ validate_headers
80
84
  data.force_encoding(encoding) if encoding
81
85
  data = StringIO.new(data) unless data.is_a? StringIO
82
86
  skip_header_rows.times { data.readline }
83
- records = data.reduce([]) do |a, row|
84
- a.tap { a << parse_row(row) unless row.strip.empty? }
87
+ records = data.each_with_object([]) do |row, a|
88
+ a << parse_row(row) unless row.strip.empty?
85
89
  end
86
90
  skip_footer_rows.times { records.pop }
87
91
  records
@@ -94,22 +98,16 @@ module Stockboy::Readers
94
98
  private
95
99
 
96
100
  def column_widths
97
- return @column_widths if @column_widths
98
- @column_widths = case headers
101
+ @column_widths ||= case headers
99
102
  when Hash then headers.values
100
103
  when Array then headers
101
- else
102
- raise "Invalid headers set for #{self.class}"
103
104
  end
104
105
  end
105
106
 
106
107
  def column_keys
107
- return @column_keys if @column_keys
108
- @column_keys = case headers
108
+ @column_keys ||= case headers
109
109
  when Hash then headers.keys.map(&:freeze)
110
110
  when Array then (0 ... headers.length).to_a
111
- else
112
- raise "Invalid headers set for #{self.class}"
113
111
  end
114
112
  end
115
113
 
@@ -117,5 +115,14 @@ module Stockboy::Readers
117
115
  Hash[column_keys.zip(row.unpack(row_format))]
118
116
  end
119
117
 
118
+ def validate_headers
119
+ @column_widths, @column_keys, @row_format = nil, nil, nil
120
+ case headers
121
+ when Hash, Array then true
122
+ else raise ArgumentError, "Invalid headers set for #{self.class}, " \
123
+ "got #{headers.class}, expected Hash or Array"
124
+ end
125
+ end
126
+
120
127
  end
121
128
  end
@@ -21,38 +21,51 @@ module Stockboy::Readers
21
21
  # Spreadsheet sheet number, defaults to first
22
22
  #
23
23
  # @!attribute [rw] sheet
24
- # @return [Fixnum]
24
+ # @return [Integer]
25
25
  #
26
26
  dsl_attr :sheet
27
27
 
28
28
  # Line number to look for headers, starts counting at 1, like in Excel
29
29
  #
30
+ # When specified without +first_row+, then the next row becomes the first
31
+ # data row by default.
32
+ #
30
33
  # @!attribute [rw] header_row
31
- # @return [Fixnum]
34
+ # @return [Integer]
32
35
  #
33
36
  dsl_attr :header_row
34
37
 
35
38
  # Line number of first data row, starts counting at 1, like in Excel
36
39
  #
37
40
  # @!attribute [rw] first_row
38
- # @return [Fixnum]
41
+ # @return [Integer]
39
42
  #
40
43
  dsl_attr :first_row
41
44
 
42
45
  # Line number of last data row, use negative numbers to count back from end
43
46
  #
44
47
  # @!attribute [rw] last_row
45
- # @return [Fixnum]
48
+ # @return [Integer]
46
49
  #
47
50
  dsl_attr :last_row
48
51
 
49
52
  # Override to set headers manually
50
53
  #
54
+ # When specified, the first spreadsheet row is the default
55
+ # first data row, unless specified by +first_row+.
56
+ #
51
57
  # @!attribute [rw] headers
52
58
  # @return [Array]
53
59
  #
54
60
  dsl_attr :headers
55
61
 
62
+ # Options passed to underlying Roo library
63
+ #
64
+ # @!attribute [rw] options
65
+ # @return [Hash]
66
+ #
67
+ dsl_attr :options
68
+
56
69
  # @!endgroup
57
70
 
58
71
  # Initialize a new Spreadsheet reader
@@ -67,7 +80,7 @@ module Stockboy::Readers
67
80
  @last_row = opts[:last_row]
68
81
  @header_row = opts[:header_row]
69
82
  @headers = opts[:headers]
70
- @roo_options = opts[:roo_options] || {}
83
+ @options = opts[:options] || {}
71
84
  DSL.new(self).instance_eval(&block) if block_given?
72
85
  end
73
86
 
@@ -81,15 +94,6 @@ module Stockboy::Readers
81
94
  end
82
95
  end
83
96
 
84
- # Roo-specific options hash passed to underlying spreadsheet parser
85
- #
86
- # @!attribute [r] options
87
- # @return [Hash]
88
- #
89
- def options
90
- @roo_options
91
- end
92
-
93
97
  private
94
98
 
95
99
  def enum_data_rows(table)
@@ -101,9 +105,9 @@ module Stockboy::Readers
101
105
  file.binmode
102
106
  file.write content
103
107
  file.fsync
104
- table = Roo::Spreadsheet.open(file.path, @roo_options)
108
+ table = Roo::Spreadsheet.open(file.path, @options)
105
109
  table.default_sheet = sheet_number(table, @sheet)
106
- table.header_line = @header_line if @header_line
110
+ table.header_line = @header_row if @header_row
107
111
  yield table
108
112
  end
109
113
  end
@@ -111,13 +115,21 @@ module Stockboy::Readers
111
115
  def sheet_number(table, id)
112
116
  case id
113
117
  when Symbol then table.sheets.public_send id
114
- when Fixnum then table.sheets[id-1]
118
+ when Integer then table.sheets[id-1]
115
119
  when String then id
116
120
  end
117
121
  end
118
122
 
119
123
  def first_table_row(table)
120
- @first_row || table.first_row
124
+ return @first_row if @first_row
125
+
126
+ if @headers
127
+ table.first_row
128
+ elsif @header_row
129
+ @header_row + 1
130
+ else
131
+ table.first_row + 1
132
+ end
121
133
  end
122
134
 
123
135
  def last_table_row(table)
@@ -28,7 +28,7 @@ module Stockboy::Translations
28
28
  #
29
29
  class DefaultZero < Stockboy::Translator
30
30
 
31
- # @return [Fixnum]
31
+ # @return [Integer]
32
32
  #
33
33
  def translate(context)
34
34
  value = field_value(context, field_key)
@@ -2,7 +2,7 @@ require 'stockboy/translator'
2
2
 
3
3
  module Stockboy::Translations
4
4
 
5
- # Translate string values to +Fixnum+
5
+ # Translate string values to +Integer+
6
6
  #
7
7
  # == Job template DSL
8
8
  #
@@ -20,7 +20,7 @@ module Stockboy::Translations
20
20
  #
21
21
  class Integer < Stockboy::Translator
22
22
 
23
- # @return [Fixnum]
23
+ # @return [Integer]
24
24
  #
25
25
  def translate(context)
26
26
  value = field_value(context, field_key)
@@ -1,3 +1,3 @@
1
1
  module Stockboy
2
- VERSION = "1.2.1"
2
+ VERSION = "1.3.0"
3
3
  end
@@ -1,11 +1,7 @@
1
- if ENV['CI']
2
- require "codeclimate-test-reporter"
3
- CodeClimate::TestReporter.start
4
- end
5
-
6
- if ENV['COVERAGE']
1
+ if ENV['COVERAGE'] || ENV['CI']
7
2
  require 'simplecov'
8
3
  SimpleCov.start do
4
+ add_filter "/.bundle/"
9
5
  add_filter "/spec/"
10
6
  add_group "Providers", "/providers/"
11
7
  add_group "Readers", "/readers/"
@@ -11,14 +11,6 @@ module Stockboy
11
11
  'birthday' => '1980-01-01'}
12
12
  end
13
13
 
14
- describe "initialize" do
15
- let(:map) { AttributeMap.new { id; email } }
16
-
17
- it "takes a hash and attributes map" do
18
- record = CandidateRecord.new(hash_attrs, map)
19
- end
20
- end
21
-
22
14
  describe "#to_hash" do
23
15
  it "remaps attributes" do
24
16
  map = AttributeMap.new { name from: 'full_name' }
@@ -78,12 +70,29 @@ module Stockboy
78
70
 
79
71
  context "with exception" do
80
72
  let(:map) { AttributeMap.new{ id as: [->(r){r.id.to_i}, ->(r){r.id / 0}] } }
73
+ let(:last_line) { __LINE__ - 1 }
74
+
81
75
  it { should eq({id: nil}) }
82
76
 
83
77
  context "while debugging" do
84
- it "raises the error" do
85
- Stockboy.configuration.translation_error_handler = -> (error) { raise error }
86
- expect { hash }.to(raise_error(Stockboy::TranslationError))
78
+ around do |example|
79
+ handler = Stockboy.configuration.translation_error_handler
80
+ example.run
81
+ Stockboy.configuration.translation_error_handler = handler
82
+ end
83
+
84
+ it "raises the error" do
85
+ captured = nil
86
+ Stockboy.configuration.translation_error_handler = ->(error) do
87
+ captured = error
88
+ raise error
89
+ end
90
+
91
+ expect { hash }.to raise_error(Stockboy::TranslationError)
92
+ expect(captured.message).to eq "Attribute [id] caused divided by 0"
93
+ expect(captured.key).to eq :id
94
+ expect(captured.record).to be record
95
+ expect(captured.backtrace[0]).to start_with "#{__FILE__}:#{last_line}:"
87
96
  end
88
97
  end
89
98
  end
@@ -78,7 +78,7 @@ module Stockboy
78
78
  it "initializes a block" do
79
79
  attribute_map = double
80
80
  expect(AttributeMap).to receive(:new).and_return(attribute_map)
81
- subject.attributes &proc{}
81
+ subject.attributes do end
82
82
  expect(subject.config[:attributes]).to be attribute_map
83
83
  end
84
84
 
@@ -171,7 +171,7 @@ module Stockboy
171
171
  Readers.register :test_read, reader_class
172
172
  subject.provider :test_prov
173
173
  subject.reader :test_read
174
- subject.attributes &proc{}
174
+ subject.attributes do end
175
175
  end
176
176
 
177
177
  it "returns a Job instance" do
@@ -30,6 +30,10 @@ module Stockboy
30
30
  expect(calls).to eq ["1", "2", "3"]
31
31
  end
32
32
 
33
+ it "raises an error if no block was given" do
34
+ expect{ repeater.data }.to raise_error ArgumentError
35
+ end
36
+
33
37
  end
34
38
 
35
39
  describe "#each" do
@@ -104,5 +108,17 @@ module Stockboy
104
108
  end
105
109
  end
106
110
 
111
+ describe "#clear" do
112
+ subject(:repeater) { ProviderRepeater.new(provider) }
113
+
114
+ it "resets iterations and data" do
115
+ expect(repeater.data_iterations).to eq 0
116
+ repeater.data do |data| end
117
+ expect(repeater.data_iterations).to eq 1
118
+ repeater.clear
119
+ expect(repeater.data_iterations).to eq 0
120
+ end
121
+ end
122
+
107
123
  end
108
124
  end
@@ -63,7 +63,7 @@ module Stockboy
63
63
  provider.file_name = "missing-file.csv"
64
64
  expect(provider.data).to be nil
65
65
  expect(provider.valid?).to be false
66
- expect(provider.errors.first).to match /not found/
66
+ expect(provider.errors.first).to include "not found"
67
67
  end
68
68
 
69
69
  it "finds last matching file from string glob" do
@@ -82,6 +82,18 @@ module Stockboy
82
82
  expect(provider.data).to eq "2012-02-02\n"
83
83
  end
84
84
 
85
+ it "selects item from single list argument proc" do
86
+ provider.file_name = "*"
87
+ provider.pick = ->(list) { list[1] }
88
+ expect(provider.data).to eq "2012-01-01\n"
89
+ end
90
+
91
+ it "reduces to single item from two-argument proc" do
92
+ provider.file_name = "*"
93
+ provider.pick = ->(last, best) { last.include?("01") ? last : best }
94
+ expect(provider.data).to eq "2012-01-01\n"
95
+ end
96
+
85
97
  context "metadata validation" do
86
98
  before { provider.file_name = '*.csv' }
87
99
  let(:recently) { Time.now - 60 }
@@ -130,6 +142,9 @@ module Stockboy
130
142
 
131
143
  expect(::File).to receive(:delete).with(target.path)
132
144
  provider.delete_data
145
+
146
+ expect(::File.exist?(non_matching_duplicate)).to be true
147
+ non_matching_duplicate.close
133
148
  end
134
149
  end
135
150