stockboy 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -11
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/lib/stockboy/configuration.rb +1 -1
- data/lib/stockboy/configurator.rb +0 -1
- data/lib/stockboy/exceptions.rb +14 -10
- data/lib/stockboy/job.rb +4 -4
- data/lib/stockboy/mapped_record.rb +0 -7
- data/lib/stockboy/provider.rb +3 -3
- data/lib/stockboy/provider_repeater.rb +0 -1
- data/lib/stockboy/providers/ftp.rb +24 -9
- data/lib/stockboy/providers/ftp/ftp_adapter.rb +50 -0
- data/lib/stockboy/providers/ftp/sftp_adapter.rb +57 -0
- data/lib/stockboy/providers/http.rb +0 -8
- data/lib/stockboy/providers/imap.rb +11 -10
- data/lib/stockboy/providers/soap.rb +3 -2
- data/lib/stockboy/readers/csv.rb +3 -3
- data/lib/stockboy/readers/fixed_width.rb +28 -21
- data/lib/stockboy/readers/spreadsheet.rb +30 -18
- data/lib/stockboy/translations/default_zero.rb +1 -1
- data/lib/stockboy/translations/integer.rb +2 -2
- data/lib/stockboy/version.rb +1 -1
- data/spec/fixtures/spreadsheets/test_data.xls.zip +0 -0
- data/spec/fixtures/spreadsheets/test_data_sheets.xls +0 -0
- data/spec/spec_helper.rb +2 -6
- data/spec/stockboy/candidate_record_spec.rb +20 -11
- data/spec/stockboy/configurator_spec.rb +2 -2
- data/spec/stockboy/provider_repeater_spec.rb +16 -0
- data/spec/stockboy/providers/file_spec.rb +16 -1
- data/spec/stockboy/providers/ftp_spec.rb +18 -27
- data/spec/stockboy/providers/http_spec.rb +11 -3
- data/spec/stockboy/providers/imap_spec.rb +3 -3
- data/spec/stockboy/providers/soap_spec.rb +17 -1
- data/spec/stockboy/readers/fixed_width_spec.rb +8 -0
- data/spec/stockboy/readers/spreadsheet_spec.rb +61 -27
- data/stockboy.gemspec +1 -0
- metadata +23 -4
@@ -28,7 +28,7 @@ module Stockboy::Providers
|
|
28
28
|
# Maximum time to establish a connection
|
29
29
|
#
|
30
30
|
# @!attribute [rw] open_timeout
|
31
|
-
# @return [
|
31
|
+
# @return [Integer]
|
32
32
|
# @example
|
33
33
|
# open_timeout 10
|
34
34
|
#
|
@@ -37,7 +37,7 @@ module Stockboy::Providers
|
|
37
37
|
# Maximum time to read data from connection
|
38
38
|
#
|
39
39
|
# @!attribute [rw] read_timeout
|
40
|
-
# @return [
|
40
|
+
# @return [Integer]
|
41
41
|
# @example
|
42
42
|
# read_timeout 10
|
43
43
|
#
|
@@ -182,6 +182,7 @@ module Stockboy::Providers
|
|
182
182
|
opts[:open_timeout] = open_timeout if open_timeout
|
183
183
|
opts[:read_timeout] = read_timeout if read_timeout
|
184
184
|
opts[:logger] = logger
|
185
|
+
opts[:log] = logger.debug?
|
185
186
|
opts[:convert_response_tags_to] = ->(tag) { string_pool(tag) }
|
186
187
|
opts[:namespace] = namespace if namespace
|
187
188
|
opts[:namespaces] = namespaces if namespaces
|
data/lib/stockboy/readers/csv.rb
CHANGED
@@ -25,14 +25,14 @@ module Stockboy::Readers
|
|
25
25
|
# Skip number of rows at start of file before data starts
|
26
26
|
#
|
27
27
|
# @!attribute [rw] skip_header_rows
|
28
|
-
# @return [
|
28
|
+
# @return [Integer]
|
29
29
|
#
|
30
30
|
dsl_attr :skip_header_rows
|
31
31
|
|
32
32
|
# Skip number of rows at end of file after data ends
|
33
33
|
#
|
34
34
|
# @!attribute [rw] skip_footer_rows
|
35
|
-
# @return [
|
35
|
+
# @return [Integer]
|
36
36
|
#
|
37
37
|
dsl_attr :skip_footer_rows
|
38
38
|
|
@@ -80,7 +80,7 @@ module Stockboy::Readers
|
|
80
80
|
chain = options[:header_converters] || []
|
81
81
|
chain << proc{ |h| h.freeze }
|
82
82
|
opts = options.merge(header_converters: chain)
|
83
|
-
::CSV.parse(sanitize(data), opts).map
|
83
|
+
::CSV.parse(sanitize(data), opts).map(&:to_hash)
|
84
84
|
end
|
85
85
|
|
86
86
|
# Hash of all CSV-specific options
|
@@ -14,7 +14,7 @@ module Stockboy::Readers
|
|
14
14
|
# Array format will use numeric indexes for field keys. Hash will use the
|
15
15
|
# keys for naming the fields.
|
16
16
|
#
|
17
|
-
# @return [Array<
|
17
|
+
# @return [Array<Integer>, Hash{Object=>Integer}]
|
18
18
|
# @example
|
19
19
|
# reader.headers = [10, 5, 10, 42]
|
20
20
|
# reader.parse(data)
|
@@ -29,27 +29,31 @@ module Stockboy::Readers
|
|
29
29
|
# String format used for unpacking rows
|
30
30
|
#
|
31
31
|
# This is read from the {#headers} attribute by default but can be
|
32
|
-
# overridden
|
32
|
+
# overridden. Uses implementation from +String#unpack+ to set field widths
|
33
|
+
# and types.
|
33
34
|
#
|
34
35
|
# @return [String]
|
36
|
+
# @see http://ruby-doc.org/core/String.html#method-i-unpack
|
37
|
+
# @example
|
38
|
+
# row_format "U16U32" # column A: 16 unicode, column B: 32 unicode
|
35
39
|
#
|
36
|
-
dsl_attr :
|
40
|
+
dsl_attr :row_format, attr_reader: false
|
37
41
|
|
38
42
|
# Number of file rows to skip from start of file
|
39
43
|
#
|
40
44
|
# Useful if the file starts with a preamble or header metadata
|
41
45
|
#
|
42
|
-
# @return [
|
46
|
+
# @return [Integer]
|
43
47
|
#
|
44
|
-
dsl_attr :
|
48
|
+
dsl_attr :skip_header_rows
|
45
49
|
|
46
50
|
# Number of file rows to skip at end of file
|
47
51
|
#
|
48
52
|
# Useful if the file ends with a summary or notice
|
49
53
|
#
|
50
|
-
# @return [
|
54
|
+
# @return [Integer]
|
51
55
|
#
|
52
|
-
dsl_attr :
|
56
|
+
dsl_attr :skip_footer_rows
|
53
57
|
|
54
58
|
# Override original file encoding
|
55
59
|
#
|
@@ -62,9 +66,9 @@ module Stockboy::Readers
|
|
62
66
|
# Initialize a new fixed-width reader
|
63
67
|
#
|
64
68
|
# @param [Hash] opts
|
65
|
-
# @option opts [Array<
|
66
|
-
# @option opts [
|
67
|
-
# @option opts [
|
69
|
+
# @option opts [Array<Integer>, Hash<Integer>] headers
|
70
|
+
# @option opts [Integer] skip_header_rows
|
71
|
+
# @option opts [Integer] skip_footer_rows
|
68
72
|
# @option opts [String] encoding
|
69
73
|
#
|
70
74
|
def initialize(opts={}, &block)
|
@@ -76,12 +80,12 @@ module Stockboy::Readers
|
|
76
80
|
end
|
77
81
|
|
78
82
|
def parse(data)
|
79
|
-
|
83
|
+
validate_headers
|
80
84
|
data.force_encoding(encoding) if encoding
|
81
85
|
data = StringIO.new(data) unless data.is_a? StringIO
|
82
86
|
skip_header_rows.times { data.readline }
|
83
|
-
records = data.
|
84
|
-
a
|
87
|
+
records = data.each_with_object([]) do |row, a|
|
88
|
+
a << parse_row(row) unless row.strip.empty?
|
85
89
|
end
|
86
90
|
skip_footer_rows.times { records.pop }
|
87
91
|
records
|
@@ -94,22 +98,16 @@ module Stockboy::Readers
|
|
94
98
|
private
|
95
99
|
|
96
100
|
def column_widths
|
97
|
-
|
98
|
-
@column_widths = case headers
|
101
|
+
@column_widths ||= case headers
|
99
102
|
when Hash then headers.values
|
100
103
|
when Array then headers
|
101
|
-
else
|
102
|
-
raise "Invalid headers set for #{self.class}"
|
103
104
|
end
|
104
105
|
end
|
105
106
|
|
106
107
|
def column_keys
|
107
|
-
|
108
|
-
@column_keys = case headers
|
108
|
+
@column_keys ||= case headers
|
109
109
|
when Hash then headers.keys.map(&:freeze)
|
110
110
|
when Array then (0 ... headers.length).to_a
|
111
|
-
else
|
112
|
-
raise "Invalid headers set for #{self.class}"
|
113
111
|
end
|
114
112
|
end
|
115
113
|
|
@@ -117,5 +115,14 @@ module Stockboy::Readers
|
|
117
115
|
Hash[column_keys.zip(row.unpack(row_format))]
|
118
116
|
end
|
119
117
|
|
118
|
+
def validate_headers
|
119
|
+
@column_widths, @column_keys, @row_format = nil, nil, nil
|
120
|
+
case headers
|
121
|
+
when Hash, Array then true
|
122
|
+
else raise ArgumentError, "Invalid headers set for #{self.class}, " \
|
123
|
+
"got #{headers.class}, expected Hash or Array"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
120
127
|
end
|
121
128
|
end
|
@@ -21,38 +21,51 @@ module Stockboy::Readers
|
|
21
21
|
# Spreadsheet sheet number, defaults to first
|
22
22
|
#
|
23
23
|
# @!attribute [rw] sheet
|
24
|
-
# @return [
|
24
|
+
# @return [Integer]
|
25
25
|
#
|
26
26
|
dsl_attr :sheet
|
27
27
|
|
28
28
|
# Line number to look for headers, starts counting at 1, like in Excel
|
29
29
|
#
|
30
|
+
# When specified without +first_row+, then the next row becomes the first
|
31
|
+
# data row by default.
|
32
|
+
#
|
30
33
|
# @!attribute [rw] header_row
|
31
|
-
# @return [
|
34
|
+
# @return [Integer]
|
32
35
|
#
|
33
36
|
dsl_attr :header_row
|
34
37
|
|
35
38
|
# Line number of first data row, starts counting at 1, like in Excel
|
36
39
|
#
|
37
40
|
# @!attribute [rw] first_row
|
38
|
-
# @return [
|
41
|
+
# @return [Integer]
|
39
42
|
#
|
40
43
|
dsl_attr :first_row
|
41
44
|
|
42
45
|
# Line number of last data row, use negative numbers to count back from end
|
43
46
|
#
|
44
47
|
# @!attribute [rw] last_row
|
45
|
-
# @return [
|
48
|
+
# @return [Integer]
|
46
49
|
#
|
47
50
|
dsl_attr :last_row
|
48
51
|
|
49
52
|
# Override to set headers manually
|
50
53
|
#
|
54
|
+
# When specified, the first spreadsheet row is the default
|
55
|
+
# first data row, unless specified by +first_row+.
|
56
|
+
#
|
51
57
|
# @!attribute [rw] headers
|
52
58
|
# @return [Array]
|
53
59
|
#
|
54
60
|
dsl_attr :headers
|
55
61
|
|
62
|
+
# Options passed to underlying Roo library
|
63
|
+
#
|
64
|
+
# @!attribute [rw] options
|
65
|
+
# @return [Hash]
|
66
|
+
#
|
67
|
+
dsl_attr :options
|
68
|
+
|
56
69
|
# @!endgroup
|
57
70
|
|
58
71
|
# Initialize a new Spreadsheet reader
|
@@ -67,7 +80,7 @@ module Stockboy::Readers
|
|
67
80
|
@last_row = opts[:last_row]
|
68
81
|
@header_row = opts[:header_row]
|
69
82
|
@headers = opts[:headers]
|
70
|
-
@
|
83
|
+
@options = opts[:options] || {}
|
71
84
|
DSL.new(self).instance_eval(&block) if block_given?
|
72
85
|
end
|
73
86
|
|
@@ -81,15 +94,6 @@ module Stockboy::Readers
|
|
81
94
|
end
|
82
95
|
end
|
83
96
|
|
84
|
-
# Roo-specific options hash passed to underlying spreadsheet parser
|
85
|
-
#
|
86
|
-
# @!attribute [r] options
|
87
|
-
# @return [Hash]
|
88
|
-
#
|
89
|
-
def options
|
90
|
-
@roo_options
|
91
|
-
end
|
92
|
-
|
93
97
|
private
|
94
98
|
|
95
99
|
def enum_data_rows(table)
|
@@ -101,9 +105,9 @@ module Stockboy::Readers
|
|
101
105
|
file.binmode
|
102
106
|
file.write content
|
103
107
|
file.fsync
|
104
|
-
table = Roo::Spreadsheet.open(file.path, @
|
108
|
+
table = Roo::Spreadsheet.open(file.path, @options)
|
105
109
|
table.default_sheet = sheet_number(table, @sheet)
|
106
|
-
table.header_line = @
|
110
|
+
table.header_line = @header_row if @header_row
|
107
111
|
yield table
|
108
112
|
end
|
109
113
|
end
|
@@ -111,13 +115,21 @@ module Stockboy::Readers
|
|
111
115
|
def sheet_number(table, id)
|
112
116
|
case id
|
113
117
|
when Symbol then table.sheets.public_send id
|
114
|
-
when
|
118
|
+
when Integer then table.sheets[id-1]
|
115
119
|
when String then id
|
116
120
|
end
|
117
121
|
end
|
118
122
|
|
119
123
|
def first_table_row(table)
|
120
|
-
@first_row
|
124
|
+
return @first_row if @first_row
|
125
|
+
|
126
|
+
if @headers
|
127
|
+
table.first_row
|
128
|
+
elsif @header_row
|
129
|
+
@header_row + 1
|
130
|
+
else
|
131
|
+
table.first_row + 1
|
132
|
+
end
|
121
133
|
end
|
122
134
|
|
123
135
|
def last_table_row(table)
|
@@ -2,7 +2,7 @@ require 'stockboy/translator'
|
|
2
2
|
|
3
3
|
module Stockboy::Translations
|
4
4
|
|
5
|
-
# Translate string values to +
|
5
|
+
# Translate string values to +Integer+
|
6
6
|
#
|
7
7
|
# == Job template DSL
|
8
8
|
#
|
@@ -20,7 +20,7 @@ module Stockboy::Translations
|
|
20
20
|
#
|
21
21
|
class Integer < Stockboy::Translator
|
22
22
|
|
23
|
-
# @return [
|
23
|
+
# @return [Integer]
|
24
24
|
#
|
25
25
|
def translate(context)
|
26
26
|
value = field_value(context, field_key)
|
data/lib/stockboy/version.rb
CHANGED
Binary file
|
Binary file
|
data/spec/spec_helper.rb
CHANGED
@@ -1,11 +1,7 @@
|
|
1
|
-
if ENV['CI']
|
2
|
-
require "codeclimate-test-reporter"
|
3
|
-
CodeClimate::TestReporter.start
|
4
|
-
end
|
5
|
-
|
6
|
-
if ENV['COVERAGE']
|
1
|
+
if ENV['COVERAGE'] || ENV['CI']
|
7
2
|
require 'simplecov'
|
8
3
|
SimpleCov.start do
|
4
|
+
add_filter "/.bundle/"
|
9
5
|
add_filter "/spec/"
|
10
6
|
add_group "Providers", "/providers/"
|
11
7
|
add_group "Readers", "/readers/"
|
@@ -11,14 +11,6 @@ module Stockboy
|
|
11
11
|
'birthday' => '1980-01-01'}
|
12
12
|
end
|
13
13
|
|
14
|
-
describe "initialize" do
|
15
|
-
let(:map) { AttributeMap.new { id; email } }
|
16
|
-
|
17
|
-
it "takes a hash and attributes map" do
|
18
|
-
record = CandidateRecord.new(hash_attrs, map)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
14
|
describe "#to_hash" do
|
23
15
|
it "remaps attributes" do
|
24
16
|
map = AttributeMap.new { name from: 'full_name' }
|
@@ -78,12 +70,29 @@ module Stockboy
|
|
78
70
|
|
79
71
|
context "with exception" do
|
80
72
|
let(:map) { AttributeMap.new{ id as: [->(r){r.id.to_i}, ->(r){r.id / 0}] } }
|
73
|
+
let(:last_line) { __LINE__ - 1 }
|
74
|
+
|
81
75
|
it { should eq({id: nil}) }
|
82
76
|
|
83
77
|
context "while debugging" do
|
84
|
-
|
85
|
-
Stockboy.configuration.translation_error_handler
|
86
|
-
|
78
|
+
around do |example|
|
79
|
+
handler = Stockboy.configuration.translation_error_handler
|
80
|
+
example.run
|
81
|
+
Stockboy.configuration.translation_error_handler = handler
|
82
|
+
end
|
83
|
+
|
84
|
+
it "raises the error" do
|
85
|
+
captured = nil
|
86
|
+
Stockboy.configuration.translation_error_handler = ->(error) do
|
87
|
+
captured = error
|
88
|
+
raise error
|
89
|
+
end
|
90
|
+
|
91
|
+
expect { hash }.to raise_error(Stockboy::TranslationError)
|
92
|
+
expect(captured.message).to eq "Attribute [id] caused divided by 0"
|
93
|
+
expect(captured.key).to eq :id
|
94
|
+
expect(captured.record).to be record
|
95
|
+
expect(captured.backtrace[0]).to start_with "#{__FILE__}:#{last_line}:"
|
87
96
|
end
|
88
97
|
end
|
89
98
|
end
|
@@ -78,7 +78,7 @@ module Stockboy
|
|
78
78
|
it "initializes a block" do
|
79
79
|
attribute_map = double
|
80
80
|
expect(AttributeMap).to receive(:new).and_return(attribute_map)
|
81
|
-
subject.attributes
|
81
|
+
subject.attributes do end
|
82
82
|
expect(subject.config[:attributes]).to be attribute_map
|
83
83
|
end
|
84
84
|
|
@@ -171,7 +171,7 @@ module Stockboy
|
|
171
171
|
Readers.register :test_read, reader_class
|
172
172
|
subject.provider :test_prov
|
173
173
|
subject.reader :test_read
|
174
|
-
subject.attributes
|
174
|
+
subject.attributes do end
|
175
175
|
end
|
176
176
|
|
177
177
|
it "returns a Job instance" do
|
@@ -30,6 +30,10 @@ module Stockboy
|
|
30
30
|
expect(calls).to eq ["1", "2", "3"]
|
31
31
|
end
|
32
32
|
|
33
|
+
it "raises an error if no block was given" do
|
34
|
+
expect{ repeater.data }.to raise_error ArgumentError
|
35
|
+
end
|
36
|
+
|
33
37
|
end
|
34
38
|
|
35
39
|
describe "#each" do
|
@@ -104,5 +108,17 @@ module Stockboy
|
|
104
108
|
end
|
105
109
|
end
|
106
110
|
|
111
|
+
describe "#clear" do
|
112
|
+
subject(:repeater) { ProviderRepeater.new(provider) }
|
113
|
+
|
114
|
+
it "resets iterations and data" do
|
115
|
+
expect(repeater.data_iterations).to eq 0
|
116
|
+
repeater.data do |data| end
|
117
|
+
expect(repeater.data_iterations).to eq 1
|
118
|
+
repeater.clear
|
119
|
+
expect(repeater.data_iterations).to eq 0
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
107
123
|
end
|
108
124
|
end
|
@@ -63,7 +63,7 @@ module Stockboy
|
|
63
63
|
provider.file_name = "missing-file.csv"
|
64
64
|
expect(provider.data).to be nil
|
65
65
|
expect(provider.valid?).to be false
|
66
|
-
expect(provider.errors.first).to
|
66
|
+
expect(provider.errors.first).to include "not found"
|
67
67
|
end
|
68
68
|
|
69
69
|
it "finds last matching file from string glob" do
|
@@ -82,6 +82,18 @@ module Stockboy
|
|
82
82
|
expect(provider.data).to eq "2012-02-02\n"
|
83
83
|
end
|
84
84
|
|
85
|
+
it "selects item from single list argument proc" do
|
86
|
+
provider.file_name = "*"
|
87
|
+
provider.pick = ->(list) { list[1] }
|
88
|
+
expect(provider.data).to eq "2012-01-01\n"
|
89
|
+
end
|
90
|
+
|
91
|
+
it "reduces to single item from two-argument proc" do
|
92
|
+
provider.file_name = "*"
|
93
|
+
provider.pick = ->(last, best) { last.include?("01") ? last : best }
|
94
|
+
expect(provider.data).to eq "2012-01-01\n"
|
95
|
+
end
|
96
|
+
|
85
97
|
context "metadata validation" do
|
86
98
|
before { provider.file_name = '*.csv' }
|
87
99
|
let(:recently) { Time.now - 60 }
|
@@ -130,6 +142,9 @@ module Stockboy
|
|
130
142
|
|
131
143
|
expect(::File).to receive(:delete).with(target.path)
|
132
144
|
provider.delete_data
|
145
|
+
|
146
|
+
expect(::File.exist?(non_matching_duplicate)).to be true
|
147
|
+
non_matching_duplicate.close
|
133
148
|
end
|
134
149
|
end
|
135
150
|
|