stockboy 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -2
  4. data/CHANGELOG.md +7 -0
  5. data/Gemfile +6 -3
  6. data/lib/stockboy/configurator.rb +3 -1
  7. data/lib/stockboy/filters.rb +0 -9
  8. data/lib/stockboy/job.rb +3 -3
  9. data/lib/stockboy/provider.rb +6 -8
  10. data/lib/stockboy/provider_repeater.rb +2 -3
  11. data/lib/stockboy/providers/file.rb +32 -21
  12. data/lib/stockboy/providers/ftp.rb +6 -2
  13. data/lib/stockboy/providers/http.rb +13 -10
  14. data/lib/stockboy/providers/imap.rb +47 -24
  15. data/lib/stockboy/providers/imap/search_options.rb +1 -1
  16. data/lib/stockboy/providers/soap.rb +3 -2
  17. data/lib/stockboy/railtie.rb +1 -0
  18. data/lib/stockboy/reader.rb +2 -0
  19. data/lib/stockboy/readers/spreadsheet.rb +1 -0
  20. data/lib/stockboy/registry.rb +4 -8
  21. data/lib/stockboy/version.rb +1 -1
  22. data/spec/fixtures/email/csv_attachment.eml +20 -0
  23. data/spec/spec_helper.rb +24 -2
  24. data/spec/stockboy/configurator_spec.rb +54 -3
  25. data/spec/stockboy/filter_spec.rb +7 -0
  26. data/spec/stockboy/job_spec.rb +65 -2
  27. data/spec/stockboy/provider_repeater_spec.rb +1 -1
  28. data/spec/stockboy/provider_spec.rb +21 -0
  29. data/spec/stockboy/providers/file_spec.rb +38 -18
  30. data/spec/stockboy/providers/http_spec.rb +1 -10
  31. data/spec/stockboy/providers/imap/search_options_spec.rb +13 -2
  32. data/spec/stockboy/providers/imap_spec.rb +83 -5
  33. data/spec/stockboy/providers/soap_spec.rb +1 -1
  34. data/spec/stockboy/reader_spec.rb +26 -0
  35. data/spec/stockboy/readers/spreadsheet_spec.rb +1 -1
  36. data/spec/stockboy/readers/xml_spec.rb +1 -1
  37. data/spec/stockboy/translations_spec.rb +25 -0
  38. metadata +27 -23
@@ -97,7 +97,7 @@ module Stockboy::Providers
97
97
  when Numeric
98
98
  Time.at(value).strftime('%v')
99
99
  when String
100
- value =~ VMS_DATE ? value : Date.parse(value).strftime('%v')
100
+ value =~ VMS_DATE ? value : Date.parse(value).strftime('%v').upcase!
101
101
  end
102
102
  pair
103
103
  end
@@ -147,8 +147,8 @@ module Stockboy::Providers
147
147
  #
148
148
  def client
149
149
  @client ||= Savon.client(client_options)
150
- return @client unless block_given?
151
- yield @client
150
+ yield @client if block_given?
151
+ @client
152
152
  end
153
153
 
154
154
  private
@@ -159,6 +159,7 @@ module Stockboy::Providers
159
159
  elsif endpoint
160
160
  {endpoint: endpoint}
161
161
  end
162
+ opts[:logger] = logger
162
163
  opts[:convert_response_tags_to] = ->(tag) { string_pool(tag) }
163
164
  opts[:namespace] = namespace if namespace
164
165
  opts[:namespaces] = namespaces if namespaces
@@ -17,6 +17,7 @@ class Railtie < Rails::Railtie
17
17
 
18
18
  initializer "stockboy.configure_rails_initialization" do
19
19
  Stockboy.configure do |config|
20
+ config.logger = Rails.logger
20
21
  config.template_load_paths = [Rails.root.join('config/stockboy_jobs')]
21
22
  end
22
23
 
@@ -24,6 +24,8 @@ module Stockboy
24
24
  class Reader
25
25
  extend Stockboy::DSL
26
26
 
27
+ attr_reader :encoding
28
+
27
29
  # Initialize a new reader
28
30
  #
29
31
  # @param [Hash] opts
@@ -99,6 +99,7 @@ module Stockboy::Readers
99
99
  Tempfile.open(tmp_name, Stockboy.configuration.tmp_dir) do |file|
100
100
  file.binmode
101
101
  file.write content
102
+ file.fsync
102
103
  table = Roo::Spreadsheet.open(file.path, @roo_options)
103
104
  table.default_sheet = sheet_number(table, @sheet)
104
105
  table.header_line = @header_line if @header_line
@@ -39,14 +39,10 @@ module Stockboy
39
39
  end
40
40
 
41
41
  def build(key, options, block)
42
- case key
43
- when Symbol
44
- find(key).new(options, &block)
45
- when Class
46
- key.new(options, &block)
47
- else
48
- key
49
- end
42
+ options = [options] unless options.is_a? Array
43
+ key = find(key) if key.is_a? Symbol
44
+ key = key.new(*options, &block) if key.is_a? Class
45
+ key
50
46
  end
51
47
 
52
48
  end
@@ -1,3 +1,3 @@
1
1
  module Stockboy
2
- VERSION = "0.7.0"
2
+ VERSION = "0.7.1"
3
3
  end
@@ -0,0 +1,20 @@
1
+ Date: Fri, 21 Mar 2014 12:34:56 -0700 (PDT)
2
+ From: The Server <server@example.com>
3
+ To: The App <app@example.com>
4
+ Subject: Daily Report
5
+ Content-Type: multipart/mixed; boundary="----=_Part_Boundary_"
6
+
7
+ ------=_Part_Boundary_
8
+ Content-Type: text/plain; charset="utf-8"
9
+ Content-Transfer-Encoding: quoted-printable
10
+
11
+ Here's the daily report.=20
12
+
13
+ ------=_Part_Boundary_
14
+ Content-Type: text/csv; name="daily_report.csv"
15
+ Content-Disposition: attachment; filename="daily_report.csv"
16
+
17
+ LAST_NAME,FIRST_NAME
18
+ Dent,Arthur
19
+ Ford,Prefect
20
+ ------=_Part_Boundary_--
data/spec/spec_helper.rb CHANGED
@@ -1,8 +1,23 @@
1
- if $DEBUG && !ENV['CI']
1
+ if ENV['CI']
2
+ require "codeclimate-test-reporter"
3
+ CodeClimate::TestReporter.start
4
+ end
5
+
6
+ if ENV['COVERAGE']
7
+ require 'simplecov'
8
+ SimpleCov.start do
9
+ add_filter "/spec/"
10
+ add_group "Providers", "/providers/"
11
+ add_group "Readers", "/readers/"
12
+ add_group "Translations", "/translations/"
13
+ end
14
+ end
15
+
16
+ if ENV['DEBUG']
2
17
  require 'pry'
3
- require 'pry-debugger'
4
18
  end
5
19
 
20
+
6
21
  require 'ostruct'
7
22
  require 'savon'
8
23
  require 'savon/mock/spec_helper'
@@ -23,6 +38,13 @@ RSpec.configure do |config|
23
38
  end
24
39
  end
25
40
 
41
+ module Helpers
42
+ def fixture_path(*args)
43
+ RSpec.configuration.fixture_path.join(*args)
44
+ end
45
+ end
46
+
47
+ config.include Helpers
26
48
  end
27
49
 
28
50
  # VCR.configure do |c|
@@ -42,6 +42,15 @@ module Stockboy
42
42
  end
43
43
  end
44
44
 
45
+ describe "#repeat" do
46
+ it "registers an enumerator block" do
47
+ expect { subject.repeat }.to raise_error ArgumentError
48
+ subject.repeat do |output, provider|
49
+ output << provider
50
+ end
51
+ end
52
+ end
53
+
45
54
  describe "#reader" do
46
55
  before do
47
56
  Readers.register(:csv, reader_class)
@@ -123,25 +132,67 @@ module Stockboy
123
132
  end
124
133
  subject.config[:filters][:pass].call(42).should == true
125
134
  end
135
+
136
+ context "with a class" do
137
+ class TestFilter
138
+ attr_reader :args, :block
139
+ def initialize(*args, &block)
140
+ @args, @block = args, block
141
+ end
142
+ end
143
+ before { Filters.register :test, TestFilter }
144
+
145
+ it "passes arguments to a registered class symbol" do
146
+ subject.filter :pass, :test, 42
147
+ subject.config[:filters][:pass].args.should == [42]
148
+ end
149
+
150
+ it "passes a block to a registered class symbol" do
151
+ subject.filter :pass, :test do 42 end
152
+ subject.config[:filters][:pass].block[].should == 42
153
+ end
154
+
155
+ it "passes arguments to a given class" do
156
+ subject.filter :pass, TestFilter, 42
157
+ subject.config[:filters][:pass].args.should == [42]
158
+ end
159
+
160
+ it "uses an instance directly" do
161
+ subject.filter :pass, TestFilter.new(42)
162
+ subject.config[:filters][:pass].args.should == [42]
163
+ end
164
+ end
165
+
126
166
  end
127
167
 
128
168
  describe "#to_job" do
129
169
  before do
130
170
  Providers.register :test_prov, provider_class
131
171
  Readers.register :test_read, reader_class
132
- end
133
-
134
- it "returns a Job instance" do
135
172
  subject.provider :test_prov
136
173
  subject.reader :test_read
137
174
  subject.attributes &proc{}
175
+ end
138
176
 
177
+ it "returns a Job instance" do
139
178
  job = subject.to_job
140
179
  job.should be_a(Job)
141
180
  job.provider.should be_a(provider_class)
142
181
  job.reader.should be_a(reader_class)
143
182
  job.attributes.should be_a(AttributeMap)
144
183
  end
184
+
185
+ context "with a repeat block" do
186
+ before do
187
+ subject.repeat do |i, o| end
188
+ end
189
+
190
+ it "adds a repeater to the provider" do
191
+ job = subject.to_job
192
+ job.provider.should be_a ProviderRepeater
193
+ job.provider.base_provider.should be_a provider_class
194
+ end
195
+ end
145
196
  end
146
197
 
147
198
  end
@@ -9,6 +9,9 @@ class FishFilter < Stockboy::Filter
9
9
  end
10
10
  end
11
11
 
12
+ class CoffeeFilter < Stockboy::Filter
13
+ end
14
+
12
15
  module Stockboy
13
16
  describe Filter do
14
17
 
@@ -37,5 +40,9 @@ module Stockboy
37
40
  end
38
41
  end
39
42
 
43
+ it "warns of a missing subclass implementation" do
44
+ expect { CoffeeFilter.new.call(double, double) }.to raise_error NoMethodError
45
+ end
46
+
40
47
  end
41
48
  end
@@ -17,16 +17,18 @@ class TestReader
17
17
  @parse = opts[:parse] || []
18
18
  end
19
19
  def parse(data)
20
- @parse
20
+ @parse.respond_to?(:call) ? @parse.call(data) : @parse
21
21
  end
22
22
  end
23
23
 
24
24
  module Stockboy
25
25
  describe Job do
26
- let(:jobs_path) { RSpec.configuration.fixture_path.join('jobs') }
26
+ let(:jobs_path) { fixture_path "jobs" }
27
27
  let(:provider) { provider_double }
28
28
  let(:reader) { reader_double }
29
29
 
30
+ subject(:job) { described_class.new }
31
+
30
32
  let(:job_template) {
31
33
  <<-END.gsub(/^ {6}/,'')
32
34
  provider :ftp do
@@ -112,6 +114,25 @@ module Stockboy
112
114
  end
113
115
  end
114
116
 
117
+ describe "#attributes=" do
118
+
119
+ before do
120
+ job.attributes = AttributeMap.new do first_name end
121
+ job.all_records << double
122
+ end
123
+
124
+ it "replaces the attribute map" do
125
+ job.attributes = AttributeMap.new do last_name end
126
+ job.attributes.map(&:to).should == [:last_name]
127
+ end
128
+
129
+ it "resets the job" do
130
+ job.attributes = AttributeMap.new do last_name end
131
+ job.all_records.should be_empty
132
+ end
133
+
134
+ end
135
+
115
136
  describe "#process" do
116
137
  let(:attribute_map) { AttributeMap.new { name } }
117
138
 
@@ -171,6 +192,24 @@ module Stockboy
171
192
  job.filters = {alpha: proc{ |r| r.name =~ /A/ }, beta: proc{ |r| r.name =~ /B/ }}
172
193
  job.records.should == {alpha: [], beta: []}
173
194
  end
195
+
196
+ context "with a repeating provider" do
197
+ let(:repeater) {
198
+ ProviderRepeater.new(provider) do |inputs|
199
+ 1.upto 3 do |i|
200
+ inputs << provider_double(data: [{"name" => i}])
201
+ end
202
+ end
203
+ }
204
+ let(:noop_reader) { reader_double(parse: ->(data) { data }) }
205
+ let(:job) { Job.new(provider: repeater, reader: noop_reader, attributes: attribute_map) }
206
+
207
+ it "it loads all records into a set" do
208
+ job.process
209
+ job.all_records.size.should == 3
210
+ end
211
+ end
212
+
174
213
  end
175
214
 
176
215
  describe "#record_counts" do
@@ -237,6 +276,16 @@ module Stockboy
237
276
 
238
277
  end
239
278
 
279
+ describe "#triggers=" do
280
+
281
+ it "replaces existing triggers" do
282
+ job.triggers = {breakfast: double}
283
+ job.triggers = {lunch: double}
284
+ job.triggers.keys.should == [:lunch]
285
+ end
286
+
287
+ end
288
+
240
289
  describe "#method_missing" do
241
290
 
242
291
  subject(:job) { Job.new(triggers: {cleanup: proc{|_|}})}
@@ -252,6 +301,20 @@ module Stockboy
252
301
 
253
302
  end
254
303
 
304
+ describe "#inspect" do
305
+ let(:job) { Job.new(provider: provider_double, reader: reader_double) }
306
+ subject { job.inspect }
307
+
308
+ it "is not extraordinarily long" do
309
+ should start_with "#<Stockboy::Job"
310
+ should include "provider=TestProvider"
311
+ should include "reader=TestReader"
312
+ should include "attributes=[]"
313
+ should include "filters=[]"
314
+ should include "record_counts={}"
315
+ end
316
+ end
317
+
255
318
  def provider_double(opts={})
256
319
  TestProvider.new(opts)
257
320
  end
@@ -28,7 +28,7 @@ module Stockboy
28
28
 
29
29
  it "yields each data set" do
30
30
  calls = []
31
- repeater.data { |data| calls << data[-1] }
31
+ repeater.data { |data| calls << data.split(",").last }
32
32
  calls.should == ["1", "2", "3"]
33
33
  end
34
34
 
@@ -50,5 +50,26 @@ module Stockboy
50
50
  end
51
51
  end
52
52
 
53
+ describe "#reload" do
54
+ subject(:provider) { ProviderSubclass.new(foo: true) }
55
+
56
+ it "clears and reloads the data" do
57
+ data = provider.data
58
+ expect(provider).to receive(:fetch_data).once.and_call_original
59
+ provider.reload.should == data
60
+ end
61
+ end
62
+
63
+ describe "#inspect" do
64
+ subject(:provider) { ProviderSubclass.new(foo: true) }
65
+ subject { provider.inspect }
66
+
67
+ it "is not extraordinarily long" do
68
+ should start_with "#<ProviderSubclass"
69
+ should include "data_size="
70
+ should include "errors="
71
+ should_not include "@data"
72
+ end
73
+ end
53
74
  end
54
75
  end
@@ -33,24 +33,31 @@ module Stockboy
33
33
  end
34
34
 
35
35
  describe "#matching_file" do
36
- subject(:provider) do
37
- Providers::File.new do |f|
38
- f.file_dir = RSpec.configuration.fixture_path.join("files")
36
+ let(:provider) { Providers::File.new(file_dir: fixture_path("files")) }
37
+ subject(:matching_file) { provider.matching_file }
38
+
39
+ context "with a matching string" do
40
+ before { provider.file_name = "test_data-*" }
41
+ it "returns the full path to the matching file name" do
42
+ should end_with "fixtures/files/test_data-20120202.csv"
43
+ end
44
+ end
45
+
46
+ context "with a matching regex" do
47
+ before { provider.file_name = /^test_data-\d+/ }
48
+ it "returns the full path to the matching file name" do
49
+ should end_with "fixtures/files/test_data-20120202.csv"
39
50
  end
40
51
  end
41
52
 
42
- it "returns the full path to the matching file name" do
43
- provider.file_name = "test_data-*"
44
- provider.matching_file.should end_with "fixtures/files/test_data-20120202.csv"
53
+ context "with an unmatched string" do
54
+ before { provider.file_name = "missing" }
55
+ it { should be_nil }
45
56
  end
46
57
  end
47
58
 
48
59
  describe "#data" do
49
- subject(:provider) do
50
- Providers::File.new do |f|
51
- f.file_dir = RSpec.configuration.fixture_path.join("files")
52
- end
53
- end
60
+ subject(:provider) { Providers::File.new(file_dir: fixture_path("files")) }
54
61
 
55
62
  it "fails with an error if the file doesn't exist" do
56
63
  provider.file_name = "missing-file.csv"
@@ -75,25 +82,38 @@ module Stockboy
75
82
  provider.data.should == "2012-02-02\n"
76
83
  end
77
84
 
78
- context "with :since validation" do
85
+ context "metadata validation" do
86
+ before { provider.file_name = '*.csv' }
79
87
  let(:recently) { Time.now - 60 }
80
88
  let(:last_week) { Time.now - 86400 }
81
89
 
82
- it "skips old files" do
90
+ it "skips old files with :since" do
83
91
  expect_any_instance_of(::File).to receive(:mtime).and_return last_week
84
- provider.file_name = '*.csv'
85
92
  provider.since = recently
86
-
87
93
  provider.data.should be_nil
88
94
  provider.errors.first.should == "no new files since #{recently}"
89
95
  end
96
+
97
+ it "skips large files with :file_smaller" do
98
+ expect_any_instance_of(::File).to receive(:size).and_return 1001
99
+ provider.file_smaller = 1000
100
+ provider.data.should be_nil
101
+ provider.errors.first.should == "file size larger than 1000"
102
+ end
103
+
104
+ it "skips small files with :file_larger" do
105
+ expect_any_instance_of(::File).to receive(:size).and_return 999
106
+ provider.file_larger = 1000
107
+ provider.data.should be_nil
108
+ provider.errors.first.should == "file size smaller than 1000"
109
+ end
90
110
  end
91
111
  end
92
112
 
93
113
  describe ".delete_data" do
94
- let(:target) { ::Tempfile.new(['delete', '.csv']) }
95
- let(:target_dir) { File.dirname(target) }
96
- let(:pick_same) { ->(best, this) { this == target.path ? this : best } }
114
+ let(:target) { ::Tempfile.new(['delete', '.csv']) }
115
+ let(:target_dir) { File.dirname(target) }
116
+ let(:pick_same) { ->(best, this) { this == target.path ? this : best } }
97
117
 
98
118
  subject(:provider) do
99
119
  Providers::File.new(file_name: 'delete*.csv', file_dir: target_dir, pick: pick_same)