stockboy 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +7 -2
- data/CHANGELOG.md +7 -0
- data/Gemfile +6 -3
- data/lib/stockboy/configurator.rb +3 -1
- data/lib/stockboy/filters.rb +0 -9
- data/lib/stockboy/job.rb +3 -3
- data/lib/stockboy/provider.rb +6 -8
- data/lib/stockboy/provider_repeater.rb +2 -3
- data/lib/stockboy/providers/file.rb +32 -21
- data/lib/stockboy/providers/ftp.rb +6 -2
- data/lib/stockboy/providers/http.rb +13 -10
- data/lib/stockboy/providers/imap.rb +47 -24
- data/lib/stockboy/providers/imap/search_options.rb +1 -1
- data/lib/stockboy/providers/soap.rb +3 -2
- data/lib/stockboy/railtie.rb +1 -0
- data/lib/stockboy/reader.rb +2 -0
- data/lib/stockboy/readers/spreadsheet.rb +1 -0
- data/lib/stockboy/registry.rb +4 -8
- data/lib/stockboy/version.rb +1 -1
- data/spec/fixtures/email/csv_attachment.eml +20 -0
- data/spec/spec_helper.rb +24 -2
- data/spec/stockboy/configurator_spec.rb +54 -3
- data/spec/stockboy/filter_spec.rb +7 -0
- data/spec/stockboy/job_spec.rb +65 -2
- data/spec/stockboy/provider_repeater_spec.rb +1 -1
- data/spec/stockboy/provider_spec.rb +21 -0
- data/spec/stockboy/providers/file_spec.rb +38 -18
- data/spec/stockboy/providers/http_spec.rb +1 -10
- data/spec/stockboy/providers/imap/search_options_spec.rb +13 -2
- data/spec/stockboy/providers/imap_spec.rb +83 -5
- data/spec/stockboy/providers/soap_spec.rb +1 -1
- data/spec/stockboy/reader_spec.rb +26 -0
- data/spec/stockboy/readers/spreadsheet_spec.rb +1 -1
- data/spec/stockboy/readers/xml_spec.rb +1 -1
- data/spec/stockboy/translations_spec.rb +25 -0
- metadata +27 -23
@@ -147,8 +147,8 @@ module Stockboy::Providers
|
|
147
147
|
#
|
148
148
|
def client
|
149
149
|
@client ||= Savon.client(client_options)
|
150
|
-
|
151
|
-
|
150
|
+
yield @client if block_given?
|
151
|
+
@client
|
152
152
|
end
|
153
153
|
|
154
154
|
private
|
@@ -159,6 +159,7 @@ module Stockboy::Providers
|
|
159
159
|
elsif endpoint
|
160
160
|
{endpoint: endpoint}
|
161
161
|
end
|
162
|
+
opts[:logger] = logger
|
162
163
|
opts[:convert_response_tags_to] = ->(tag) { string_pool(tag) }
|
163
164
|
opts[:namespace] = namespace if namespace
|
164
165
|
opts[:namespaces] = namespaces if namespaces
|
data/lib/stockboy/railtie.rb
CHANGED
data/lib/stockboy/reader.rb
CHANGED
@@ -99,6 +99,7 @@ module Stockboy::Readers
|
|
99
99
|
Tempfile.open(tmp_name, Stockboy.configuration.tmp_dir) do |file|
|
100
100
|
file.binmode
|
101
101
|
file.write content
|
102
|
+
file.fsync
|
102
103
|
table = Roo::Spreadsheet.open(file.path, @roo_options)
|
103
104
|
table.default_sheet = sheet_number(table, @sheet)
|
104
105
|
table.header_line = @header_line if @header_line
|
data/lib/stockboy/registry.rb
CHANGED
@@ -39,14 +39,10 @@ module Stockboy
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def build(key, options, block)
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
key.new(options, &block)
|
47
|
-
else
|
48
|
-
key
|
49
|
-
end
|
42
|
+
options = [options] unless options.is_a? Array
|
43
|
+
key = find(key) if key.is_a? Symbol
|
44
|
+
key = key.new(*options, &block) if key.is_a? Class
|
45
|
+
key
|
50
46
|
end
|
51
47
|
|
52
48
|
end
|
data/lib/stockboy/version.rb
CHANGED
@@ -0,0 +1,20 @@
|
|
1
|
+
Date: Fri, 21 Mar 2014 12:34:56 -0700 (PDT)
|
2
|
+
From: The Server <server@example.com>
|
3
|
+
To: The App <app@example.com>
|
4
|
+
Subject: Daily Report
|
5
|
+
Content-Type: multipart/mixed; boundary="----=_Part_Boundary_"
|
6
|
+
|
7
|
+
------=_Part_Boundary_
|
8
|
+
Content-Type: text/plain; charset="utf-8"
|
9
|
+
Content-Transfer-Encoding: quoted-printable
|
10
|
+
|
11
|
+
Here's the daily report.=20
|
12
|
+
|
13
|
+
------=_Part_Boundary_
|
14
|
+
Content-Type: text/csv; name="daily_report.csv"
|
15
|
+
Content-Disposition: attachment; filename="daily_report.csv"
|
16
|
+
|
17
|
+
LAST_NAME,FIRST_NAME
|
18
|
+
Dent,Arthur
|
19
|
+
Ford,Prefect
|
20
|
+
------=_Part_Boundary_--
|
data/spec/spec_helper.rb
CHANGED
@@ -1,8 +1,23 @@
|
|
1
|
-
if
|
1
|
+
if ENV['CI']
|
2
|
+
require "codeclimate-test-reporter"
|
3
|
+
CodeClimate::TestReporter.start
|
4
|
+
end
|
5
|
+
|
6
|
+
if ENV['COVERAGE']
|
7
|
+
require 'simplecov'
|
8
|
+
SimpleCov.start do
|
9
|
+
add_filter "/spec/"
|
10
|
+
add_group "Providers", "/providers/"
|
11
|
+
add_group "Readers", "/readers/"
|
12
|
+
add_group "Translations", "/translations/"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
if ENV['DEBUG']
|
2
17
|
require 'pry'
|
3
|
-
require 'pry-debugger'
|
4
18
|
end
|
5
19
|
|
20
|
+
|
6
21
|
require 'ostruct'
|
7
22
|
require 'savon'
|
8
23
|
require 'savon/mock/spec_helper'
|
@@ -23,6 +38,13 @@ RSpec.configure do |config|
|
|
23
38
|
end
|
24
39
|
end
|
25
40
|
|
41
|
+
module Helpers
|
42
|
+
def fixture_path(*args)
|
43
|
+
RSpec.configuration.fixture_path.join(*args)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
config.include Helpers
|
26
48
|
end
|
27
49
|
|
28
50
|
# VCR.configure do |c|
|
@@ -42,6 +42,15 @@ module Stockboy
|
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
|
+
describe "#repeat" do
|
46
|
+
it "registers an enumerator block" do
|
47
|
+
expect { subject.repeat }.to raise_error ArgumentError
|
48
|
+
subject.repeat do |output, provider|
|
49
|
+
output << provider
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
45
54
|
describe "#reader" do
|
46
55
|
before do
|
47
56
|
Readers.register(:csv, reader_class)
|
@@ -123,25 +132,67 @@ module Stockboy
|
|
123
132
|
end
|
124
133
|
subject.config[:filters][:pass].call(42).should == true
|
125
134
|
end
|
135
|
+
|
136
|
+
context "with a class" do
|
137
|
+
class TestFilter
|
138
|
+
attr_reader :args, :block
|
139
|
+
def initialize(*args, &block)
|
140
|
+
@args, @block = args, block
|
141
|
+
end
|
142
|
+
end
|
143
|
+
before { Filters.register :test, TestFilter }
|
144
|
+
|
145
|
+
it "passes arguments to a registered class symbol" do
|
146
|
+
subject.filter :pass, :test, 42
|
147
|
+
subject.config[:filters][:pass].args.should == [42]
|
148
|
+
end
|
149
|
+
|
150
|
+
it "passes a block to a registered class symbol" do
|
151
|
+
subject.filter :pass, :test do 42 end
|
152
|
+
subject.config[:filters][:pass].block[].should == 42
|
153
|
+
end
|
154
|
+
|
155
|
+
it "passes arguments to a given class" do
|
156
|
+
subject.filter :pass, TestFilter, 42
|
157
|
+
subject.config[:filters][:pass].args.should == [42]
|
158
|
+
end
|
159
|
+
|
160
|
+
it "uses an instance directly" do
|
161
|
+
subject.filter :pass, TestFilter.new(42)
|
162
|
+
subject.config[:filters][:pass].args.should == [42]
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
126
166
|
end
|
127
167
|
|
128
168
|
describe "#to_job" do
|
129
169
|
before do
|
130
170
|
Providers.register :test_prov, provider_class
|
131
171
|
Readers.register :test_read, reader_class
|
132
|
-
end
|
133
|
-
|
134
|
-
it "returns a Job instance" do
|
135
172
|
subject.provider :test_prov
|
136
173
|
subject.reader :test_read
|
137
174
|
subject.attributes &proc{}
|
175
|
+
end
|
138
176
|
|
177
|
+
it "returns a Job instance" do
|
139
178
|
job = subject.to_job
|
140
179
|
job.should be_a(Job)
|
141
180
|
job.provider.should be_a(provider_class)
|
142
181
|
job.reader.should be_a(reader_class)
|
143
182
|
job.attributes.should be_a(AttributeMap)
|
144
183
|
end
|
184
|
+
|
185
|
+
context "with a repeat block" do
|
186
|
+
before do
|
187
|
+
subject.repeat do |i, o| end
|
188
|
+
end
|
189
|
+
|
190
|
+
it "adds a repeater to the provider" do
|
191
|
+
job = subject.to_job
|
192
|
+
job.provider.should be_a ProviderRepeater
|
193
|
+
job.provider.base_provider.should be_a provider_class
|
194
|
+
end
|
195
|
+
end
|
145
196
|
end
|
146
197
|
|
147
198
|
end
|
@@ -9,6 +9,9 @@ class FishFilter < Stockboy::Filter
|
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
|
+
class CoffeeFilter < Stockboy::Filter
|
13
|
+
end
|
14
|
+
|
12
15
|
module Stockboy
|
13
16
|
describe Filter do
|
14
17
|
|
@@ -37,5 +40,9 @@ module Stockboy
|
|
37
40
|
end
|
38
41
|
end
|
39
42
|
|
43
|
+
it "warns of a missing subclass implementation" do
|
44
|
+
expect { CoffeeFilter.new.call(double, double) }.to raise_error NoMethodError
|
45
|
+
end
|
46
|
+
|
40
47
|
end
|
41
48
|
end
|
data/spec/stockboy/job_spec.rb
CHANGED
@@ -17,16 +17,18 @@ class TestReader
|
|
17
17
|
@parse = opts[:parse] || []
|
18
18
|
end
|
19
19
|
def parse(data)
|
20
|
-
@parse
|
20
|
+
@parse.respond_to?(:call) ? @parse.call(data) : @parse
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
24
|
module Stockboy
|
25
25
|
describe Job do
|
26
|
-
let(:jobs_path) {
|
26
|
+
let(:jobs_path) { fixture_path "jobs" }
|
27
27
|
let(:provider) { provider_double }
|
28
28
|
let(:reader) { reader_double }
|
29
29
|
|
30
|
+
subject(:job) { described_class.new }
|
31
|
+
|
30
32
|
let(:job_template) {
|
31
33
|
<<-END.gsub(/^ {6}/,'')
|
32
34
|
provider :ftp do
|
@@ -112,6 +114,25 @@ module Stockboy
|
|
112
114
|
end
|
113
115
|
end
|
114
116
|
|
117
|
+
describe "#attributes=" do
|
118
|
+
|
119
|
+
before do
|
120
|
+
job.attributes = AttributeMap.new do first_name end
|
121
|
+
job.all_records << double
|
122
|
+
end
|
123
|
+
|
124
|
+
it "replaces the attribute map" do
|
125
|
+
job.attributes = AttributeMap.new do last_name end
|
126
|
+
job.attributes.map(&:to).should == [:last_name]
|
127
|
+
end
|
128
|
+
|
129
|
+
it "resets the job" do
|
130
|
+
job.attributes = AttributeMap.new do last_name end
|
131
|
+
job.all_records.should be_empty
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
135
|
+
|
115
136
|
describe "#process" do
|
116
137
|
let(:attribute_map) { AttributeMap.new { name } }
|
117
138
|
|
@@ -171,6 +192,24 @@ module Stockboy
|
|
171
192
|
job.filters = {alpha: proc{ |r| r.name =~ /A/ }, beta: proc{ |r| r.name =~ /B/ }}
|
172
193
|
job.records.should == {alpha: [], beta: []}
|
173
194
|
end
|
195
|
+
|
196
|
+
context "with a repeating provider" do
|
197
|
+
let(:repeater) {
|
198
|
+
ProviderRepeater.new(provider) do |inputs|
|
199
|
+
1.upto 3 do |i|
|
200
|
+
inputs << provider_double(data: [{"name" => i}])
|
201
|
+
end
|
202
|
+
end
|
203
|
+
}
|
204
|
+
let(:noop_reader) { reader_double(parse: ->(data) { data }) }
|
205
|
+
let(:job) { Job.new(provider: repeater, reader: noop_reader, attributes: attribute_map) }
|
206
|
+
|
207
|
+
it "it loads all records into a set" do
|
208
|
+
job.process
|
209
|
+
job.all_records.size.should == 3
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
174
213
|
end
|
175
214
|
|
176
215
|
describe "#record_counts" do
|
@@ -237,6 +276,16 @@ module Stockboy
|
|
237
276
|
|
238
277
|
end
|
239
278
|
|
279
|
+
describe "#triggers=" do
|
280
|
+
|
281
|
+
it "replaces existing triggers" do
|
282
|
+
job.triggers = {breakfast: double}
|
283
|
+
job.triggers = {lunch: double}
|
284
|
+
job.triggers.keys.should == [:lunch]
|
285
|
+
end
|
286
|
+
|
287
|
+
end
|
288
|
+
|
240
289
|
describe "#method_missing" do
|
241
290
|
|
242
291
|
subject(:job) { Job.new(triggers: {cleanup: proc{|_|}})}
|
@@ -252,6 +301,20 @@ module Stockboy
|
|
252
301
|
|
253
302
|
end
|
254
303
|
|
304
|
+
describe "#inspect" do
|
305
|
+
let(:job) { Job.new(provider: provider_double, reader: reader_double) }
|
306
|
+
subject { job.inspect }
|
307
|
+
|
308
|
+
it "is not extraordinarily long" do
|
309
|
+
should start_with "#<Stockboy::Job"
|
310
|
+
should include "provider=TestProvider"
|
311
|
+
should include "reader=TestReader"
|
312
|
+
should include "attributes=[]"
|
313
|
+
should include "filters=[]"
|
314
|
+
should include "record_counts={}"
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
255
318
|
def provider_double(opts={})
|
256
319
|
TestProvider.new(opts)
|
257
320
|
end
|
@@ -50,5 +50,26 @@ module Stockboy
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
+
describe "#reload" do
|
54
|
+
subject(:provider) { ProviderSubclass.new(foo: true) }
|
55
|
+
|
56
|
+
it "clears and reloads the data" do
|
57
|
+
data = provider.data
|
58
|
+
expect(provider).to receive(:fetch_data).once.and_call_original
|
59
|
+
provider.reload.should == data
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
describe "#inspect" do
|
64
|
+
subject(:provider) { ProviderSubclass.new(foo: true) }
|
65
|
+
subject { provider.inspect }
|
66
|
+
|
67
|
+
it "is not extraordinarily long" do
|
68
|
+
should start_with "#<ProviderSubclass"
|
69
|
+
should include "data_size="
|
70
|
+
should include "errors="
|
71
|
+
should_not include "@data"
|
72
|
+
end
|
73
|
+
end
|
53
74
|
end
|
54
75
|
end
|
@@ -33,24 +33,31 @@ module Stockboy
|
|
33
33
|
end
|
34
34
|
|
35
35
|
describe "#matching_file" do
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
let(:provider) { Providers::File.new(file_dir: fixture_path("files")) }
|
37
|
+
subject(:matching_file) { provider.matching_file }
|
38
|
+
|
39
|
+
context "with a matching string" do
|
40
|
+
before { provider.file_name = "test_data-*" }
|
41
|
+
it "returns the full path to the matching file name" do
|
42
|
+
should end_with "fixtures/files/test_data-20120202.csv"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
context "with a matching regex" do
|
47
|
+
before { provider.file_name = /^test_data-\d+/ }
|
48
|
+
it "returns the full path to the matching file name" do
|
49
|
+
should end_with "fixtures/files/test_data-20120202.csv"
|
39
50
|
end
|
40
51
|
end
|
41
52
|
|
42
|
-
|
43
|
-
provider.file_name = "
|
44
|
-
|
53
|
+
context "with an unmatched string" do
|
54
|
+
before { provider.file_name = "missing" }
|
55
|
+
it { should be_nil }
|
45
56
|
end
|
46
57
|
end
|
47
58
|
|
48
59
|
describe "#data" do
|
49
|
-
subject(:provider)
|
50
|
-
Providers::File.new do |f|
|
51
|
-
f.file_dir = RSpec.configuration.fixture_path.join("files")
|
52
|
-
end
|
53
|
-
end
|
60
|
+
subject(:provider) { Providers::File.new(file_dir: fixture_path("files")) }
|
54
61
|
|
55
62
|
it "fails with an error if the file doesn't exist" do
|
56
63
|
provider.file_name = "missing-file.csv"
|
@@ -75,25 +82,38 @@ module Stockboy
|
|
75
82
|
provider.data.should == "2012-02-02\n"
|
76
83
|
end
|
77
84
|
|
78
|
-
context "
|
85
|
+
context "metadata validation" do
|
86
|
+
before { provider.file_name = '*.csv' }
|
79
87
|
let(:recently) { Time.now - 60 }
|
80
88
|
let(:last_week) { Time.now - 86400 }
|
81
89
|
|
82
|
-
it "skips old files" do
|
90
|
+
it "skips old files with :since" do
|
83
91
|
expect_any_instance_of(::File).to receive(:mtime).and_return last_week
|
84
|
-
provider.file_name = '*.csv'
|
85
92
|
provider.since = recently
|
86
|
-
|
87
93
|
provider.data.should be_nil
|
88
94
|
provider.errors.first.should == "no new files since #{recently}"
|
89
95
|
end
|
96
|
+
|
97
|
+
it "skips large files with :file_smaller" do
|
98
|
+
expect_any_instance_of(::File).to receive(:size).and_return 1001
|
99
|
+
provider.file_smaller = 1000
|
100
|
+
provider.data.should be_nil
|
101
|
+
provider.errors.first.should == "file size larger than 1000"
|
102
|
+
end
|
103
|
+
|
104
|
+
it "skips small files with :file_larger" do
|
105
|
+
expect_any_instance_of(::File).to receive(:size).and_return 999
|
106
|
+
provider.file_larger = 1000
|
107
|
+
provider.data.should be_nil
|
108
|
+
provider.errors.first.should == "file size smaller than 1000"
|
109
|
+
end
|
90
110
|
end
|
91
111
|
end
|
92
112
|
|
93
113
|
describe ".delete_data" do
|
94
|
-
let(:target)
|
95
|
-
let(:target_dir)
|
96
|
-
let(:pick_same)
|
114
|
+
let(:target) { ::Tempfile.new(['delete', '.csv']) }
|
115
|
+
let(:target_dir) { File.dirname(target) }
|
116
|
+
let(:pick_same) { ->(best, this) { this == target.path ? this : best } }
|
97
117
|
|
98
118
|
subject(:provider) do
|
99
119
|
Providers::File.new(file_name: 'delete*.csv', file_dir: target_dir, pick: pick_same)
|