exel 1.4.0 → 1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.codeclimate.yml +4 -4
- data/.rubocop.yml +20 -12
- data/.rubocop_airbnb.yml +2 -0
- data/.travis.yml +17 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +64 -60
- data/README.md +1 -1
- data/Rakefile +1 -0
- data/exel.gemspec +4 -4
- data/lib/exel.rb +1 -0
- data/lib/exel/ast_node.rb +2 -1
- data/lib/exel/context.rb +2 -1
- data/lib/exel/deferred_context_value.rb +1 -0
- data/lib/exel/error/job_termination.rb +1 -0
- data/lib/exel/events.rb +1 -0
- data/lib/exel/instruction.rb +2 -1
- data/lib/exel/instruction_node.rb +1 -0
- data/lib/exel/job.rb +6 -4
- data/lib/exel/listen_instruction.rb +1 -0
- data/lib/exel/logging.rb +1 -0
- data/lib/exel/logging/logger_wrapper.rb +4 -1
- data/lib/exel/logging_helper.rb +1 -0
- data/lib/exel/middleware/chain.rb +1 -0
- data/lib/exel/middleware/logging.rb +1 -1
- data/lib/exel/null_instruction.rb +1 -0
- data/lib/exel/processor_helper.rb +1 -0
- data/lib/exel/processors/run_processor.rb +1 -0
- data/lib/exel/processors/split_processor.rb +2 -1
- data/lib/exel/providers/local_file_provider.rb +2 -1
- data/lib/exel/providers/threaded_async_provider.rb +1 -0
- data/lib/exel/sequence_node.rb +1 -0
- data/lib/exel/value.rb +1 -0
- data/lib/exel/version.rb +2 -1
- data/spec/exel/ast_node_spec.rb +42 -42
- data/spec/exel/context_spec.rb +76 -77
- data/spec/exel/deferred_context_value_spec.rb +41 -42
- data/spec/exel/events_spec.rb +65 -65
- data/spec/exel/instruction_node_spec.rb +16 -16
- data/spec/exel/instruction_spec.rb +46 -45
- data/spec/exel/job_spec.rb +94 -91
- data/spec/exel/listen_instruction_spec.rb +10 -10
- data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
- data/spec/exel/logging_helper_spec.rb +15 -16
- data/spec/exel/logging_spec.rb +56 -56
- data/spec/exel/middleware/chain_spec.rb +51 -53
- data/spec/exel/middleware/logging_spec.rb +21 -23
- data/spec/exel/middleware_spec.rb +49 -50
- data/spec/exel/null_instruction_spec.rb +3 -4
- data/spec/exel/processors/async_processor_spec.rb +16 -18
- data/spec/exel/processors/run_processor_spec.rb +9 -11
- data/spec/exel/processors/split_processor_spec.rb +91 -93
- data/spec/exel/providers/local_file_provider_spec.rb +25 -28
- data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
- data/spec/exel/sequence_node_spec.rb +11 -11
- data/spec/exel/value_spec.rb +32 -33
- data/spec/exel_spec.rb +8 -7
- data/spec/integration/integration_spec.rb +2 -1
- data/spec/spec_helper.rb +3 -2
- data/spec/support/integration_test_classes.rb +3 -1
- metadata +16 -30
@@ -1,37 +1,34 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Providers
|
4
|
-
describe LocalFileProvider do
|
5
|
-
let(:file) { instance_double(File, path: '/path/to/file') }
|
6
2
|
|
7
|
-
|
8
|
-
|
9
|
-
expect(subject.upload(file)).to eq('file:///path/to/file')
|
10
|
-
end
|
11
|
-
end
|
3
|
+
describe EXEL::Providers::LocalFileProvider do
|
4
|
+
let(:file) { instance_double(File, path: '/path/to/file') }
|
12
5
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
6
|
+
describe '#upload' do
|
7
|
+
it 'returns a file:// URI for the file' do
|
8
|
+
expect(subject.upload(file)).to eq('file:///path/to/file')
|
9
|
+
end
|
10
|
+
end
|
18
11
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
12
|
+
describe '#download' do
|
13
|
+
it 'returns the file indicated by the URI' do
|
14
|
+
expect(File).to receive(:open).with('/path/to/file').and_return(file)
|
15
|
+
expect(subject.download('file:///path/to/file')).to eq(file)
|
16
|
+
end
|
23
17
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
18
|
+
it 'doesn`t accept URIs for schemes other than file://' do
|
19
|
+
expect { subject.download('s3://') }.to raise_error 'URI must begin with "file://"'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe '.remote?' do
|
24
|
+
it 'returns true for file:// URIs' do
|
25
|
+
expect(EXEL::Providers::LocalFileProvider.remote?('file:///path/to/file')).to be_truthy
|
26
|
+
end
|
28
27
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
28
|
+
it 'returns false for anything else' do
|
29
|
+
expect(EXEL::Providers::LocalFileProvider.remote?('s3://file')).to be_falsey
|
30
|
+
expect(EXEL::Providers::LocalFileProvider.remote?(1)).to be_falsey
|
31
|
+
expect(EXEL::Providers::LocalFileProvider.remote?(nil)).to be_falsey
|
35
32
|
end
|
36
33
|
end
|
37
34
|
end
|
@@ -1,54 +1,52 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Providers
|
4
|
-
class ContextMutatingProcessor
|
5
|
-
def initialize(context)
|
6
|
-
@context = context
|
7
|
-
end
|
8
2
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
class ContextMutatingProcessor
|
4
|
+
def initialize(context)
|
5
|
+
@context = context
|
6
|
+
end
|
13
7
|
|
14
|
-
|
15
|
-
|
16
|
-
|
8
|
+
def process(_block)
|
9
|
+
@context[:array] << @context[:arg]
|
10
|
+
end
|
11
|
+
end
|
17
12
|
|
18
|
-
|
19
|
-
|
13
|
+
describe EXEL::Providers::ThreadedAsyncProvider do
|
14
|
+
subject { described_class.new(context) }
|
20
15
|
|
21
|
-
|
22
|
-
expect(dsl_block).to receive(:start).with(context)
|
23
|
-
expect(Thread).to receive(:new).and_yield
|
16
|
+
let(:context) { EXEL::Context.new }
|
24
17
|
|
25
|
-
|
26
|
-
|
18
|
+
describe '#do_async' do
|
19
|
+
let(:dsl_block) { instance_double(EXEL::ASTNode) }
|
27
20
|
|
28
|
-
|
29
|
-
|
30
|
-
|
21
|
+
it 'runs the block in a new thread' do
|
22
|
+
expect(dsl_block).to receive(:start).with(context)
|
23
|
+
expect(Thread).to receive(:new).and_yield
|
31
24
|
|
32
|
-
|
33
|
-
|
34
|
-
process with: ContextMutatingProcessor, arg: 1
|
35
|
-
complete += 1
|
36
|
-
end
|
37
|
-
|
38
|
-
async do
|
39
|
-
process with: ContextMutatingProcessor, arg: 2
|
40
|
-
complete += 1
|
41
|
-
end
|
42
|
-
end
|
25
|
+
subject.do_async(dsl_block)
|
26
|
+
end
|
43
27
|
|
44
|
-
|
28
|
+
it 'passes a copy of the context to each thread' do
|
29
|
+
context[:array] = []
|
30
|
+
complete = 0
|
45
31
|
|
46
|
-
|
47
|
-
|
32
|
+
EXEL::Job.define :thread_test do
|
33
|
+
async do
|
34
|
+
process with: ContextMutatingProcessor, arg: 1
|
35
|
+
complete += 1
|
36
|
+
end
|
48
37
|
|
49
|
-
|
38
|
+
async do
|
39
|
+
process with: ContextMutatingProcessor, arg: 2
|
40
|
+
complete += 1
|
50
41
|
end
|
51
42
|
end
|
43
|
+
|
44
|
+
EXEL::Job.run(:thread_test, context)
|
45
|
+
|
46
|
+
start_time = Time.now
|
47
|
+
sleep 0.1 while complete < 2 && Time.now - start_time < 2
|
48
|
+
|
49
|
+
expect(context[:array]).to be_empty
|
52
50
|
end
|
53
51
|
end
|
54
52
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe SequenceNode do
|
4
|
-
subject(:node) { described_class.new(instance_double(ASTNode), instance_double(ASTNode)) }
|
5
|
-
let(:context) { instance_double(EXEL::Context) }
|
6
2
|
|
7
|
-
|
3
|
+
describe EXEL::SequenceNode do
|
4
|
+
subject(:node) { described_class.new(instance_double(EXEL::ASTNode), instance_double(EXEL::ASTNode)) }
|
8
5
|
|
9
|
-
|
10
|
-
it 'runs each child node in sequence' do
|
11
|
-
expect(node.children.first).to receive(:run).with(context).once.ordered
|
12
|
-
expect(node.children.last).to receive(:run).with(context).once.ordered
|
6
|
+
let(:context) { instance_double(EXEL::Context) }
|
13
7
|
|
14
|
-
|
15
|
-
|
8
|
+
it { is_expected.to be_an(EXEL::ASTNode) }
|
9
|
+
|
10
|
+
describe '#run' do
|
11
|
+
it 'runs each child node in sequence' do
|
12
|
+
expect(node.children.first).to receive(:run).with(context).once.ordered
|
13
|
+
expect(node.children.last).to receive(:run).with(context).once.ordered
|
14
|
+
|
15
|
+
node.run(context)
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
data/spec/exel/value_spec.rb
CHANGED
@@ -1,51 +1,50 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe Value do
|
4
|
-
let(:uri) { 's3://test_file.csv' }
|
5
2
|
|
6
|
-
|
3
|
+
describe EXEL::Value do
|
4
|
+
let(:uri) { 's3://test_file.csv' }
|
7
5
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
before { allow(EXEL).to receive(:remote_provider).and_return(EXEL::Providers::DummyRemoteProvider) }
|
7
|
+
|
8
|
+
describe '.remotize' do
|
9
|
+
context 'when the value is not a file' do
|
10
|
+
it 'returns the value' do
|
11
|
+
expect(EXEL::Value.remotize('test')).to eq('test')
|
13
12
|
end
|
13
|
+
end
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
[File, Tempfile].each do |file_class|
|
16
|
+
context "when the value is an instance of #{file_class}" do
|
17
|
+
let(:file) { instance_double(file_class) }
|
18
18
|
|
19
|
-
|
19
|
+
before { allow(file).to receive(:is_a?) { |klass| klass == file_class } }
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
21
|
+
it 'uploads the file using the remote provider' do
|
22
|
+
expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file)
|
23
|
+
EXEL::Value.remotize(file)
|
24
|
+
end
|
25
25
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
end
|
26
|
+
it 'returns the URI of the uploaded file' do
|
27
|
+
allow_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file).and_return(uri)
|
28
|
+
expect(EXEL::Value.remotize(file)).to eq(uri)
|
30
29
|
end
|
31
30
|
end
|
32
31
|
end
|
32
|
+
end
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
end
|
34
|
+
describe '.localize' do
|
35
|
+
context 'with a local value' do
|
36
|
+
it 'returns the value' do
|
37
|
+
expect(EXEL::Value.localize('test')).to eq('test')
|
39
38
|
end
|
39
|
+
end
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
41
|
+
context 'with a remote file' do
|
42
|
+
it 'returns the downloaded file' do
|
43
|
+
expect(EXEL::Providers::DummyRemoteProvider).to receive(:remote?).with(uri).and_return(true)
|
44
|
+
file = double(:file)
|
45
|
+
expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:download).with(uri).and_return(file)
|
46
46
|
|
47
|
-
|
48
|
-
end
|
47
|
+
expect(EXEL::Value.localize(uri)).to eq(file)
|
49
48
|
end
|
50
49
|
end
|
51
50
|
end
|
data/spec/exel_spec.rb
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
describe EXEL do
|
4
|
+
after :each do
|
5
|
+
EXEL.configure do |config|
|
6
|
+
config.remote_provider = nil # reset providers to default
|
7
|
+
config.async_provider = nil
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
3
11
|
describe '.async_provider' do
|
4
12
|
context 'with no async provider set in the configuration' do
|
5
13
|
it 'defaults to ThreadedAsyncProvider' do
|
@@ -39,11 +47,4 @@ describe EXEL do
|
|
39
47
|
end
|
40
48
|
end
|
41
49
|
end
|
42
|
-
|
43
|
-
after :each do
|
44
|
-
EXEL.configure do |config|
|
45
|
-
config.remote_provider = nil # reset providers to default
|
46
|
-
config.async_provider = nil
|
47
|
-
end
|
48
|
-
end
|
49
50
|
end
|
@@ -1,10 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
describe EXEL do
|
3
4
|
let(:context) { EXEL::Context.new(resource: csv_file, email_service: email_service, delete_resource: false) }
|
4
5
|
let(:csv_file) { File.open(File.expand_path('../../fixtures/sample.csv', __FILE__)) }
|
5
6
|
let(:email_service) { EmailService.new }
|
6
7
|
|
7
|
-
before :all do
|
8
|
+
before :all do # rubocop:disable RSpec/BeforeAfterAll
|
8
9
|
EXEL::Job.define :processing_steps do
|
9
10
|
process with: RecordLoader
|
10
11
|
process with: EmailProcessor
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,57 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: exel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yroo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: rake
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '13'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '13'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rspec
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '3'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '3'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: simplecov
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '0.17'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0.17'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: guard
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -123,33 +123,19 @@ dependencies:
|
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: 1.7.0
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
|
-
name: rubocop
|
126
|
+
name: rubocop-airbnb
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version:
|
131
|
+
version: '3.0'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version:
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: rubocop-rspec-focused
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - "~>"
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '0'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - "~>"
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '0'
|
138
|
+
version: '3.0'
|
153
139
|
- !ruby/object:Gem::Dependency
|
154
140
|
name: pry-byebug
|
155
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -175,6 +161,7 @@ files:
|
|
175
161
|
- ".gitignore"
|
176
162
|
- ".rspec"
|
177
163
|
- ".rubocop.yml"
|
164
|
+
- ".rubocop_airbnb.yml"
|
178
165
|
- ".rubocop_todo.yml"
|
179
166
|
- ".travis.yml"
|
180
167
|
- Gemfile
|
@@ -255,8 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
255
242
|
- !ruby/object:Gem::Version
|
256
243
|
version: '0'
|
257
244
|
requirements: []
|
258
|
-
|
259
|
-
rubygems_version: 2.6.13
|
245
|
+
rubygems_version: 3.1.2
|
260
246
|
signing_key:
|
261
247
|
specification_version: 4
|
262
248
|
summary: EXEL, the Elastic eXEcution Language
|