exel 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.codeclimate.yml +4 -4
- data/.rubocop.yml +20 -12
- data/.rubocop_airbnb.yml +2 -0
- data/.travis.yml +17 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +64 -60
- data/README.md +1 -1
- data/Rakefile +1 -0
- data/exel.gemspec +4 -4
- data/lib/exel.rb +1 -0
- data/lib/exel/ast_node.rb +2 -1
- data/lib/exel/context.rb +2 -1
- data/lib/exel/deferred_context_value.rb +1 -0
- data/lib/exel/error/job_termination.rb +1 -0
- data/lib/exel/events.rb +1 -0
- data/lib/exel/instruction.rb +2 -1
- data/lib/exel/instruction_node.rb +1 -0
- data/lib/exel/job.rb +6 -4
- data/lib/exel/listen_instruction.rb +1 -0
- data/lib/exel/logging.rb +1 -0
- data/lib/exel/logging/logger_wrapper.rb +4 -1
- data/lib/exel/logging_helper.rb +1 -0
- data/lib/exel/middleware/chain.rb +1 -0
- data/lib/exel/middleware/logging.rb +1 -1
- data/lib/exel/null_instruction.rb +1 -0
- data/lib/exel/processor_helper.rb +1 -0
- data/lib/exel/processors/run_processor.rb +1 -0
- data/lib/exel/processors/split_processor.rb +2 -1
- data/lib/exel/providers/local_file_provider.rb +2 -1
- data/lib/exel/providers/threaded_async_provider.rb +1 -0
- data/lib/exel/sequence_node.rb +1 -0
- data/lib/exel/value.rb +1 -0
- data/lib/exel/version.rb +2 -1
- data/spec/exel/ast_node_spec.rb +42 -42
- data/spec/exel/context_spec.rb +76 -77
- data/spec/exel/deferred_context_value_spec.rb +41 -42
- data/spec/exel/events_spec.rb +65 -65
- data/spec/exel/instruction_node_spec.rb +16 -16
- data/spec/exel/instruction_spec.rb +46 -45
- data/spec/exel/job_spec.rb +94 -91
- data/spec/exel/listen_instruction_spec.rb +10 -10
- data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
- data/spec/exel/logging_helper_spec.rb +15 -16
- data/spec/exel/logging_spec.rb +56 -56
- data/spec/exel/middleware/chain_spec.rb +51 -53
- data/spec/exel/middleware/logging_spec.rb +21 -23
- data/spec/exel/middleware_spec.rb +49 -50
- data/spec/exel/null_instruction_spec.rb +3 -4
- data/spec/exel/processors/async_processor_spec.rb +16 -18
- data/spec/exel/processors/run_processor_spec.rb +9 -11
- data/spec/exel/processors/split_processor_spec.rb +91 -93
- data/spec/exel/providers/local_file_provider_spec.rb +25 -28
- data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
- data/spec/exel/sequence_node_spec.rb +11 -11
- data/spec/exel/value_spec.rb +32 -33
- data/spec/exel_spec.rb +8 -7
- data/spec/integration/integration_spec.rb +2 -1
- data/spec/spec_helper.rb +3 -2
- data/spec/support/integration_test_classes.rb +3 -1
- metadata +16 -30
@@ -1,37 +1,34 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Providers
|
4
|
-
describe LocalFileProvider do
|
5
|
-
let(:file) { instance_double(File, path: '/path/to/file') }
|
6
2
|
|
7
|
-
|
8
|
-
|
9
|
-
expect(subject.upload(file)).to eq('file:///path/to/file')
|
10
|
-
end
|
11
|
-
end
|
3
|
+
describe EXEL::Providers::LocalFileProvider do
|
4
|
+
let(:file) { instance_double(File, path: '/path/to/file') }
|
12
5
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
6
|
+
describe '#upload' do
|
7
|
+
it 'returns a file:// URI for the file' do
|
8
|
+
expect(subject.upload(file)).to eq('file:///path/to/file')
|
9
|
+
end
|
10
|
+
end
|
18
11
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
12
|
+
describe '#download' do
|
13
|
+
it 'returns the file indicated by the URI' do
|
14
|
+
expect(File).to receive(:open).with('/path/to/file').and_return(file)
|
15
|
+
expect(subject.download('file:///path/to/file')).to eq(file)
|
16
|
+
end
|
23
17
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
18
|
+
it 'doesn`t accept URIs for schemes other than file://' do
|
19
|
+
expect { subject.download('s3://') }.to raise_error 'URI must begin with "file://"'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe '.remote?' do
|
24
|
+
it 'returns true for file:// URIs' do
|
25
|
+
expect(EXEL::Providers::LocalFileProvider.remote?('file:///path/to/file')).to be_truthy
|
26
|
+
end
|
28
27
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
28
|
+
it 'returns false for anything else' do
|
29
|
+
expect(EXEL::Providers::LocalFileProvider.remote?('s3://file')).to be_falsey
|
30
|
+
expect(EXEL::Providers::LocalFileProvider.remote?(1)).to be_falsey
|
31
|
+
expect(EXEL::Providers::LocalFileProvider.remote?(nil)).to be_falsey
|
35
32
|
end
|
36
33
|
end
|
37
34
|
end
|
@@ -1,54 +1,52 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Providers
|
4
|
-
class ContextMutatingProcessor
|
5
|
-
def initialize(context)
|
6
|
-
@context = context
|
7
|
-
end
|
8
2
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
class ContextMutatingProcessor
|
4
|
+
def initialize(context)
|
5
|
+
@context = context
|
6
|
+
end
|
13
7
|
|
14
|
-
|
15
|
-
|
16
|
-
|
8
|
+
def process(_block)
|
9
|
+
@context[:array] << @context[:arg]
|
10
|
+
end
|
11
|
+
end
|
17
12
|
|
18
|
-
|
19
|
-
|
13
|
+
describe EXEL::Providers::ThreadedAsyncProvider do
|
14
|
+
subject { described_class.new(context) }
|
20
15
|
|
21
|
-
|
22
|
-
expect(dsl_block).to receive(:start).with(context)
|
23
|
-
expect(Thread).to receive(:new).and_yield
|
16
|
+
let(:context) { EXEL::Context.new }
|
24
17
|
|
25
|
-
|
26
|
-
|
18
|
+
describe '#do_async' do
|
19
|
+
let(:dsl_block) { instance_double(EXEL::ASTNode) }
|
27
20
|
|
28
|
-
|
29
|
-
|
30
|
-
|
21
|
+
it 'runs the block in a new thread' do
|
22
|
+
expect(dsl_block).to receive(:start).with(context)
|
23
|
+
expect(Thread).to receive(:new).and_yield
|
31
24
|
|
32
|
-
|
33
|
-
|
34
|
-
process with: ContextMutatingProcessor, arg: 1
|
35
|
-
complete += 1
|
36
|
-
end
|
37
|
-
|
38
|
-
async do
|
39
|
-
process with: ContextMutatingProcessor, arg: 2
|
40
|
-
complete += 1
|
41
|
-
end
|
42
|
-
end
|
25
|
+
subject.do_async(dsl_block)
|
26
|
+
end
|
43
27
|
|
44
|
-
|
28
|
+
it 'passes a copy of the context to each thread' do
|
29
|
+
context[:array] = []
|
30
|
+
complete = 0
|
45
31
|
|
46
|
-
|
47
|
-
|
32
|
+
EXEL::Job.define :thread_test do
|
33
|
+
async do
|
34
|
+
process with: ContextMutatingProcessor, arg: 1
|
35
|
+
complete += 1
|
36
|
+
end
|
48
37
|
|
49
|
-
|
38
|
+
async do
|
39
|
+
process with: ContextMutatingProcessor, arg: 2
|
40
|
+
complete += 1
|
50
41
|
end
|
51
42
|
end
|
43
|
+
|
44
|
+
EXEL::Job.run(:thread_test, context)
|
45
|
+
|
46
|
+
start_time = Time.now
|
47
|
+
sleep 0.1 while complete < 2 && Time.now - start_time < 2
|
48
|
+
|
49
|
+
expect(context[:array]).to be_empty
|
52
50
|
end
|
53
51
|
end
|
54
52
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe SequenceNode do
|
4
|
-
subject(:node) { described_class.new(instance_double(ASTNode), instance_double(ASTNode)) }
|
5
|
-
let(:context) { instance_double(EXEL::Context) }
|
6
2
|
|
7
|
-
|
3
|
+
describe EXEL::SequenceNode do
|
4
|
+
subject(:node) { described_class.new(instance_double(EXEL::ASTNode), instance_double(EXEL::ASTNode)) }
|
8
5
|
|
9
|
-
|
10
|
-
it 'runs each child node in sequence' do
|
11
|
-
expect(node.children.first).to receive(:run).with(context).once.ordered
|
12
|
-
expect(node.children.last).to receive(:run).with(context).once.ordered
|
6
|
+
let(:context) { instance_double(EXEL::Context) }
|
13
7
|
|
14
|
-
|
15
|
-
|
8
|
+
it { is_expected.to be_an(EXEL::ASTNode) }
|
9
|
+
|
10
|
+
describe '#run' do
|
11
|
+
it 'runs each child node in sequence' do
|
12
|
+
expect(node.children.first).to receive(:run).with(context).once.ordered
|
13
|
+
expect(node.children.last).to receive(:run).with(context).once.ordered
|
14
|
+
|
15
|
+
node.run(context)
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
data/spec/exel/value_spec.rb
CHANGED
@@ -1,51 +1,50 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe Value do
|
4
|
-
let(:uri) { 's3://test_file.csv' }
|
5
2
|
|
6
|
-
|
3
|
+
describe EXEL::Value do
|
4
|
+
let(:uri) { 's3://test_file.csv' }
|
7
5
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
before { allow(EXEL).to receive(:remote_provider).and_return(EXEL::Providers::DummyRemoteProvider) }
|
7
|
+
|
8
|
+
describe '.remotize' do
|
9
|
+
context 'when the value is not a file' do
|
10
|
+
it 'returns the value' do
|
11
|
+
expect(EXEL::Value.remotize('test')).to eq('test')
|
13
12
|
end
|
13
|
+
end
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
[File, Tempfile].each do |file_class|
|
16
|
+
context "when the value is an instance of #{file_class}" do
|
17
|
+
let(:file) { instance_double(file_class) }
|
18
18
|
|
19
|
-
|
19
|
+
before { allow(file).to receive(:is_a?) { |klass| klass == file_class } }
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
21
|
+
it 'uploads the file using the remote provider' do
|
22
|
+
expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file)
|
23
|
+
EXEL::Value.remotize(file)
|
24
|
+
end
|
25
25
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
end
|
26
|
+
it 'returns the URI of the uploaded file' do
|
27
|
+
allow_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file).and_return(uri)
|
28
|
+
expect(EXEL::Value.remotize(file)).to eq(uri)
|
30
29
|
end
|
31
30
|
end
|
32
31
|
end
|
32
|
+
end
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
end
|
34
|
+
describe '.localize' do
|
35
|
+
context 'with a local value' do
|
36
|
+
it 'returns the value' do
|
37
|
+
expect(EXEL::Value.localize('test')).to eq('test')
|
39
38
|
end
|
39
|
+
end
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
41
|
+
context 'with a remote file' do
|
42
|
+
it 'returns the downloaded file' do
|
43
|
+
expect(EXEL::Providers::DummyRemoteProvider).to receive(:remote?).with(uri).and_return(true)
|
44
|
+
file = double(:file)
|
45
|
+
expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:download).with(uri).and_return(file)
|
46
46
|
|
47
|
-
|
48
|
-
end
|
47
|
+
expect(EXEL::Value.localize(uri)).to eq(file)
|
49
48
|
end
|
50
49
|
end
|
51
50
|
end
|
data/spec/exel_spec.rb
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
describe EXEL do
|
4
|
+
after :each do
|
5
|
+
EXEL.configure do |config|
|
6
|
+
config.remote_provider = nil # reset providers to default
|
7
|
+
config.async_provider = nil
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
3
11
|
describe '.async_provider' do
|
4
12
|
context 'with no async provider set in the configuration' do
|
5
13
|
it 'defaults to ThreadedAsyncProvider' do
|
@@ -39,11 +47,4 @@ describe EXEL do
|
|
39
47
|
end
|
40
48
|
end
|
41
49
|
end
|
42
|
-
|
43
|
-
after :each do
|
44
|
-
EXEL.configure do |config|
|
45
|
-
config.remote_provider = nil # reset providers to default
|
46
|
-
config.async_provider = nil
|
47
|
-
end
|
48
|
-
end
|
49
50
|
end
|
@@ -1,10 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
describe EXEL do
|
3
4
|
let(:context) { EXEL::Context.new(resource: csv_file, email_service: email_service, delete_resource: false) }
|
4
5
|
let(:csv_file) { File.open(File.expand_path('../../fixtures/sample.csv', __FILE__)) }
|
5
6
|
let(:email_service) { EmailService.new }
|
6
7
|
|
7
|
-
before :all do
|
8
|
+
before :all do # rubocop:disable RSpec/BeforeAfterAll
|
8
9
|
EXEL::Job.define :processing_steps do
|
9
10
|
process with: RecordLoader
|
10
11
|
process with: EmailProcessor
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,57 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: exel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yroo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: rake
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '13'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '13'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rspec
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '3'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '3'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: simplecov
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '0.17'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0.17'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: guard
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -123,33 +123,19 @@ dependencies:
|
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: 1.7.0
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
|
-
name: rubocop
|
126
|
+
name: rubocop-airbnb
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version:
|
131
|
+
version: '3.0'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version:
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: rubocop-rspec-focused
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - "~>"
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '0'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - "~>"
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '0'
|
138
|
+
version: '3.0'
|
153
139
|
- !ruby/object:Gem::Dependency
|
154
140
|
name: pry-byebug
|
155
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -175,6 +161,7 @@ files:
|
|
175
161
|
- ".gitignore"
|
176
162
|
- ".rspec"
|
177
163
|
- ".rubocop.yml"
|
164
|
+
- ".rubocop_airbnb.yml"
|
178
165
|
- ".rubocop_todo.yml"
|
179
166
|
- ".travis.yml"
|
180
167
|
- Gemfile
|
@@ -255,8 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
255
242
|
- !ruby/object:Gem::Version
|
256
243
|
version: '0'
|
257
244
|
requirements: []
|
258
|
-
|
259
|
-
rubygems_version: 2.6.13
|
245
|
+
rubygems_version: 3.1.2
|
260
246
|
signing_key:
|
261
247
|
specification_version: 4
|
262
248
|
summary: EXEL, the Elastic eXEcution Language
|