exel 1.4.0 → 1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.codeclimate.yml +4 -4
- data/.rubocop.yml +20 -12
- data/.rubocop_airbnb.yml +2 -0
- data/.travis.yml +17 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +64 -60
- data/README.md +1 -1
- data/Rakefile +1 -0
- data/exel.gemspec +4 -4
- data/lib/exel.rb +1 -0
- data/lib/exel/ast_node.rb +2 -1
- data/lib/exel/context.rb +2 -1
- data/lib/exel/deferred_context_value.rb +1 -0
- data/lib/exel/error/job_termination.rb +1 -0
- data/lib/exel/events.rb +1 -0
- data/lib/exel/instruction.rb +2 -1
- data/lib/exel/instruction_node.rb +1 -0
- data/lib/exel/job.rb +6 -4
- data/lib/exel/listen_instruction.rb +1 -0
- data/lib/exel/logging.rb +1 -0
- data/lib/exel/logging/logger_wrapper.rb +4 -1
- data/lib/exel/logging_helper.rb +1 -0
- data/lib/exel/middleware/chain.rb +1 -0
- data/lib/exel/middleware/logging.rb +1 -1
- data/lib/exel/null_instruction.rb +1 -0
- data/lib/exel/processor_helper.rb +1 -0
- data/lib/exel/processors/run_processor.rb +1 -0
- data/lib/exel/processors/split_processor.rb +2 -1
- data/lib/exel/providers/local_file_provider.rb +2 -1
- data/lib/exel/providers/threaded_async_provider.rb +1 -0
- data/lib/exel/sequence_node.rb +1 -0
- data/lib/exel/value.rb +1 -0
- data/lib/exel/version.rb +2 -1
- data/spec/exel/ast_node_spec.rb +42 -42
- data/spec/exel/context_spec.rb +76 -77
- data/spec/exel/deferred_context_value_spec.rb +41 -42
- data/spec/exel/events_spec.rb +65 -65
- data/spec/exel/instruction_node_spec.rb +16 -16
- data/spec/exel/instruction_spec.rb +46 -45
- data/spec/exel/job_spec.rb +94 -91
- data/spec/exel/listen_instruction_spec.rb +10 -10
- data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
- data/spec/exel/logging_helper_spec.rb +15 -16
- data/spec/exel/logging_spec.rb +56 -56
- data/spec/exel/middleware/chain_spec.rb +51 -53
- data/spec/exel/middleware/logging_spec.rb +21 -23
- data/spec/exel/middleware_spec.rb +49 -50
- data/spec/exel/null_instruction_spec.rb +3 -4
- data/spec/exel/processors/async_processor_spec.rb +16 -18
- data/spec/exel/processors/run_processor_spec.rb +9 -11
- data/spec/exel/processors/split_processor_spec.rb +91 -93
- data/spec/exel/providers/local_file_provider_spec.rb +25 -28
- data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
- data/spec/exel/sequence_node_spec.rb +11 -11
- data/spec/exel/value_spec.rb +32 -33
- data/spec/exel_spec.rb +8 -7
- data/spec/integration/integration_spec.rb +2 -1
- data/spec/spec_helper.rb +3 -2
- data/spec/support/integration_test_classes.rb +3 -1
- metadata +16 -30
@@ -1,33 +1,31 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Middleware
|
4
|
-
describe Logging do
|
5
|
-
TestProcessor = Class.new
|
6
2
|
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
describe EXEL::Middleware::Logging do
|
4
|
+
class TestProcessor
|
5
|
+
end
|
10
6
|
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
describe '#call' do
|
8
|
+
it 'yields to the given block' do
|
9
|
+
called = false
|
14
10
|
|
15
|
-
|
16
|
-
|
11
|
+
subject.call(Object, {}, {}) do
|
12
|
+
called = true
|
13
|
+
end
|
17
14
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
15
|
+
expect(called).to be_truthy
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'logs with context[:log_prefix] and the processor class as the prefix' do
|
19
|
+
expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][TestProcessor] ')
|
20
|
+
subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
|
21
|
+
end
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
end
|
28
|
-
end.to raise_error Exception, 're-raise me'
|
23
|
+
it 'raises rescued exceptions' do
|
24
|
+
expect do
|
25
|
+
subject.call(Object, {}, {}) do
|
26
|
+
raise Exception, 're-raise me'
|
29
27
|
end
|
30
|
-
end
|
28
|
+
end.to raise_error Exception, 're-raise me'
|
31
29
|
end
|
32
30
|
end
|
33
31
|
end
|
@@ -1,69 +1,68 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe Middleware do
|
4
|
-
class IORecorder
|
5
|
-
def initialize(input, output)
|
6
|
-
@input = input
|
7
|
-
@output = output
|
8
|
-
end
|
9
2
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
3
|
+
describe EXEL::Middleware do
|
4
|
+
class IORecorder
|
5
|
+
def initialize(input, output)
|
6
|
+
@input = input
|
7
|
+
@output = output
|
15
8
|
end
|
16
9
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
nil
|
22
|
-
end
|
10
|
+
def call(_processor, context, args)
|
11
|
+
@input << args[:input]
|
12
|
+
yield
|
13
|
+
@output << context[:output]
|
23
14
|
end
|
15
|
+
end
|
24
16
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
def process(_)
|
31
|
-
@context[:output] = @context[:input] + 1
|
32
|
-
raise 'rescue me'
|
33
|
-
end
|
17
|
+
class RescueErrors
|
18
|
+
def call(_processor, _context, _args)
|
19
|
+
yield
|
20
|
+
rescue
|
21
|
+
nil
|
34
22
|
end
|
23
|
+
end
|
35
24
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
25
|
+
class AddProcessor
|
26
|
+
def initialize(context)
|
27
|
+
@context = context
|
40
28
|
end
|
41
29
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
before do
|
46
|
-
EXEL.configure do |config|
|
47
|
-
config.middleware.add(IORecorder, input, output)
|
48
|
-
config.middleware.add(RescueErrors)
|
49
|
-
end
|
30
|
+
def process(_)
|
31
|
+
@context[:output] = @context[:input] + 1
|
32
|
+
raise 'rescue me'
|
50
33
|
end
|
34
|
+
end
|
51
35
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
config.middleware.remove(RescueErrors)
|
56
|
-
end
|
36
|
+
before :all do # rubocop:disable RSpec/BeforeAfterAll
|
37
|
+
EXEL::Job.define :middleware_test_job do
|
38
|
+
process with: AddProcessor, input: 1
|
57
39
|
end
|
40
|
+
end
|
58
41
|
|
59
|
-
|
60
|
-
|
42
|
+
let(:input) { [] }
|
43
|
+
let(:output) { [] }
|
44
|
+
|
45
|
+
before do
|
46
|
+
EXEL.configure do |config|
|
47
|
+
config.middleware.add(IORecorder, input, output)
|
48
|
+
config.middleware.add(RescueErrors)
|
61
49
|
end
|
50
|
+
end
|
62
51
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
52
|
+
after do
|
53
|
+
EXEL.configure do |config|
|
54
|
+
config.middleware.remove(IORecorder)
|
55
|
+
config.middleware.remove(RescueErrors)
|
67
56
|
end
|
68
57
|
end
|
58
|
+
|
59
|
+
it 'can configure custom middleware' do
|
60
|
+
expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'invokes the middleware around the processor' do
|
64
|
+
EXEL::Job.run(:middleware_test_job)
|
65
|
+
expect(input).to eq([1])
|
66
|
+
expect(output).to eq([2])
|
67
|
+
end
|
69
68
|
end
|
@@ -1,25 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe AsyncProcessor do
|
5
|
-
subject(:processor) { described_class.new(context) }
|
6
|
-
let(:context) { EXEL::Context.new }
|
7
|
-
let(:block) { instance_double(SequenceNode) }
|
8
2
|
|
9
|
-
|
10
|
-
|
11
|
-
end
|
3
|
+
describe EXEL::Processors::AsyncProcessor do
|
4
|
+
subject(:processor) { described_class.new(context) }
|
12
5
|
|
13
|
-
|
14
|
-
|
15
|
-
end
|
6
|
+
let(:context) { EXEL::Context.new }
|
7
|
+
let(:block) { instance_double(EXEL::SequenceNode) }
|
16
8
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
9
|
+
before do
|
10
|
+
allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'looks up the async provider on initialization' do
|
14
|
+
expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#process' do
|
18
|
+
it 'calls do_async on the async provider' do
|
19
|
+
expect(processor.provider).to receive(:do_async).with(block)
|
20
|
+
processor.process(block)
|
23
21
|
end
|
24
22
|
end
|
25
23
|
end
|
@@ -1,16 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe RunProcessor do
|
5
|
-
subject { RunProcessor.new(context) }
|
6
|
-
let(:context) { EXEL::Context.new(job: :test_job) }
|
7
2
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
describe EXEL::Processors::RunProcessor do
|
4
|
+
subject { EXEL::Processors::RunProcessor.new(context) }
|
5
|
+
|
6
|
+
let(:context) { EXEL::Context.new(job: :test_job) }
|
7
|
+
|
8
|
+
describe '#process' do
|
9
|
+
it 'runs the job named in context[:job] with the current context' do
|
10
|
+
expect(EXEL::Job).to receive(:run).with(:test_job, context)
|
11
|
+
subject.process
|
14
12
|
end
|
15
13
|
end
|
16
14
|
end
|
@@ -1,123 +1,121 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe SplitProcessor do
|
5
|
-
let(:chunk_file) { instance_double(File) }
|
6
|
-
let(:file) { create_file(1) }
|
7
|
-
let(:context) { Context.new(resource: file) }
|
8
|
-
let(:callback) { instance_double(SequenceNode) }
|
9
|
-
subject(:splitter) { SplitProcessor.new(context) }
|
10
|
-
|
11
|
-
before do
|
12
|
-
allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
|
13
|
-
allow(File).to receive(:delete)
|
14
|
-
end
|
15
2
|
|
16
|
-
|
17
|
-
|
3
|
+
describe EXEL::Processors::SplitProcessor do
|
4
|
+
subject(:splitter) { EXEL::Processors::SplitProcessor.new(context) }
|
18
5
|
|
19
|
-
|
20
|
-
|
6
|
+
let(:chunk_file) { instance_double(File) }
|
7
|
+
let(:file) { create_file(1) }
|
8
|
+
let(:context) { EXEL::Context.new(resource: file) }
|
9
|
+
let(:callback) { instance_double(EXEL::SequenceNode) }
|
21
10
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
11
|
+
before do
|
12
|
+
allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
|
13
|
+
allow(File).to receive(:delete)
|
14
|
+
end
|
26
15
|
|
27
|
-
|
28
|
-
|
16
|
+
describe '#process' do
|
17
|
+
let(:file) { create_file(3) }
|
29
18
|
|
30
|
-
|
31
|
-
|
19
|
+
it 'processes file with 3 lines line by line' do
|
20
|
+
allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
|
32
21
|
|
33
|
-
|
34
|
-
|
35
|
-
|
22
|
+
3.times do |i|
|
23
|
+
expect(splitter).to receive(:process_line).with("line#{i}", callback)
|
24
|
+
end
|
25
|
+
expect(splitter).to receive(:process_line).with(:eof, callback)
|
36
26
|
|
37
|
-
|
38
|
-
|
27
|
+
expect(File).to receive(:delete).with(file.path)
|
28
|
+
expect(file).to receive(:close)
|
39
29
|
|
40
|
-
|
41
|
-
|
42
|
-
expect(File).not_to receive(:delete).with(file.path)
|
30
|
+
splitter.process(callback)
|
31
|
+
end
|
43
32
|
|
44
|
-
|
45
|
-
|
46
|
-
|
33
|
+
it 'aborts parsing the csv file if it is malformed' do
|
34
|
+
allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError.new('message', '1'))
|
35
|
+
expect(splitter).to receive(:process_line).with(:eof, callback)
|
47
36
|
|
48
|
-
|
49
|
-
|
37
|
+
splitter.process(callback)
|
38
|
+
end
|
50
39
|
|
51
|
-
|
40
|
+
it 'does not delete the resource file if :delete_resource is set to false in the context' do
|
41
|
+
allow(CSV).to receive(:foreach).and_yield(:eof)
|
42
|
+
expect(File).not_to receive(:delete).with(file.path)
|
52
43
|
|
53
|
-
|
54
|
-
|
55
|
-
|
44
|
+
context[:delete_resource] = false
|
45
|
+
splitter.process(callback)
|
46
|
+
end
|
56
47
|
|
57
|
-
|
48
|
+
it 'stops splitting at :max_chunks if it is set in the context' do
|
49
|
+
allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
|
58
50
|
|
59
|
-
|
60
|
-
context[:max_chunks] = 1
|
61
|
-
splitter.process(callback)
|
51
|
+
chunk_file = create_file(0)
|
62
52
|
|
63
|
-
|
64
|
-
|
53
|
+
allow(Tempfile).to receive(:new).and_return(chunk_file)
|
54
|
+
expect(callback).to receive(:run).once
|
55
|
+
allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
|
65
56
|
|
66
|
-
|
67
|
-
allow(CSV).to receive(:foreach).and_raise(Interrupt)
|
57
|
+
expect(File).to receive(:delete).with(file.path)
|
68
58
|
|
69
|
-
|
70
|
-
|
59
|
+
context[:chunk_size] = 2
|
60
|
+
context[:max_chunks] = 1
|
61
|
+
splitter.process(callback)
|
71
62
|
|
72
|
-
|
73
|
-
|
74
|
-
rescue Interrupt
|
75
|
-
nil
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
63
|
+
expect(chunk_file.read).to eq("line0\nline1\n")
|
64
|
+
end
|
79
65
|
|
80
|
-
|
81
|
-
|
82
|
-
{input: 1, chunks: %W(0\n)},
|
83
|
-
{input: 3, chunks: %W(0\n1\n 2\n)},
|
84
|
-
{input: 4, chunks: %W(0\n1\n 2\n3\n)}
|
85
|
-
].each do |data|
|
86
|
-
it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
|
87
|
-
context[:chunk_size] = 2
|
88
|
-
|
89
|
-
data[:chunks].each do |chunk|
|
90
|
-
expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
|
91
|
-
expect(callback).to receive(:run).with(context) do
|
92
|
-
expect(context[:resource]).to eq(chunk_file)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
data[:input].times { |i| splitter.process_line([i.to_s], callback) }
|
97
|
-
splitter.process_line(:eof, callback)
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
66
|
+
it 'ensures that the source file gets closed and deleted' do
|
67
|
+
allow(CSV).to receive(:foreach).and_raise(Interrupt)
|
101
68
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
69
|
+
expect(File).to receive(:delete).with(file.path)
|
70
|
+
expect(file).to receive(:close)
|
71
|
+
|
72
|
+
begin
|
73
|
+
splitter.process(callback)
|
74
|
+
rescue Interrupt
|
75
|
+
nil
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
108
79
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
80
|
+
describe '#process_line' do
|
81
|
+
[
|
82
|
+
{input: 1, chunks: %W(0\n)},
|
83
|
+
{input: 3, chunks: %W(0\n1\n 2\n)},
|
84
|
+
{input: 4, chunks: %W(0\n1\n 2\n3\n)},
|
85
|
+
].each do |data|
|
86
|
+
it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
|
87
|
+
context[:chunk_size] = 2
|
88
|
+
|
89
|
+
data[:chunks].each do |chunk|
|
90
|
+
expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
|
91
|
+
expect(callback).to receive(:run).with(context) do
|
92
|
+
expect(context[:resource]).to eq(chunk_file)
|
113
93
|
end
|
114
94
|
end
|
95
|
+
|
96
|
+
data[:input].times { |i| splitter.process_line([i.to_s], callback) }
|
97
|
+
splitter.process_line(:eof, callback)
|
115
98
|
end
|
99
|
+
end
|
100
|
+
end
|
116
101
|
|
117
|
-
|
118
|
-
|
119
|
-
|
102
|
+
describe '#generate_chunk' do
|
103
|
+
it 'creates a file with the contents of the given string' do
|
104
|
+
file = splitter.generate_chunk('abc')
|
105
|
+
content = file.read
|
106
|
+
expect(content).to eq('abc')
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'creates a file with a unique name' do
|
110
|
+
3.times do |i|
|
111
|
+
file = splitter.generate_chunk('content')
|
112
|
+
expect(file.path).to include("text_#{i + 1}_")
|
120
113
|
end
|
121
114
|
end
|
122
115
|
end
|
116
|
+
|
117
|
+
def create_file(lines)
|
118
|
+
content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
|
119
|
+
StringIO.new(content)
|
120
|
+
end
|
123
121
|
end
|