exel 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.codeclimate.yml +4 -4
- data/.rubocop.yml +20 -12
- data/.rubocop_airbnb.yml +2 -0
- data/.travis.yml +17 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +64 -60
- data/README.md +1 -1
- data/Rakefile +1 -0
- data/exel.gemspec +4 -4
- data/lib/exel.rb +1 -0
- data/lib/exel/ast_node.rb +2 -1
- data/lib/exel/context.rb +2 -1
- data/lib/exel/deferred_context_value.rb +1 -0
- data/lib/exel/error/job_termination.rb +1 -0
- data/lib/exel/events.rb +1 -0
- data/lib/exel/instruction.rb +2 -1
- data/lib/exel/instruction_node.rb +1 -0
- data/lib/exel/job.rb +6 -4
- data/lib/exel/listen_instruction.rb +1 -0
- data/lib/exel/logging.rb +1 -0
- data/lib/exel/logging/logger_wrapper.rb +4 -1
- data/lib/exel/logging_helper.rb +1 -0
- data/lib/exel/middleware/chain.rb +1 -0
- data/lib/exel/middleware/logging.rb +1 -1
- data/lib/exel/null_instruction.rb +1 -0
- data/lib/exel/processor_helper.rb +1 -0
- data/lib/exel/processors/run_processor.rb +1 -0
- data/lib/exel/processors/split_processor.rb +2 -1
- data/lib/exel/providers/local_file_provider.rb +2 -1
- data/lib/exel/providers/threaded_async_provider.rb +1 -0
- data/lib/exel/sequence_node.rb +1 -0
- data/lib/exel/value.rb +1 -0
- data/lib/exel/version.rb +2 -1
- data/spec/exel/ast_node_spec.rb +42 -42
- data/spec/exel/context_spec.rb +76 -77
- data/spec/exel/deferred_context_value_spec.rb +41 -42
- data/spec/exel/events_spec.rb +65 -65
- data/spec/exel/instruction_node_spec.rb +16 -16
- data/spec/exel/instruction_spec.rb +46 -45
- data/spec/exel/job_spec.rb +94 -91
- data/spec/exel/listen_instruction_spec.rb +10 -10
- data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
- data/spec/exel/logging_helper_spec.rb +15 -16
- data/spec/exel/logging_spec.rb +56 -56
- data/spec/exel/middleware/chain_spec.rb +51 -53
- data/spec/exel/middleware/logging_spec.rb +21 -23
- data/spec/exel/middleware_spec.rb +49 -50
- data/spec/exel/null_instruction_spec.rb +3 -4
- data/spec/exel/processors/async_processor_spec.rb +16 -18
- data/spec/exel/processors/run_processor_spec.rb +9 -11
- data/spec/exel/processors/split_processor_spec.rb +91 -93
- data/spec/exel/providers/local_file_provider_spec.rb +25 -28
- data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
- data/spec/exel/sequence_node_spec.rb +11 -11
- data/spec/exel/value_spec.rb +32 -33
- data/spec/exel_spec.rb +8 -7
- data/spec/integration/integration_spec.rb +2 -1
- data/spec/spec_helper.rb +3 -2
- data/spec/support/integration_test_classes.rb +3 -1
- metadata +16 -30
@@ -1,33 +1,31 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Middleware
|
4
|
-
describe Logging do
|
5
|
-
TestProcessor = Class.new
|
6
2
|
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
describe EXEL::Middleware::Logging do
|
4
|
+
class TestProcessor
|
5
|
+
end
|
10
6
|
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
describe '#call' do
|
8
|
+
it 'yields to the given block' do
|
9
|
+
called = false
|
14
10
|
|
15
|
-
|
16
|
-
|
11
|
+
subject.call(Object, {}, {}) do
|
12
|
+
called = true
|
13
|
+
end
|
17
14
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
15
|
+
expect(called).to be_truthy
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'logs with context[:log_prefix] and the processor class as the prefix' do
|
19
|
+
expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][TestProcessor] ')
|
20
|
+
subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
|
21
|
+
end
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
end
|
28
|
-
end.to raise_error Exception, 're-raise me'
|
23
|
+
it 'raises rescued exceptions' do
|
24
|
+
expect do
|
25
|
+
subject.call(Object, {}, {}) do
|
26
|
+
raise Exception, 're-raise me'
|
29
27
|
end
|
30
|
-
end
|
28
|
+
end.to raise_error Exception, 're-raise me'
|
31
29
|
end
|
32
30
|
end
|
33
31
|
end
|
@@ -1,69 +1,68 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
describe Middleware do
|
4
|
-
class IORecorder
|
5
|
-
def initialize(input, output)
|
6
|
-
@input = input
|
7
|
-
@output = output
|
8
|
-
end
|
9
2
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
3
|
+
describe EXEL::Middleware do
|
4
|
+
class IORecorder
|
5
|
+
def initialize(input, output)
|
6
|
+
@input = input
|
7
|
+
@output = output
|
15
8
|
end
|
16
9
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
nil
|
22
|
-
end
|
10
|
+
def call(_processor, context, args)
|
11
|
+
@input << args[:input]
|
12
|
+
yield
|
13
|
+
@output << context[:output]
|
23
14
|
end
|
15
|
+
end
|
24
16
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
def process(_)
|
31
|
-
@context[:output] = @context[:input] + 1
|
32
|
-
raise 'rescue me'
|
33
|
-
end
|
17
|
+
class RescueErrors
|
18
|
+
def call(_processor, _context, _args)
|
19
|
+
yield
|
20
|
+
rescue
|
21
|
+
nil
|
34
22
|
end
|
23
|
+
end
|
35
24
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
25
|
+
class AddProcessor
|
26
|
+
def initialize(context)
|
27
|
+
@context = context
|
40
28
|
end
|
41
29
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
before do
|
46
|
-
EXEL.configure do |config|
|
47
|
-
config.middleware.add(IORecorder, input, output)
|
48
|
-
config.middleware.add(RescueErrors)
|
49
|
-
end
|
30
|
+
def process(_)
|
31
|
+
@context[:output] = @context[:input] + 1
|
32
|
+
raise 'rescue me'
|
50
33
|
end
|
34
|
+
end
|
51
35
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
config.middleware.remove(RescueErrors)
|
56
|
-
end
|
36
|
+
before :all do # rubocop:disable RSpec/BeforeAfterAll
|
37
|
+
EXEL::Job.define :middleware_test_job do
|
38
|
+
process with: AddProcessor, input: 1
|
57
39
|
end
|
40
|
+
end
|
58
41
|
|
59
|
-
|
60
|
-
|
42
|
+
let(:input) { [] }
|
43
|
+
let(:output) { [] }
|
44
|
+
|
45
|
+
before do
|
46
|
+
EXEL.configure do |config|
|
47
|
+
config.middleware.add(IORecorder, input, output)
|
48
|
+
config.middleware.add(RescueErrors)
|
61
49
|
end
|
50
|
+
end
|
62
51
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
52
|
+
after do
|
53
|
+
EXEL.configure do |config|
|
54
|
+
config.middleware.remove(IORecorder)
|
55
|
+
config.middleware.remove(RescueErrors)
|
67
56
|
end
|
68
57
|
end
|
58
|
+
|
59
|
+
it 'can configure custom middleware' do
|
60
|
+
expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'invokes the middleware around the processor' do
|
64
|
+
EXEL::Job.run(:middleware_test_job)
|
65
|
+
expect(input).to eq([1])
|
66
|
+
expect(output).to eq([2])
|
67
|
+
end
|
69
68
|
end
|
@@ -1,25 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe AsyncProcessor do
|
5
|
-
subject(:processor) { described_class.new(context) }
|
6
|
-
let(:context) { EXEL::Context.new }
|
7
|
-
let(:block) { instance_double(SequenceNode) }
|
8
2
|
|
9
|
-
|
10
|
-
|
11
|
-
end
|
3
|
+
describe EXEL::Processors::AsyncProcessor do
|
4
|
+
subject(:processor) { described_class.new(context) }
|
12
5
|
|
13
|
-
|
14
|
-
|
15
|
-
end
|
6
|
+
let(:context) { EXEL::Context.new }
|
7
|
+
let(:block) { instance_double(EXEL::SequenceNode) }
|
16
8
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
9
|
+
before do
|
10
|
+
allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'looks up the async provider on initialization' do
|
14
|
+
expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#process' do
|
18
|
+
it 'calls do_async on the async provider' do
|
19
|
+
expect(processor.provider).to receive(:do_async).with(block)
|
20
|
+
processor.process(block)
|
23
21
|
end
|
24
22
|
end
|
25
23
|
end
|
@@ -1,16 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe RunProcessor do
|
5
|
-
subject { RunProcessor.new(context) }
|
6
|
-
let(:context) { EXEL::Context.new(job: :test_job) }
|
7
2
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
describe EXEL::Processors::RunProcessor do
|
4
|
+
subject { EXEL::Processors::RunProcessor.new(context) }
|
5
|
+
|
6
|
+
let(:context) { EXEL::Context.new(job: :test_job) }
|
7
|
+
|
8
|
+
describe '#process' do
|
9
|
+
it 'runs the job named in context[:job] with the current context' do
|
10
|
+
expect(EXEL::Job).to receive(:run).with(:test_job, context)
|
11
|
+
subject.process
|
14
12
|
end
|
15
13
|
end
|
16
14
|
end
|
@@ -1,123 +1,121 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module EXEL
|
3
|
-
module Processors
|
4
|
-
describe SplitProcessor do
|
5
|
-
let(:chunk_file) { instance_double(File) }
|
6
|
-
let(:file) { create_file(1) }
|
7
|
-
let(:context) { Context.new(resource: file) }
|
8
|
-
let(:callback) { instance_double(SequenceNode) }
|
9
|
-
subject(:splitter) { SplitProcessor.new(context) }
|
10
|
-
|
11
|
-
before do
|
12
|
-
allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
|
13
|
-
allow(File).to receive(:delete)
|
14
|
-
end
|
15
2
|
|
16
|
-
|
17
|
-
|
3
|
+
describe EXEL::Processors::SplitProcessor do
|
4
|
+
subject(:splitter) { EXEL::Processors::SplitProcessor.new(context) }
|
18
5
|
|
19
|
-
|
20
|
-
|
6
|
+
let(:chunk_file) { instance_double(File) }
|
7
|
+
let(:file) { create_file(1) }
|
8
|
+
let(:context) { EXEL::Context.new(resource: file) }
|
9
|
+
let(:callback) { instance_double(EXEL::SequenceNode) }
|
21
10
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
11
|
+
before do
|
12
|
+
allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
|
13
|
+
allow(File).to receive(:delete)
|
14
|
+
end
|
26
15
|
|
27
|
-
|
28
|
-
|
16
|
+
describe '#process' do
|
17
|
+
let(:file) { create_file(3) }
|
29
18
|
|
30
|
-
|
31
|
-
|
19
|
+
it 'processes file with 3 lines line by line' do
|
20
|
+
allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
|
32
21
|
|
33
|
-
|
34
|
-
|
35
|
-
|
22
|
+
3.times do |i|
|
23
|
+
expect(splitter).to receive(:process_line).with("line#{i}", callback)
|
24
|
+
end
|
25
|
+
expect(splitter).to receive(:process_line).with(:eof, callback)
|
36
26
|
|
37
|
-
|
38
|
-
|
27
|
+
expect(File).to receive(:delete).with(file.path)
|
28
|
+
expect(file).to receive(:close)
|
39
29
|
|
40
|
-
|
41
|
-
|
42
|
-
expect(File).not_to receive(:delete).with(file.path)
|
30
|
+
splitter.process(callback)
|
31
|
+
end
|
43
32
|
|
44
|
-
|
45
|
-
|
46
|
-
|
33
|
+
it 'aborts parsing the csv file if it is malformed' do
|
34
|
+
allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError.new('message', '1'))
|
35
|
+
expect(splitter).to receive(:process_line).with(:eof, callback)
|
47
36
|
|
48
|
-
|
49
|
-
|
37
|
+
splitter.process(callback)
|
38
|
+
end
|
50
39
|
|
51
|
-
|
40
|
+
it 'does not delete the resource file if :delete_resource is set to false in the context' do
|
41
|
+
allow(CSV).to receive(:foreach).and_yield(:eof)
|
42
|
+
expect(File).not_to receive(:delete).with(file.path)
|
52
43
|
|
53
|
-
|
54
|
-
|
55
|
-
|
44
|
+
context[:delete_resource] = false
|
45
|
+
splitter.process(callback)
|
46
|
+
end
|
56
47
|
|
57
|
-
|
48
|
+
it 'stops splitting at :max_chunks if it is set in the context' do
|
49
|
+
allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
|
58
50
|
|
59
|
-
|
60
|
-
context[:max_chunks] = 1
|
61
|
-
splitter.process(callback)
|
51
|
+
chunk_file = create_file(0)
|
62
52
|
|
63
|
-
|
64
|
-
|
53
|
+
allow(Tempfile).to receive(:new).and_return(chunk_file)
|
54
|
+
expect(callback).to receive(:run).once
|
55
|
+
allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
|
65
56
|
|
66
|
-
|
67
|
-
allow(CSV).to receive(:foreach).and_raise(Interrupt)
|
57
|
+
expect(File).to receive(:delete).with(file.path)
|
68
58
|
|
69
|
-
|
70
|
-
|
59
|
+
context[:chunk_size] = 2
|
60
|
+
context[:max_chunks] = 1
|
61
|
+
splitter.process(callback)
|
71
62
|
|
72
|
-
|
73
|
-
|
74
|
-
rescue Interrupt
|
75
|
-
nil
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
63
|
+
expect(chunk_file.read).to eq("line0\nline1\n")
|
64
|
+
end
|
79
65
|
|
80
|
-
|
81
|
-
|
82
|
-
{input: 1, chunks: %W(0\n)},
|
83
|
-
{input: 3, chunks: %W(0\n1\n 2\n)},
|
84
|
-
{input: 4, chunks: %W(0\n1\n 2\n3\n)}
|
85
|
-
].each do |data|
|
86
|
-
it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
|
87
|
-
context[:chunk_size] = 2
|
88
|
-
|
89
|
-
data[:chunks].each do |chunk|
|
90
|
-
expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
|
91
|
-
expect(callback).to receive(:run).with(context) do
|
92
|
-
expect(context[:resource]).to eq(chunk_file)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
data[:input].times { |i| splitter.process_line([i.to_s], callback) }
|
97
|
-
splitter.process_line(:eof, callback)
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
66
|
+
it 'ensures that the source file gets closed and deleted' do
|
67
|
+
allow(CSV).to receive(:foreach).and_raise(Interrupt)
|
101
68
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
69
|
+
expect(File).to receive(:delete).with(file.path)
|
70
|
+
expect(file).to receive(:close)
|
71
|
+
|
72
|
+
begin
|
73
|
+
splitter.process(callback)
|
74
|
+
rescue Interrupt
|
75
|
+
nil
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
108
79
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
80
|
+
describe '#process_line' do
|
81
|
+
[
|
82
|
+
{input: 1, chunks: %W(0\n)},
|
83
|
+
{input: 3, chunks: %W(0\n1\n 2\n)},
|
84
|
+
{input: 4, chunks: %W(0\n1\n 2\n3\n)},
|
85
|
+
].each do |data|
|
86
|
+
it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
|
87
|
+
context[:chunk_size] = 2
|
88
|
+
|
89
|
+
data[:chunks].each do |chunk|
|
90
|
+
expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
|
91
|
+
expect(callback).to receive(:run).with(context) do
|
92
|
+
expect(context[:resource]).to eq(chunk_file)
|
113
93
|
end
|
114
94
|
end
|
95
|
+
|
96
|
+
data[:input].times { |i| splitter.process_line([i.to_s], callback) }
|
97
|
+
splitter.process_line(:eof, callback)
|
115
98
|
end
|
99
|
+
end
|
100
|
+
end
|
116
101
|
|
117
|
-
|
118
|
-
|
119
|
-
|
102
|
+
describe '#generate_chunk' do
|
103
|
+
it 'creates a file with the contents of the given string' do
|
104
|
+
file = splitter.generate_chunk('abc')
|
105
|
+
content = file.read
|
106
|
+
expect(content).to eq('abc')
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'creates a file with a unique name' do
|
110
|
+
3.times do |i|
|
111
|
+
file = splitter.generate_chunk('content')
|
112
|
+
expect(file.path).to include("text_#{i + 1}_")
|
120
113
|
end
|
121
114
|
end
|
122
115
|
end
|
116
|
+
|
117
|
+
def create_file(lines)
|
118
|
+
content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
|
119
|
+
StringIO.new(content)
|
120
|
+
end
|
123
121
|
end
|