exel 1.4.0 → 1.5.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +5 -5
  2. data/.codeclimate.yml +4 -4
  3. data/.rubocop.yml +20 -12
  4. data/.rubocop_airbnb.yml +2 -0
  5. data/.travis.yml +17 -4
  6. data/Gemfile +1 -2
  7. data/Gemfile.lock +64 -60
  8. data/README.md +1 -1
  9. data/Rakefile +1 -0
  10. data/exel.gemspec +4 -4
  11. data/lib/exel.rb +1 -0
  12. data/lib/exel/ast_node.rb +2 -1
  13. data/lib/exel/context.rb +2 -1
  14. data/lib/exel/deferred_context_value.rb +1 -0
  15. data/lib/exel/error/job_termination.rb +1 -0
  16. data/lib/exel/events.rb +1 -0
  17. data/lib/exel/instruction.rb +2 -1
  18. data/lib/exel/instruction_node.rb +1 -0
  19. data/lib/exel/job.rb +6 -4
  20. data/lib/exel/listen_instruction.rb +1 -0
  21. data/lib/exel/logging.rb +1 -0
  22. data/lib/exel/logging/logger_wrapper.rb +4 -1
  23. data/lib/exel/logging_helper.rb +1 -0
  24. data/lib/exel/middleware/chain.rb +1 -0
  25. data/lib/exel/middleware/logging.rb +1 -1
  26. data/lib/exel/null_instruction.rb +1 -0
  27. data/lib/exel/processor_helper.rb +1 -0
  28. data/lib/exel/processors/run_processor.rb +1 -0
  29. data/lib/exel/processors/split_processor.rb +2 -1
  30. data/lib/exel/providers/local_file_provider.rb +2 -1
  31. data/lib/exel/providers/threaded_async_provider.rb +1 -0
  32. data/lib/exel/sequence_node.rb +1 -0
  33. data/lib/exel/value.rb +1 -0
  34. data/lib/exel/version.rb +2 -1
  35. data/spec/exel/ast_node_spec.rb +42 -42
  36. data/spec/exel/context_spec.rb +76 -77
  37. data/spec/exel/deferred_context_value_spec.rb +41 -42
  38. data/spec/exel/events_spec.rb +65 -65
  39. data/spec/exel/instruction_node_spec.rb +16 -16
  40. data/spec/exel/instruction_spec.rb +46 -45
  41. data/spec/exel/job_spec.rb +94 -91
  42. data/spec/exel/listen_instruction_spec.rb +10 -10
  43. data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
  44. data/spec/exel/logging_helper_spec.rb +15 -16
  45. data/spec/exel/logging_spec.rb +56 -56
  46. data/spec/exel/middleware/chain_spec.rb +51 -53
  47. data/spec/exel/middleware/logging_spec.rb +21 -23
  48. data/spec/exel/middleware_spec.rb +49 -50
  49. data/spec/exel/null_instruction_spec.rb +3 -4
  50. data/spec/exel/processors/async_processor_spec.rb +16 -18
  51. data/spec/exel/processors/run_processor_spec.rb +9 -11
  52. data/spec/exel/processors/split_processor_spec.rb +91 -93
  53. data/spec/exel/providers/local_file_provider_spec.rb +25 -28
  54. data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
  55. data/spec/exel/sequence_node_spec.rb +11 -11
  56. data/spec/exel/value_spec.rb +32 -33
  57. data/spec/exel_spec.rb +8 -7
  58. data/spec/integration/integration_spec.rb +2 -1
  59. data/spec/spec_helper.rb +3 -2
  60. data/spec/support/integration_test_classes.rb +3 -1
  61. metadata +16 -30
@@ -1,33 +1,31 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Middleware
4
- describe Logging do
5
- TestProcessor = Class.new
6
2
 
7
- describe '#call' do
8
- it 'yields to the given block' do
9
- called = false
3
+ describe EXEL::Middleware::Logging do
4
+ class TestProcessor
5
+ end
10
6
 
11
- subject.call(Object, {}, {}) do
12
- called = true
13
- end
7
+ describe '#call' do
8
+ it 'yields to the given block' do
9
+ called = false
14
10
 
15
- expect(called).to be_truthy
16
- end
11
+ subject.call(Object, {}, {}) do
12
+ called = true
13
+ end
17
14
 
18
- it 'logs with context[:log_prefix] and the processor class as the prefix' do
19
- expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][EXEL::Middleware::TestProcessor] ')
20
- subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
21
- end
15
+ expect(called).to be_truthy
16
+ end
17
+
18
+ it 'logs with context[:log_prefix] and the processor class as the prefix' do
19
+ expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][TestProcessor] ')
20
+ subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
21
+ end
22
22
 
23
- it 'raises rescued exceptions' do
24
- expect do
25
- subject.call(Object, {}, {}) do
26
- raise Exception, 're-raise me'
27
- end
28
- end.to raise_error Exception, 're-raise me'
23
+ it 'raises rescued exceptions' do
24
+ expect do
25
+ subject.call(Object, {}, {}) do
26
+ raise Exception, 're-raise me'
29
27
  end
30
- end
28
+ end.to raise_error Exception, 're-raise me'
31
29
  end
32
30
  end
33
31
  end
@@ -1,69 +1,68 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- describe Middleware do
4
- class IORecorder
5
- def initialize(input, output)
6
- @input = input
7
- @output = output
8
- end
9
2
 
10
- def call(_processor, context, args)
11
- @input << args[:input]
12
- yield
13
- @output << context[:output]
14
- end
3
+ describe EXEL::Middleware do
4
+ class IORecorder
5
+ def initialize(input, output)
6
+ @input = input
7
+ @output = output
15
8
  end
16
9
 
17
- class RescueErrors
18
- def call(_processor, _context, _args)
19
- yield
20
- rescue
21
- nil
22
- end
10
+ def call(_processor, context, args)
11
+ @input << args[:input]
12
+ yield
13
+ @output << context[:output]
23
14
  end
15
+ end
24
16
 
25
- class AddProcessor
26
- def initialize(context)
27
- @context = context
28
- end
29
-
30
- def process(_)
31
- @context[:output] = @context[:input] + 1
32
- raise 'rescue me'
33
- end
17
+ class RescueErrors
18
+ def call(_processor, _context, _args)
19
+ yield
20
+ rescue
21
+ nil
34
22
  end
23
+ end
35
24
 
36
- before :all do
37
- EXEL::Job.define :middleware_test_job do
38
- process with: AddProcessor, input: 1
39
- end
25
+ class AddProcessor
26
+ def initialize(context)
27
+ @context = context
40
28
  end
41
29
 
42
- let(:input) { [] }
43
- let(:output) { [] }
44
-
45
- before do
46
- EXEL.configure do |config|
47
- config.middleware.add(IORecorder, input, output)
48
- config.middleware.add(RescueErrors)
49
- end
30
+ def process(_)
31
+ @context[:output] = @context[:input] + 1
32
+ raise 'rescue me'
50
33
  end
34
+ end
51
35
 
52
- after do
53
- EXEL.configure do |config|
54
- config.middleware.remove(IORecorder)
55
- config.middleware.remove(RescueErrors)
56
- end
36
+ before :all do # rubocop:disable RSpec/BeforeAfterAll
37
+ EXEL::Job.define :middleware_test_job do
38
+ process with: AddProcessor, input: 1
57
39
  end
40
+ end
58
41
 
59
- it 'can configure custom middleware' do
60
- expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
42
+ let(:input) { [] }
43
+ let(:output) { [] }
44
+
45
+ before do
46
+ EXEL.configure do |config|
47
+ config.middleware.add(IORecorder, input, output)
48
+ config.middleware.add(RescueErrors)
61
49
  end
50
+ end
62
51
 
63
- it 'invokes the middleware around the processor' do
64
- EXEL::Job.run(:middleware_test_job)
65
- expect(input).to eq([1])
66
- expect(output).to eq([2])
52
+ after do
53
+ EXEL.configure do |config|
54
+ config.middleware.remove(IORecorder)
55
+ config.middleware.remove(RescueErrors)
67
56
  end
68
57
  end
58
+
59
+ it 'can configure custom middleware' do
60
+ expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
61
+ end
62
+
63
+ it 'invokes the middleware around the processor' do
64
+ EXEL::Job.run(:middleware_test_job)
65
+ expect(input).to eq([1])
66
+ expect(output).to eq([2])
67
+ end
69
68
  end
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- describe NullInstruction do
4
- it { is_expected.to respond_to :execute }
5
- end
2
+
3
+ describe EXEL::NullInstruction do
4
+ it { is_expected.to respond_to :execute }
6
5
  end
@@ -1,25 +1,23 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe AsyncProcessor do
5
- subject(:processor) { described_class.new(context) }
6
- let(:context) { EXEL::Context.new }
7
- let(:block) { instance_double(SequenceNode) }
8
2
 
9
- before do
10
- allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
11
- end
3
+ describe EXEL::Processors::AsyncProcessor do
4
+ subject(:processor) { described_class.new(context) }
12
5
 
13
- it 'looks up the async provider on initialization' do
14
- expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
15
- end
6
+ let(:context) { EXEL::Context.new }
7
+ let(:block) { instance_double(EXEL::SequenceNode) }
16
8
 
17
- describe '#process' do
18
- it 'calls do_async on the async provider' do
19
- expect(processor.provider).to receive(:do_async).with(block)
20
- processor.process(block)
21
- end
22
- end
9
+ before do
10
+ allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
11
+ end
12
+
13
+ it 'looks up the async provider on initialization' do
14
+ expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
15
+ end
16
+
17
+ describe '#process' do
18
+ it 'calls do_async on the async provider' do
19
+ expect(processor.provider).to receive(:do_async).with(block)
20
+ processor.process(block)
23
21
  end
24
22
  end
25
23
  end
@@ -1,16 +1,14 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe RunProcessor do
5
- subject { RunProcessor.new(context) }
6
- let(:context) { EXEL::Context.new(job: :test_job) }
7
2
 
8
- describe '#process' do
9
- it 'runs the job named in context[:job] with the current context' do
10
- expect(EXEL::Job).to receive(:run).with(:test_job, context)
11
- subject.process
12
- end
13
- end
3
+ describe EXEL::Processors::RunProcessor do
4
+ subject { EXEL::Processors::RunProcessor.new(context) }
5
+
6
+ let(:context) { EXEL::Context.new(job: :test_job) }
7
+
8
+ describe '#process' do
9
+ it 'runs the job named in context[:job] with the current context' do
10
+ expect(EXEL::Job).to receive(:run).with(:test_job, context)
11
+ subject.process
14
12
  end
15
13
  end
16
14
  end
@@ -1,123 +1,121 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe SplitProcessor do
5
- let(:chunk_file) { instance_double(File) }
6
- let(:file) { create_file(1) }
7
- let(:context) { Context.new(resource: file) }
8
- let(:callback) { instance_double(SequenceNode) }
9
- subject(:splitter) { SplitProcessor.new(context) }
10
-
11
- before do
12
- allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
13
- allow(File).to receive(:delete)
14
- end
15
2
 
16
- describe '#process' do
17
- let(:file) { create_file(3) }
3
+ describe EXEL::Processors::SplitProcessor do
4
+ subject(:splitter) { EXEL::Processors::SplitProcessor.new(context) }
18
5
 
19
- it 'processes file with 3 lines line by line' do
20
- allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
6
+ let(:chunk_file) { instance_double(File) }
7
+ let(:file) { create_file(1) }
8
+ let(:context) { EXEL::Context.new(resource: file) }
9
+ let(:callback) { instance_double(EXEL::SequenceNode) }
21
10
 
22
- 3.times do |i|
23
- expect(splitter).to receive(:process_line).with("line#{i}", callback)
24
- end
25
- expect(splitter).to receive(:process_line).with(:eof, callback)
11
+ before do
12
+ allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
13
+ allow(File).to receive(:delete)
14
+ end
26
15
 
27
- expect(File).to receive(:delete).with(file.path)
28
- expect(file).to receive(:close)
16
+ describe '#process' do
17
+ let(:file) { create_file(3) }
29
18
 
30
- splitter.process(callback)
31
- end
19
+ it 'processes file with 3 lines line by line' do
20
+ allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
32
21
 
33
- it 'aborts parsing the csv file if it is malformed' do
34
- allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
35
- expect(splitter).to receive(:process_line).with(:eof, callback)
22
+ 3.times do |i|
23
+ expect(splitter).to receive(:process_line).with("line#{i}", callback)
24
+ end
25
+ expect(splitter).to receive(:process_line).with(:eof, callback)
36
26
 
37
- splitter.process(callback)
38
- end
27
+ expect(File).to receive(:delete).with(file.path)
28
+ expect(file).to receive(:close)
39
29
 
40
- it 'does not delete the resource file if :delete_resource is set to false in the context' do
41
- allow(CSV).to receive(:foreach).and_yield(:eof)
42
- expect(File).not_to receive(:delete).with(file.path)
30
+ splitter.process(callback)
31
+ end
43
32
 
44
- context[:delete_resource] = false
45
- splitter.process(callback)
46
- end
33
+ it 'aborts parsing the csv file if it is malformed' do
34
+ allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError.new('message', '1'))
35
+ expect(splitter).to receive(:process_line).with(:eof, callback)
47
36
 
48
- it 'stops splitting at :max_chunks if it is set in the context' do
49
- allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
37
+ splitter.process(callback)
38
+ end
50
39
 
51
- chunk_file = create_file(0)
40
+ it 'does not delete the resource file if :delete_resource is set to false in the context' do
41
+ allow(CSV).to receive(:foreach).and_yield(:eof)
42
+ expect(File).not_to receive(:delete).with(file.path)
52
43
 
53
- allow(Tempfile).to receive(:new).and_return(chunk_file)
54
- expect(callback).to receive(:run).once
55
- allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
44
+ context[:delete_resource] = false
45
+ splitter.process(callback)
46
+ end
56
47
 
57
- expect(File).to receive(:delete).with(file.path)
48
+ it 'stops splitting at :max_chunks if it is set in the context' do
49
+ allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
58
50
 
59
- context[:chunk_size] = 2
60
- context[:max_chunks] = 1
61
- splitter.process(callback)
51
+ chunk_file = create_file(0)
62
52
 
63
- expect(chunk_file.read).to eq("line0\nline1\n")
64
- end
53
+ allow(Tempfile).to receive(:new).and_return(chunk_file)
54
+ expect(callback).to receive(:run).once
55
+ allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
65
56
 
66
- it 'ensures that the source file gets closed and deleted' do
67
- allow(CSV).to receive(:foreach).and_raise(Interrupt)
57
+ expect(File).to receive(:delete).with(file.path)
68
58
 
69
- expect(File).to receive(:delete).with(file.path)
70
- expect(file).to receive(:close)
59
+ context[:chunk_size] = 2
60
+ context[:max_chunks] = 1
61
+ splitter.process(callback)
71
62
 
72
- begin
73
- splitter.process(callback)
74
- rescue Interrupt
75
- nil
76
- end
77
- end
78
- end
63
+ expect(chunk_file.read).to eq("line0\nline1\n")
64
+ end
79
65
 
80
- describe '#process_line' do
81
- [
82
- {input: 1, chunks: %W(0\n)},
83
- {input: 3, chunks: %W(0\n1\n 2\n)},
84
- {input: 4, chunks: %W(0\n1\n 2\n3\n)}
85
- ].each do |data|
86
- it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
87
- context[:chunk_size] = 2
88
-
89
- data[:chunks].each do |chunk|
90
- expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
91
- expect(callback).to receive(:run).with(context) do
92
- expect(context[:resource]).to eq(chunk_file)
93
- end
94
- end
95
-
96
- data[:input].times { |i| splitter.process_line([i.to_s], callback) }
97
- splitter.process_line(:eof, callback)
98
- end
99
- end
100
- end
66
+ it 'ensures that the source file gets closed and deleted' do
67
+ allow(CSV).to receive(:foreach).and_raise(Interrupt)
101
68
 
102
- describe '#generate_chunk' do
103
- it 'creates a file with the contents of the given string' do
104
- file = splitter.generate_chunk('abc')
105
- content = file.read
106
- expect(content).to eq('abc')
107
- end
69
+ expect(File).to receive(:delete).with(file.path)
70
+ expect(file).to receive(:close)
71
+
72
+ begin
73
+ splitter.process(callback)
74
+ rescue Interrupt
75
+ nil
76
+ end
77
+ end
78
+ end
108
79
 
109
- it 'creates a file with a unique name' do
110
- 3.times do |i|
111
- file = splitter.generate_chunk('content')
112
- expect(file.path).to include("text_#{i + 1}_")
80
+ describe '#process_line' do
81
+ [
82
+ {input: 1, chunks: %W(0\n)},
83
+ {input: 3, chunks: %W(0\n1\n 2\n)},
84
+ {input: 4, chunks: %W(0\n1\n 2\n3\n)},
85
+ ].each do |data|
86
+ it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
87
+ context[:chunk_size] = 2
88
+
89
+ data[:chunks].each do |chunk|
90
+ expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
91
+ expect(callback).to receive(:run).with(context) do
92
+ expect(context[:resource]).to eq(chunk_file)
113
93
  end
114
94
  end
95
+
96
+ data[:input].times { |i| splitter.process_line([i.to_s], callback) }
97
+ splitter.process_line(:eof, callback)
115
98
  end
99
+ end
100
+ end
116
101
 
117
- def create_file(lines)
118
- content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
119
- StringIO.new(content)
102
+ describe '#generate_chunk' do
103
+ it 'creates a file with the contents of the given string' do
104
+ file = splitter.generate_chunk('abc')
105
+ content = file.read
106
+ expect(content).to eq('abc')
107
+ end
108
+
109
+ it 'creates a file with a unique name' do
110
+ 3.times do |i|
111
+ file = splitter.generate_chunk('content')
112
+ expect(file.path).to include("text_#{i + 1}_")
120
113
  end
121
114
  end
122
115
  end
116
+
117
+ def create_file(lines)
118
+ content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
119
+ StringIO.new(content)
120
+ end
123
121
  end