exel 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +5 -5
  2. data/.codeclimate.yml +4 -4
  3. data/.rubocop.yml +20 -12
  4. data/.rubocop_airbnb.yml +2 -0
  5. data/.travis.yml +17 -4
  6. data/Gemfile +1 -2
  7. data/Gemfile.lock +64 -60
  8. data/README.md +1 -1
  9. data/Rakefile +1 -0
  10. data/exel.gemspec +4 -4
  11. data/lib/exel.rb +1 -0
  12. data/lib/exel/ast_node.rb +2 -1
  13. data/lib/exel/context.rb +2 -1
  14. data/lib/exel/deferred_context_value.rb +1 -0
  15. data/lib/exel/error/job_termination.rb +1 -0
  16. data/lib/exel/events.rb +1 -0
  17. data/lib/exel/instruction.rb +2 -1
  18. data/lib/exel/instruction_node.rb +1 -0
  19. data/lib/exel/job.rb +6 -4
  20. data/lib/exel/listen_instruction.rb +1 -0
  21. data/lib/exel/logging.rb +1 -0
  22. data/lib/exel/logging/logger_wrapper.rb +4 -1
  23. data/lib/exel/logging_helper.rb +1 -0
  24. data/lib/exel/middleware/chain.rb +1 -0
  25. data/lib/exel/middleware/logging.rb +1 -1
  26. data/lib/exel/null_instruction.rb +1 -0
  27. data/lib/exel/processor_helper.rb +1 -0
  28. data/lib/exel/processors/run_processor.rb +1 -0
  29. data/lib/exel/processors/split_processor.rb +2 -1
  30. data/lib/exel/providers/local_file_provider.rb +2 -1
  31. data/lib/exel/providers/threaded_async_provider.rb +1 -0
  32. data/lib/exel/sequence_node.rb +1 -0
  33. data/lib/exel/value.rb +1 -0
  34. data/lib/exel/version.rb +2 -1
  35. data/spec/exel/ast_node_spec.rb +42 -42
  36. data/spec/exel/context_spec.rb +76 -77
  37. data/spec/exel/deferred_context_value_spec.rb +41 -42
  38. data/spec/exel/events_spec.rb +65 -65
  39. data/spec/exel/instruction_node_spec.rb +16 -16
  40. data/spec/exel/instruction_spec.rb +46 -45
  41. data/spec/exel/job_spec.rb +94 -91
  42. data/spec/exel/listen_instruction_spec.rb +10 -10
  43. data/spec/exel/logging/logger_wrapper_spec.rb +67 -69
  44. data/spec/exel/logging_helper_spec.rb +15 -16
  45. data/spec/exel/logging_spec.rb +56 -56
  46. data/spec/exel/middleware/chain_spec.rb +51 -53
  47. data/spec/exel/middleware/logging_spec.rb +21 -23
  48. data/spec/exel/middleware_spec.rb +49 -50
  49. data/spec/exel/null_instruction_spec.rb +3 -4
  50. data/spec/exel/processors/async_processor_spec.rb +16 -18
  51. data/spec/exel/processors/run_processor_spec.rb +9 -11
  52. data/spec/exel/processors/split_processor_spec.rb +91 -93
  53. data/spec/exel/providers/local_file_provider_spec.rb +25 -28
  54. data/spec/exel/providers/threaded_async_provider_spec.rb +36 -38
  55. data/spec/exel/sequence_node_spec.rb +11 -11
  56. data/spec/exel/value_spec.rb +32 -33
  57. data/spec/exel_spec.rb +8 -7
  58. data/spec/integration/integration_spec.rb +2 -1
  59. data/spec/spec_helper.rb +3 -2
  60. data/spec/support/integration_test_classes.rb +3 -1
  61. metadata +16 -30
@@ -1,33 +1,31 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Middleware
4
- describe Logging do
5
- TestProcessor = Class.new
6
2
 
7
- describe '#call' do
8
- it 'yields to the given block' do
9
- called = false
3
+ describe EXEL::Middleware::Logging do
4
+ class TestProcessor
5
+ end
10
6
 
11
- subject.call(Object, {}, {}) do
12
- called = true
13
- end
7
+ describe '#call' do
8
+ it 'yields to the given block' do
9
+ called = false
14
10
 
15
- expect(called).to be_truthy
16
- end
11
+ subject.call(Object, {}, {}) do
12
+ called = true
13
+ end
17
14
 
18
- it 'logs with context[:log_prefix] and the processor class as the prefix' do
19
- expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][EXEL::Middleware::TestProcessor] ')
20
- subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
21
- end
15
+ expect(called).to be_truthy
16
+ end
17
+
18
+ it 'logs with context[:log_prefix] and the processor class as the prefix' do
19
+ expect(EXEL::Logging).to receive(:with_prefix).with('[prefix][TestProcessor] ')
20
+ subject.call(TestProcessor, {log_prefix: '[prefix]'}, {}) {}
21
+ end
22
22
 
23
- it 'raises rescued exceptions' do
24
- expect do
25
- subject.call(Object, {}, {}) do
26
- raise Exception, 're-raise me'
27
- end
28
- end.to raise_error Exception, 're-raise me'
23
+ it 'raises rescued exceptions' do
24
+ expect do
25
+ subject.call(Object, {}, {}) do
26
+ raise Exception, 're-raise me'
29
27
  end
30
- end
28
+ end.to raise_error Exception, 're-raise me'
31
29
  end
32
30
  end
33
31
  end
@@ -1,69 +1,68 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- describe Middleware do
4
- class IORecorder
5
- def initialize(input, output)
6
- @input = input
7
- @output = output
8
- end
9
2
 
10
- def call(_processor, context, args)
11
- @input << args[:input]
12
- yield
13
- @output << context[:output]
14
- end
3
+ describe EXEL::Middleware do
4
+ class IORecorder
5
+ def initialize(input, output)
6
+ @input = input
7
+ @output = output
15
8
  end
16
9
 
17
- class RescueErrors
18
- def call(_processor, _context, _args)
19
- yield
20
- rescue
21
- nil
22
- end
10
+ def call(_processor, context, args)
11
+ @input << args[:input]
12
+ yield
13
+ @output << context[:output]
23
14
  end
15
+ end
24
16
 
25
- class AddProcessor
26
- def initialize(context)
27
- @context = context
28
- end
29
-
30
- def process(_)
31
- @context[:output] = @context[:input] + 1
32
- raise 'rescue me'
33
- end
17
+ class RescueErrors
18
+ def call(_processor, _context, _args)
19
+ yield
20
+ rescue
21
+ nil
34
22
  end
23
+ end
35
24
 
36
- before :all do
37
- EXEL::Job.define :middleware_test_job do
38
- process with: AddProcessor, input: 1
39
- end
25
+ class AddProcessor
26
+ def initialize(context)
27
+ @context = context
40
28
  end
41
29
 
42
- let(:input) { [] }
43
- let(:output) { [] }
44
-
45
- before do
46
- EXEL.configure do |config|
47
- config.middleware.add(IORecorder, input, output)
48
- config.middleware.add(RescueErrors)
49
- end
30
+ def process(_)
31
+ @context[:output] = @context[:input] + 1
32
+ raise 'rescue me'
50
33
  end
34
+ end
51
35
 
52
- after do
53
- EXEL.configure do |config|
54
- config.middleware.remove(IORecorder)
55
- config.middleware.remove(RescueErrors)
56
- end
36
+ before :all do # rubocop:disable RSpec/BeforeAfterAll
37
+ EXEL::Job.define :middleware_test_job do
38
+ process with: AddProcessor, input: 1
57
39
  end
40
+ end
58
41
 
59
- it 'can configure custom middleware' do
60
- expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
42
+ let(:input) { [] }
43
+ let(:output) { [] }
44
+
45
+ before do
46
+ EXEL.configure do |config|
47
+ config.middleware.add(IORecorder, input, output)
48
+ config.middleware.add(RescueErrors)
61
49
  end
50
+ end
62
51
 
63
- it 'invokes the middleware around the processor' do
64
- EXEL::Job.run(:middleware_test_job)
65
- expect(input).to eq([1])
66
- expect(output).to eq([2])
52
+ after do
53
+ EXEL.configure do |config|
54
+ config.middleware.remove(IORecorder)
55
+ config.middleware.remove(RescueErrors)
67
56
  end
68
57
  end
58
+
59
+ it 'can configure custom middleware' do
60
+ expect(EXEL.configuration.middleware.entries.map(&:klass)).to eq([IORecorder, RescueErrors])
61
+ end
62
+
63
+ it 'invokes the middleware around the processor' do
64
+ EXEL::Job.run(:middleware_test_job)
65
+ expect(input).to eq([1])
66
+ expect(output).to eq([2])
67
+ end
69
68
  end
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- describe NullInstruction do
4
- it { is_expected.to respond_to :execute }
5
- end
2
+
3
+ describe EXEL::NullInstruction do
4
+ it { is_expected.to respond_to :execute }
6
5
  end
@@ -1,25 +1,23 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe AsyncProcessor do
5
- subject(:processor) { described_class.new(context) }
6
- let(:context) { EXEL::Context.new }
7
- let(:block) { instance_double(SequenceNode) }
8
2
 
9
- before do
10
- allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
11
- end
3
+ describe EXEL::Processors::AsyncProcessor do
4
+ subject(:processor) { described_class.new(context) }
12
5
 
13
- it 'looks up the async provider on initialization' do
14
- expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
15
- end
6
+ let(:context) { EXEL::Context.new }
7
+ let(:block) { instance_double(EXEL::SequenceNode) }
16
8
 
17
- describe '#process' do
18
- it 'calls do_async on the async provider' do
19
- expect(processor.provider).to receive(:do_async).with(block)
20
- processor.process(block)
21
- end
22
- end
9
+ before do
10
+ allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
11
+ end
12
+
13
+ it 'looks up the async provider on initialization' do
14
+ expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
15
+ end
16
+
17
+ describe '#process' do
18
+ it 'calls do_async on the async provider' do
19
+ expect(processor.provider).to receive(:do_async).with(block)
20
+ processor.process(block)
23
21
  end
24
22
  end
25
23
  end
@@ -1,16 +1,14 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe RunProcessor do
5
- subject { RunProcessor.new(context) }
6
- let(:context) { EXEL::Context.new(job: :test_job) }
7
2
 
8
- describe '#process' do
9
- it 'runs the job named in context[:job] with the current context' do
10
- expect(EXEL::Job).to receive(:run).with(:test_job, context)
11
- subject.process
12
- end
13
- end
3
+ describe EXEL::Processors::RunProcessor do
4
+ subject { EXEL::Processors::RunProcessor.new(context) }
5
+
6
+ let(:context) { EXEL::Context.new(job: :test_job) }
7
+
8
+ describe '#process' do
9
+ it 'runs the job named in context[:job] with the current context' do
10
+ expect(EXEL::Job).to receive(:run).with(:test_job, context)
11
+ subject.process
14
12
  end
15
13
  end
16
14
  end
@@ -1,123 +1,121 @@
1
1
  # frozen_string_literal: true
2
- module EXEL
3
- module Processors
4
- describe SplitProcessor do
5
- let(:chunk_file) { instance_double(File) }
6
- let(:file) { create_file(1) }
7
- let(:context) { Context.new(resource: file) }
8
- let(:callback) { instance_double(SequenceNode) }
9
- subject(:splitter) { SplitProcessor.new(context) }
10
-
11
- before do
12
- allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
13
- allow(File).to receive(:delete)
14
- end
15
2
 
16
- describe '#process' do
17
- let(:file) { create_file(3) }
3
+ describe EXEL::Processors::SplitProcessor do
4
+ subject(:splitter) { EXEL::Processors::SplitProcessor.new(context) }
18
5
 
19
- it 'processes file with 3 lines line by line' do
20
- allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
6
+ let(:chunk_file) { instance_double(File) }
7
+ let(:file) { create_file(1) }
8
+ let(:context) { EXEL::Context.new(resource: file) }
9
+ let(:callback) { instance_double(EXEL::SequenceNode) }
21
10
 
22
- 3.times do |i|
23
- expect(splitter).to receive(:process_line).with("line#{i}", callback)
24
- end
25
- expect(splitter).to receive(:process_line).with(:eof, callback)
11
+ before do
12
+ allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
13
+ allow(File).to receive(:delete)
14
+ end
26
15
 
27
- expect(File).to receive(:delete).with(file.path)
28
- expect(file).to receive(:close)
16
+ describe '#process' do
17
+ let(:file) { create_file(3) }
29
18
 
30
- splitter.process(callback)
31
- end
19
+ it 'processes file with 3 lines line by line' do
20
+ allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
32
21
 
33
- it 'aborts parsing the csv file if it is malformed' do
34
- allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
35
- expect(splitter).to receive(:process_line).with(:eof, callback)
22
+ 3.times do |i|
23
+ expect(splitter).to receive(:process_line).with("line#{i}", callback)
24
+ end
25
+ expect(splitter).to receive(:process_line).with(:eof, callback)
36
26
 
37
- splitter.process(callback)
38
- end
27
+ expect(File).to receive(:delete).with(file.path)
28
+ expect(file).to receive(:close)
39
29
 
40
- it 'does not delete the resource file if :delete_resource is set to false in the context' do
41
- allow(CSV).to receive(:foreach).and_yield(:eof)
42
- expect(File).not_to receive(:delete).with(file.path)
30
+ splitter.process(callback)
31
+ end
43
32
 
44
- context[:delete_resource] = false
45
- splitter.process(callback)
46
- end
33
+ it 'aborts parsing the csv file if it is malformed' do
34
+ allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError.new('message', '1'))
35
+ expect(splitter).to receive(:process_line).with(:eof, callback)
47
36
 
48
- it 'stops splitting at :max_chunks if it is set in the context' do
49
- allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
37
+ splitter.process(callback)
38
+ end
50
39
 
51
- chunk_file = create_file(0)
40
+ it 'does not delete the resource file if :delete_resource is set to false in the context' do
41
+ allow(CSV).to receive(:foreach).and_yield(:eof)
42
+ expect(File).not_to receive(:delete).with(file.path)
52
43
 
53
- allow(Tempfile).to receive(:new).and_return(chunk_file)
54
- expect(callback).to receive(:run).once
55
- allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
44
+ context[:delete_resource] = false
45
+ splitter.process(callback)
46
+ end
56
47
 
57
- expect(File).to receive(:delete).with(file.path)
48
+ it 'stops splitting at :max_chunks if it is set in the context' do
49
+ allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
58
50
 
59
- context[:chunk_size] = 2
60
- context[:max_chunks] = 1
61
- splitter.process(callback)
51
+ chunk_file = create_file(0)
62
52
 
63
- expect(chunk_file.read).to eq("line0\nline1\n")
64
- end
53
+ allow(Tempfile).to receive(:new).and_return(chunk_file)
54
+ expect(callback).to receive(:run).once
55
+ allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
65
56
 
66
- it 'ensures that the source file gets closed and deleted' do
67
- allow(CSV).to receive(:foreach).and_raise(Interrupt)
57
+ expect(File).to receive(:delete).with(file.path)
68
58
 
69
- expect(File).to receive(:delete).with(file.path)
70
- expect(file).to receive(:close)
59
+ context[:chunk_size] = 2
60
+ context[:max_chunks] = 1
61
+ splitter.process(callback)
71
62
 
72
- begin
73
- splitter.process(callback)
74
- rescue Interrupt
75
- nil
76
- end
77
- end
78
- end
63
+ expect(chunk_file.read).to eq("line0\nline1\n")
64
+ end
79
65
 
80
- describe '#process_line' do
81
- [
82
- {input: 1, chunks: %W(0\n)},
83
- {input: 3, chunks: %W(0\n1\n 2\n)},
84
- {input: 4, chunks: %W(0\n1\n 2\n3\n)}
85
- ].each do |data|
86
- it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
87
- context[:chunk_size] = 2
88
-
89
- data[:chunks].each do |chunk|
90
- expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
91
- expect(callback).to receive(:run).with(context) do
92
- expect(context[:resource]).to eq(chunk_file)
93
- end
94
- end
95
-
96
- data[:input].times { |i| splitter.process_line([i.to_s], callback) }
97
- splitter.process_line(:eof, callback)
98
- end
99
- end
100
- end
66
+ it 'ensures that the source file gets closed and deleted' do
67
+ allow(CSV).to receive(:foreach).and_raise(Interrupt)
101
68
 
102
- describe '#generate_chunk' do
103
- it 'creates a file with the contents of the given string' do
104
- file = splitter.generate_chunk('abc')
105
- content = file.read
106
- expect(content).to eq('abc')
107
- end
69
+ expect(File).to receive(:delete).with(file.path)
70
+ expect(file).to receive(:close)
71
+
72
+ begin
73
+ splitter.process(callback)
74
+ rescue Interrupt
75
+ nil
76
+ end
77
+ end
78
+ end
108
79
 
109
- it 'creates a file with a unique name' do
110
- 3.times do |i|
111
- file = splitter.generate_chunk('content')
112
- expect(file.path).to include("text_#{i + 1}_")
80
+ describe '#process_line' do
81
+ [
82
+ {input: 1, chunks: %W(0\n)},
83
+ {input: 3, chunks: %W(0\n1\n 2\n)},
84
+ {input: 4, chunks: %W(0\n1\n 2\n3\n)},
85
+ ].each do |data|
86
+ it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
87
+ context[:chunk_size] = 2
88
+
89
+ data[:chunks].each do |chunk|
90
+ expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
91
+ expect(callback).to receive(:run).with(context) do
92
+ expect(context[:resource]).to eq(chunk_file)
113
93
  end
114
94
  end
95
+
96
+ data[:input].times { |i| splitter.process_line([i.to_s], callback) }
97
+ splitter.process_line(:eof, callback)
115
98
  end
99
+ end
100
+ end
116
101
 
117
- def create_file(lines)
118
- content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
119
- StringIO.new(content)
102
+ describe '#generate_chunk' do
103
+ it 'creates a file with the contents of the given string' do
104
+ file = splitter.generate_chunk('abc')
105
+ content = file.read
106
+ expect(content).to eq('abc')
107
+ end
108
+
109
+ it 'creates a file with a unique name' do
110
+ 3.times do |i|
111
+ file = splitter.generate_chunk('content')
112
+ expect(file.path).to include("text_#{i + 1}_")
120
113
  end
121
114
  end
122
115
  end
116
+
117
+ def create_file(lines)
118
+ content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
119
+ StringIO.new(content)
120
+ end
123
121
  end