exel 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/.rubocop.yml +1 -0
 - data/.rubocop_todo.yml +10 -16
 - data/Gemfile +2 -0
 - data/README.md +5 -2
 - data/exel.gemspec +2 -1
 - data/lib/exel/ast_node.rb +2 -1
 - data/lib/exel/context.rb +45 -42
 - data/lib/exel/deferred_context_value.rb +35 -0
 - data/lib/exel/error/job_termination.rb +3 -5
 - data/lib/exel/events.rb +26 -0
 - data/lib/exel/instruction.rb +2 -4
 - data/lib/exel/instruction_node.rb +1 -0
 - data/lib/exel/job.rb +32 -10
 - data/lib/exel/listen_instruction.rb +17 -0
 - data/lib/exel/null_instruction.rb +1 -0
 - data/lib/exel/old_context.rb +109 -0
 - data/lib/exel/processor_helper.rb +1 -4
 - data/lib/exel/processors/async_processor.rb +1 -0
 - data/lib/exel/processors/run_processor.rb +3 -0
 - data/lib/exel/processors/split_processor.rb +12 -2
 - data/lib/exel/providers/local_file_provider.rb +3 -1
 - data/lib/exel/providers/threaded_async_provider.rb +1 -0
 - data/lib/exel/sequence_node.rb +1 -0
 - data/lib/exel/value.rb +1 -0
 - data/lib/exel/version.rb +1 -1
 - data/lib/exel.rb +20 -1
 - data/spec/exel/ast_node_spec.rb +4 -4
 - data/spec/exel/context_spec.rb +35 -109
 - data/spec/exel/deferred_context_value_spec.rb +51 -7
 - data/spec/exel/events_spec.rb +89 -0
 - data/spec/exel/instruction_node_spec.rb +3 -3
 - data/spec/exel/instruction_spec.rb +9 -9
 - data/spec/exel/job_spec.rb +23 -13
 - data/spec/exel/listen_instruction_spec.rb +14 -0
 - data/spec/exel/logging_spec.rb +3 -3
 - data/spec/exel/processors/split_processor_spec.rb +14 -6
 - data/spec/exel/sequence_node_spec.rb +1 -1
 - data/spec/exel_spec.rb +7 -0
 - data/spec/fixtures/sample.csv +501 -0
 - data/spec/integration/integration_spec.rb +51 -0
 - data/spec/spec_helper.rb +17 -1
 - data/spec/support/integration_test_classes.rb +44 -0
 - metadata +32 -5
 
    
        data/spec/exel/job_spec.rb
    CHANGED
    
    | 
         @@ -128,15 +128,13 @@ module EXEL 
     | 
|
| 
       128 
128 
     | 
    
         | 
| 
       129 
129 
     | 
    
         
             
                describe '#process' do
         
     | 
| 
       130 
130 
     | 
    
         
             
                  let(:block) { proc {} }
         
     | 
| 
      
 131 
     | 
    
         
            +
                  let(:processor_class) { class_double(Class) }
         
     | 
| 
       131 
132 
     | 
    
         | 
| 
       132 
     | 
    
         
            -
                  before  
     | 
| 
       133 
     | 
    
         
            -
                    allow(Job::Parser).to receive(:parse).and_return(ast)
         
     | 
| 
       134 
     | 
    
         
            -
                  end
         
     | 
| 
      
 133 
     | 
    
         
            +
                  before { allow(Job::Parser).to receive(:parse).and_return(ast) }
         
     | 
| 
       135 
134 
     | 
    
         | 
| 
       136 
135 
     | 
    
         
             
                  context 'without a block' do
         
     | 
| 
       137 
136 
     | 
    
         
             
                    it 'creates a process instruction' do
         
     | 
| 
       138 
     | 
    
         
            -
                       
     | 
| 
       139 
     | 
    
         
            -
                      expect(Instruction).to receive(:new).with('process', processor_class, {arg1: 'arg1_value'}, nil)
         
     | 
| 
      
 137 
     | 
    
         
            +
                      expect(Instruction).to receive(:new).with(processor_class, {arg1: 'arg1_value'}, nil)
         
     | 
| 
       140 
138 
     | 
    
         | 
| 
       141 
139 
     | 
    
         
             
                      parser.process with: processor_class, arg1: 'arg1_value'
         
     | 
| 
       142 
140 
     | 
    
         
             
                    end
         
     | 
| 
         @@ -144,19 +142,17 @@ module EXEL 
     | 
|
| 
       144 
142 
     | 
    
         
             
                    it 'appends an instruction node to the AST with no children' do
         
     | 
| 
       145 
143 
     | 
    
         
             
                      expect(parser.ast).to receive(:add_child) do |node|
         
     | 
| 
       146 
144 
     | 
    
         
             
                        expect(node).to be_a_kind_of(InstructionNode)
         
     | 
| 
       147 
     | 
    
         
            -
                        expect(node.instruction.name).to eq('process')
         
     | 
| 
       148 
145 
     | 
    
         
             
                        expect(node.children).to eq([])
         
     | 
| 
       149 
146 
     | 
    
         
             
                      end
         
     | 
| 
       150 
147 
     | 
    
         | 
| 
       151 
     | 
    
         
            -
                      parser.process with:  
     | 
| 
      
 148 
     | 
    
         
            +
                      parser.process with: processor_class
         
     | 
| 
       152 
149 
     | 
    
         
             
                    end
         
     | 
| 
       153 
150 
     | 
    
         
             
                  end
         
     | 
| 
       154 
151 
     | 
    
         | 
| 
       155 
152 
     | 
    
         
             
                  context 'with a block' do
         
     | 
| 
       156 
153 
     | 
    
         
             
                    it 'passes the parsed subtree to the instruction' do
         
     | 
| 
       157 
     | 
    
         
            -
                      processor_class = double(:processor_class)
         
     | 
| 
       158 
154 
     | 
    
         
             
                      expect(Job::Parser).to receive(:parse).with(block).and_return(ast)
         
     | 
| 
       159 
     | 
    
         
            -
                      expect(Instruction).to receive(:new).with( 
     | 
| 
      
 155 
     | 
    
         
            +
                      expect(Instruction).to receive(:new).with(processor_class, {arg1: 'arg1_value'}, ast)
         
     | 
| 
       160 
156 
     | 
    
         | 
| 
       161 
157 
     | 
    
         
             
                      parser.process with: processor_class, arg1: 'arg1_value', &block
         
     | 
| 
       162 
158 
     | 
    
         
             
                    end
         
     | 
| 
         @@ -164,11 +160,10 @@ module EXEL 
     | 
|
| 
       164 
160 
     | 
    
         
             
                    it 'appends an instruction node to the AST with the parsed block as its subtree' do
         
     | 
| 
       165 
161 
     | 
    
         
             
                      expect(parser.ast).to receive(:add_child) do |node|
         
     | 
| 
       166 
162 
     | 
    
         
             
                        expect(node).to be_a_kind_of(InstructionNode)
         
     | 
| 
       167 
     | 
    
         
            -
                        expect(node.instruction.name).to eq('process')
         
     | 
| 
       168 
163 
     | 
    
         
             
                        expect(node.children).to eq([ast])
         
     | 
| 
       169 
164 
     | 
    
         
             
                      end
         
     | 
| 
       170 
165 
     | 
    
         | 
| 
       171 
     | 
    
         
            -
                      parser.process with:  
     | 
| 
      
 166 
     | 
    
         
            +
                      parser.process with: processor_class, &block
         
     | 
| 
       172 
167 
     | 
    
         
             
                    end
         
     | 
| 
       173 
168 
     | 
    
         
             
                  end
         
     | 
| 
       174 
169 
     | 
    
         
             
                end
         
     | 
| 
         @@ -184,7 +179,7 @@ module EXEL 
     | 
|
| 
       184 
179 
     | 
    
         
             
                    end
         
     | 
| 
       185 
180 
     | 
    
         | 
| 
       186 
181 
     | 
    
         
             
                    it "creates a #{data[:method]} instruction" do
         
     | 
| 
       187 
     | 
    
         
            -
                      expect(Instruction).to receive(:new).with(data[: 
     | 
| 
      
 182 
     | 
    
         
            +
                      expect(Instruction).to receive(:new).with(data[:processor], {arg1: 'arg1_value'}, ast)
         
     | 
| 
       188 
183 
     | 
    
         
             
                      parser.send(data[:method], arg1: 'arg1_value') {}
         
     | 
| 
       189 
184 
     | 
    
         
             
                    end
         
     | 
| 
       190 
185 
     | 
    
         | 
| 
         @@ -198,7 +193,6 @@ module EXEL 
     | 
|
| 
       198 
193 
     | 
    
         
             
                    it 'adds parsed subtree and instruction to the AST' do
         
     | 
| 
       199 
194 
     | 
    
         
             
                      expect(parser.ast).to receive(:add_child) do |node|
         
     | 
| 
       200 
195 
     | 
    
         
             
                        expect(node).to be_a_kind_of(InstructionNode)
         
     | 
| 
       201 
     | 
    
         
            -
                        expect(node.instruction.name).to eq(data[:method].to_s)
         
     | 
| 
       202 
196 
     | 
    
         
             
                        expect(node.children).to eq([ast])
         
     | 
| 
       203 
197 
     | 
    
         
             
                      end
         
     | 
| 
       204 
198 
     | 
    
         | 
| 
         @@ -212,5 +206,21 @@ module EXEL 
     | 
|
| 
       212 
206 
     | 
    
         
             
                    expect(parser.context).to be_a_kind_of(DeferredContextValue)
         
     | 
| 
       213 
207 
     | 
    
         
             
                  end
         
     | 
| 
       214 
208 
     | 
    
         
             
                end
         
     | 
| 
      
 209 
     | 
    
         
            +
             
     | 
| 
      
 210 
     | 
    
         
            +
                describe '#listen' do
         
     | 
| 
      
 211 
     | 
    
         
            +
                  let(:listener_class) { class_double(Class) }
         
     | 
| 
      
 212 
     | 
    
         
            +
             
     | 
| 
      
 213 
     | 
    
         
            +
                  it 'creates a listen instruction' do
         
     | 
| 
      
 214 
     | 
    
         
            +
                    expect(ListenInstruction).to receive(:new).with(:event, listener_class)
         
     | 
| 
      
 215 
     | 
    
         
            +
                    parser.listen for: :event, with: listener_class
         
     | 
| 
      
 216 
     | 
    
         
            +
                  end
         
     | 
| 
      
 217 
     | 
    
         
            +
             
     | 
| 
      
 218 
     | 
    
         
            +
                  it 'adds an InstructionNode containing the listen instruction' do
         
     | 
| 
      
 219 
     | 
    
         
            +
                    parser.listen for: :event, with: listener_class
         
     | 
| 
      
 220 
     | 
    
         
            +
                    node = parser.ast.children.first
         
     | 
| 
      
 221 
     | 
    
         
            +
                    expect(node).to be_a_kind_of(InstructionNode)
         
     | 
| 
      
 222 
     | 
    
         
            +
                    expect(node.instruction).to be_a_kind_of(ListenInstruction)
         
     | 
| 
      
 223 
     | 
    
         
            +
                  end
         
     | 
| 
      
 224 
     | 
    
         
            +
                end
         
     | 
| 
       215 
225 
     | 
    
         
             
              end
         
     | 
| 
       216 
226 
     | 
    
         
             
            end
         
     | 
| 
         @@ -0,0 +1,14 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module EXEL
         
     | 
| 
      
 2 
     | 
    
         
            +
              describe ListenInstruction do
         
     | 
| 
      
 3 
     | 
    
         
            +
                subject(:instruction) { EXEL::ListenInstruction.new(:event, listener) }
         
     | 
| 
      
 4 
     | 
    
         
            +
                let(:listener) { double(:listener) }
         
     | 
| 
      
 5 
     | 
    
         
            +
                let(:context) { EXEL::Context.new }
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                describe '#execute' do
         
     | 
| 
      
 8 
     | 
    
         
            +
                  it 'registers the event listener' do
         
     | 
| 
      
 9 
     | 
    
         
            +
                    expect(instruction).to receive(:register_listener).with(context, :event, listener)
         
     | 
| 
      
 10 
     | 
    
         
            +
                    instruction.execute(context)
         
     | 
| 
      
 11 
     | 
    
         
            +
                  end
         
     | 
| 
      
 12 
     | 
    
         
            +
                end
         
     | 
| 
      
 13 
     | 
    
         
            +
              end
         
     | 
| 
      
 14 
     | 
    
         
            +
            end
         
     | 
    
        data/spec/exel/logging_spec.rb
    CHANGED
    
    | 
         @@ -4,13 +4,13 @@ module EXEL 
     | 
|
| 
       4 
4 
     | 
    
         
             
                after { Logging.logger = @restore_logger }
         
     | 
| 
       5 
5 
     | 
    
         | 
| 
       6 
6 
     | 
    
         
             
                describe '.logger=' do
         
     | 
| 
       7 
     | 
    
         
            -
                  it ' 
     | 
| 
      
 7 
     | 
    
         
            +
                  it 'sets a logger' do
         
     | 
| 
       8 
8 
     | 
    
         
             
                    logger = double(:logger)
         
     | 
| 
       9 
9 
     | 
    
         
             
                    Logging.logger = logger
         
     | 
| 
       10 
10 
     | 
    
         
             
                    expect(Logging.logger).to be(logger)
         
     | 
| 
       11 
11 
     | 
    
         
             
                  end
         
     | 
| 
       12 
12 
     | 
    
         | 
| 
       13 
     | 
    
         
            -
                  it ' 
     | 
| 
      
 13 
     | 
    
         
            +
                  it 'sets a null logger when nil given' do
         
     | 
| 
       14 
14 
     | 
    
         
             
                    expect(Logger).to receive(:new).with('/dev/null')
         
     | 
| 
       15 
15 
     | 
    
         
             
                    Logging.logger = nil
         
     | 
| 
       16 
16 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -19,7 +19,7 @@ module EXEL 
     | 
|
| 
       19 
19 
     | 
    
         
             
                describe '.logger' do
         
     | 
| 
       20 
20 
     | 
    
         
             
                  before { Logging.instance_variable_set(:@logger, nil) }
         
     | 
| 
       21 
21 
     | 
    
         | 
| 
       22 
     | 
    
         
            -
                  it ' 
     | 
| 
      
 22 
     | 
    
         
            +
                  it 'initializes the logger on first read if not already set' do
         
     | 
| 
       23 
23 
     | 
    
         
             
                    EXEL.configure do |config|
         
     | 
| 
       24 
24 
     | 
    
         
             
                      config.log_level = :warn
         
     | 
| 
       25 
25 
     | 
    
         
             
                      config.log_filename = 'log.txt'
         
     | 
| 
         @@ -15,7 +15,7 @@ module EXEL 
     | 
|
| 
       15 
15 
     | 
    
         
             
                  describe '#process' do
         
     | 
| 
       16 
16 
     | 
    
         
             
                    let(:file) { create_file(3) }
         
     | 
| 
       17 
17 
     | 
    
         | 
| 
       18 
     | 
    
         
            -
                    it ' 
     | 
| 
      
 18 
     | 
    
         
            +
                    it 'processes file with 3 lines line by line' do
         
     | 
| 
       19 
19 
     | 
    
         
             
                      allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
         
     | 
| 
       20 
20 
     | 
    
         | 
| 
       21 
21 
     | 
    
         
             
                      3.times do |i|
         
     | 
| 
         @@ -28,12 +28,20 @@ module EXEL 
     | 
|
| 
       28 
28 
     | 
    
         
             
                      splitter.process(callback)
         
     | 
| 
       29 
29 
     | 
    
         
             
                    end
         
     | 
| 
       30 
30 
     | 
    
         | 
| 
       31 
     | 
    
         
            -
                    it ' 
     | 
| 
      
 31 
     | 
    
         
            +
                    it 'aborts parsing the csv file if it is malformed' do
         
     | 
| 
       32 
32 
     | 
    
         
             
                      allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
         
     | 
| 
       33 
33 
     | 
    
         
             
                      expect(splitter).to receive(:process_line).with(:eof, callback)
         
     | 
| 
       34 
34 
     | 
    
         | 
| 
       35 
35 
     | 
    
         
             
                      splitter.process(callback)
         
     | 
| 
       36 
36 
     | 
    
         
             
                    end
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
                    it 'does not delete the resource file if :delete_resource is set to false in the context' do
         
     | 
| 
      
 39 
     | 
    
         
            +
                      allow(CSV).to receive(:foreach).and_yield(:eof)
         
     | 
| 
      
 40 
     | 
    
         
            +
                      expect(File).not_to receive(:delete).with(file.path)
         
     | 
| 
      
 41 
     | 
    
         
            +
             
     | 
| 
      
 42 
     | 
    
         
            +
                      context[:delete_resource] = false
         
     | 
| 
      
 43 
     | 
    
         
            +
                      splitter.process(callback)
         
     | 
| 
      
 44 
     | 
    
         
            +
                    end
         
     | 
| 
       37 
45 
     | 
    
         
             
                  end
         
     | 
| 
       38 
46 
     | 
    
         | 
| 
       39 
47 
     | 
    
         
             
                  describe '#process_line' do
         
     | 
| 
         @@ -42,8 +50,8 @@ module EXEL 
     | 
|
| 
       42 
50 
     | 
    
         
             
                      {input: 3, chunks: %W(0\n1\n 2\n)},
         
     | 
| 
       43 
51 
     | 
    
         
             
                      {input: 4, chunks: %W(0\n1\n 2\n3\n)}
         
     | 
| 
       44 
52 
     | 
    
         
             
                    ].each do |data|
         
     | 
| 
       45 
     | 
    
         
            -
                      it " 
     | 
| 
       46 
     | 
    
         
            -
                         
     | 
| 
      
 53 
     | 
    
         
            +
                      it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
         
     | 
| 
      
 54 
     | 
    
         
            +
                        context[:chunk_size] = 2
         
     | 
| 
       47 
55 
     | 
    
         | 
| 
       48 
56 
     | 
    
         
             
                        data[:chunks].each do |chunk|
         
     | 
| 
       49 
57 
     | 
    
         
             
                          expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
         
     | 
| 
         @@ -59,13 +67,13 @@ module EXEL 
     | 
|
| 
       59 
67 
     | 
    
         
             
                  end
         
     | 
| 
       60 
68 
     | 
    
         | 
| 
       61 
69 
     | 
    
         
             
                  describe '#generate_chunk' do
         
     | 
| 
       62 
     | 
    
         
            -
                    it ' 
     | 
| 
      
 70 
     | 
    
         
            +
                    it 'creates a file with the contents of the given string' do
         
     | 
| 
       63 
71 
     | 
    
         
             
                      file = splitter.generate_chunk('abc')
         
     | 
| 
       64 
72 
     | 
    
         
             
                      content = file.read
         
     | 
| 
       65 
73 
     | 
    
         
             
                      expect(content).to eq('abc')
         
     | 
| 
       66 
74 
     | 
    
         
             
                    end
         
     | 
| 
       67 
75 
     | 
    
         | 
| 
       68 
     | 
    
         
            -
                    it ' 
     | 
| 
      
 76 
     | 
    
         
            +
                    it 'creates a file with a unique name' do
         
     | 
| 
       69 
77 
     | 
    
         
             
                      3.times do |i|
         
     | 
| 
       70 
78 
     | 
    
         
             
                        file = splitter.generate_chunk('content')
         
     | 
| 
       71 
79 
     | 
    
         
             
                        expect(file.path).to include("text_#{i + 1}_")
         
     | 
| 
         @@ -6,7 +6,7 @@ module EXEL 
     | 
|
| 
       6 
6 
     | 
    
         
             
                it { is_expected.to be_an(ASTNode) }
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
8 
     | 
    
         
             
                describe '#run' do
         
     | 
| 
       9 
     | 
    
         
            -
                  it ' 
     | 
| 
      
 9 
     | 
    
         
            +
                  it 'runs each child node in sequence' do
         
     | 
| 
       10 
10 
     | 
    
         
             
                    expect(node.children.first).to receive(:run).with(context).once.ordered
         
     | 
| 
       11 
11 
     | 
    
         
             
                    expect(node.children.last).to receive(:run).with(context).once.ordered
         
     | 
| 
       12 
12 
     | 
    
         |