exel 1.2.1 → 1.5.2

data/lib/exel/processors/async_processor.rb

@@ -1,24 +1,18 @@
-require_relative '../processor_helper'
+# frozen_string_literal: true
 
 module EXEL
   module Processors
     # Implements the +async+ instruction by using the configured async provider to run a block asynchronously.
     class AsyncProcessor
-      include EXEL::ProcessorHelper
       attr_reader :provider
 
       def initialize(context)
         @context = context
         @provider = EXEL.async_provider.new(context)
-
-        log_prefix_with '[AsyncProcessor]'
       end
 
       def process(block)
-        log_process do
-          @provider.do_async(block)
-          log_info 'call to async completed'
-        end
+        @provider.do_async(block)
       end
     end
   end

data/lib/exel/processors/run_processor.rb

@@ -1,11 +1,9 @@
-require_relative '../processor_helper'
+# frozen_string_literal: true
 
 module EXEL
   module Processors
     # Implements the +run+ instruction.
     class RunProcessor
-      include EXEL::ProcessorHelper
-
       # Requires +context[:job]+ to contain the name of the job to be run.
       def initialize(context)
         @context = context
@@ -13,9 +11,7 @@ module EXEL
 
       # Runs the specified job with the current context
       def process(_block = nil)
-        log_process "running job #{@context[:job]}" do
-          EXEL::Job.run(@context[:job], @context)
-        end
+        EXEL::Job.run(@context[:job], @context)
       end
     end
   end

data/lib/exel/processors/split_processor.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require 'csv'
 require 'tempfile'
-require_relative '../processor_helper'
+require_relative '../logging_helper'
 
 module EXEL
   module Processors
@@ -11,8 +13,10 @@ module EXEL
     # * +:delete_resource+ Defaults to true, can be set to false to preserve the original resource. Otherwise, it will
     #   be deleted when splitting is complete
     # * +:chunk_size+ Set to specify the number of lines that each chunk should contain
+    # * +:max_chunks+ Set to specify the maximum number of chunks that should be processed. The resource will not be
+    #   consumed beyond this limit.
     class SplitProcessor
-      include EXEL::ProcessorHelper
+      include EXEL::LoggingHelper
 
       attr_accessor :file_name, :block
 
@@ -25,16 +29,16 @@ module EXEL
         @tempfile_count = 0
         @context = context
         @file = context[:resource]
+        @max_chunks = @context[:max_chunks] || Float::INFINITY
         @context[:delete_resource] = true if @context[:delete_resource].nil?
-
-        log_prefix_with '[SplitProcessor]'
       end
 
       def process(callback)
-        log_process do
-          process_file(callback)
-          finish(callback)
-        end
+        process_file(callback)
+        finish(callback)
+      ensure
+        @file.close
+        File.delete(@file.path) if @context[:delete_resource]
       end
 
       def process_line(line, callback)
@@ -62,8 +66,10 @@ module EXEL
       def process_file(callback)
         csv_options = @context[:csv_options] || {col_sep: ','}
 
-        CSV.foreach(@file.path, csv_options) do |line|
+        CSV.foreach(@file.path, **csv_options) do |line|
           process_line(line, callback)
+
+          break if @tempfile_count == @max_chunks
         end
       rescue CSV::MalformedCSVError => e
         log_error "CSV::MalformedCSVError => #{e.message}"
@@ -97,7 +103,6 @@ module EXEL
 
       def finish(callback)
         process_line(:eof, callback)
-        File.delete(@file.path) if @context[:delete_resource]
       end
     end
   end

data/lib/exel/providers/local_file_provider.rb

@@ -1,19 +1,22 @@
+# frozen_string_literal: true
+
 module EXEL
   module Providers
     # The default remote provider. Doesn't actually upload and download files to and from remote storage, but rather
     # just works with local files.
     class LocalFileProvider
       def upload(file)
-        "file://#{file.path}"
+        RemoteValue.new(URI("file://#{File.absolute_path(file)}"))
       end
 
-      def download(uri)
-        raise 'URI must begin with "file://"' unless uri.start_with? 'file://'
-        File.open(uri.split('file://').last)
+      def download(remote_value)
+        scheme = remote_value.uri.scheme
+        raise "Unsupported URI scheme '#{scheme}'" unless scheme == 'file'
+        File.open(remote_value.uri.path)
       end
 
-      def self.remote?(uri)
-        uri =~ %r{file://}
+      def self.remote?(value)
+        value.is_a?(RemoteValue)
       end
     end
   end

data/lib/exel/providers/threaded_async_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module EXEL
   module Providers
     # The default remote provider. Provides async execution by running the given EXEL block in a new Thread

data/lib/exel/remote_value.rb

@@ -0,0 +1,11 @@
+class RemoteValue
+  attr_reader :uri
+
+  def initialize(uri)
+    @uri = uri
+  end
+
+  def ==(other)
+    other.class == self.class && other.uri == @uri
+  end
+end

data/lib/exel/sequence_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative './ast_node'
 
 module EXEL

data/lib/exel/value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module EXEL
   # Contains methods to handle remote and local values. Used for {Context} serialization
   module Value

data/lib/exel/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module EXEL
-  VERSION = '1.2.1'.freeze
+  VERSION = '1.5.2'
 end

data/spec/exel/ast_node_spec.rb

@@ -1,40 +1,61 @@
-module EXEL
-  describe ASTNode do
-    let(:context) { instance_double(EXEL::Context) }
+# frozen_string_literal: true
 
-    def instruction
-      instance_double(Instruction, execute: nil)
-    end
+describe EXEL::ASTNode do
+  let(:context) { instance_double(EXEL::Context) }
+
+  def instruction
+    instance_double(EXEL::Instruction, execute: nil)
+  end
+
+  class TestNode < EXEL::ASTNode
+  end
 
-    TestNode = Class.new(ASTNode)
+  describe '#start' do
+    context 'when a JobTermination error bubbles up' do
+      let(:node) { TestNode.new(instruction) }
+
+      before do
+        allow(node).to receive(:run).and_raise(EXEL::Error::JobTermination, 'Error')
+      end
+
+      it 'ensures the process fails silently' do
+        expect(EXEL.logger).to receive(:error).with('JobTerminationError: Error')
+        expect { node.start(context) }.not_to raise_error
+      end
 
-    describe '#start' do
-      context 'when an JobTermination error bubbles up' do
-        it 'ensures the process fails silently' do
-          node = TestNode.new(instruction)
-          allow(node).to receive(:run).and_raise(EXEL::Error::JobTermination, 'Error')
-          expect(EXEL.logger).to receive(:error).with('JobTerminationError: Error')
-          expect { node.start(context) }.not_to raise_error
+      it 'logs the error by default' do
+        expect(EXEL.logger).to receive(:error).with('JobTerminationError: Error')
+        node.start(context)
+      end
+
+      context 'given a log instruction' do
+        before do
+          allow(node).to receive(:run).and_raise(EXEL::Error::JobTermination.new('Error', :warn))
+        end
+
+        it 'logs the error with the given cmd' do
+          expect(EXEL.logger).to receive(:warn).with('JobTerminationError: Error')
+          node.start(context)
         end
       end
     end
+  end
 
-    describe '#run' do
-      it 'raises an error if not implemented' do
-        expect { TestNode.new(instruction).run(context) }.to raise_error 'EXEL::TestNode does not implement #process'
-      end
+  describe '#run' do
+    it 'raises an error if not implemented' do
+      expect { TestNode.new(instruction).run(context) }.to raise_error 'TestNode does not implement #process'
     end
+  end
 
-    describe '#add_child' do
-      it 'adds the given node to its children' do
-        root = ASTNode.new(instruction)
-        child_node = ASTNode.new(instruction)
-        child_node2 = ASTNode.new(instruction)
-        root.add_child(child_node)
-        root.add_child(child_node2)
+  describe '#add_child' do
+    it 'adds the given node to its children' do
+      root = EXEL::ASTNode.new(instruction)
+      child_node = EXEL::ASTNode.new(instruction)
+      child_node2 = EXEL::ASTNode.new(instruction)
+      root.add_child(child_node)
+      root.add_child(child_node2)
 
-        expect(root.children).to eq([child_node, child_node2])
-      end
+      expect(root.children).to eq([child_node, child_node2])
     end
   end
 end

data/spec/exel/context_spec.rb

@@ -1,108 +1,108 @@
-module EXEL
-  describe Context do
-    subject(:context) { EXEL::Context.new(key1: '1', key2: 2) }
+# frozen_string_literal: true
 
-    it { is_expected.to be_a(Hash) }
+describe EXEL::Context do
+  subject(:context) { EXEL::Context.new(key1: '1', key2: 2) }
 
-    describe '#initialize' do
-      it 'initializes with a hash' do
-        expect(context[:key1]).to eq('1')
-        expect(context[:key2]).to eq(2)
-        expect(context[:key3]).to be_nil
-      end
+  it { is_expected.to be_a(Hash) }
+
+  describe '#initialize' do
+    it 'initializes with a hash' do
+      expect(context[:key1]).to eq('1')
+      expect(context[:key2]).to eq(2)
+      expect(context[:key3]).to be_nil
     end
+  end
 
-    describe '#deep_dup' do
-      it 'returns a deep copy of itself' do
-        context[:a] = {nested: []}
+  describe '#deep_dup' do
+    it 'returns a deep copy of itself' do
+      context[:a] = {nested: []}
 
-        dup = context.deep_dup
-        expect(context).to eq(dup)
-        expect(context).not_to be_equal(dup)
+      dup = context.deep_dup
+      expect(context).to eq(dup)
+      expect(context).not_to be_equal(dup)
 
-        dup[:a][:nested] << 1
-        expect(context[:a][:nested]).to be_empty
-      end
+      dup[:a][:nested] << 1
+      expect(context[:a][:nested]).to be_empty
     end
+  end
 
-    describe '#serialize' do
-      before { allow(Value).to receive(:upload) }
-
-      it 'writes the serialized context to a file and upload it' do
-        expect(Value).to receive(:remotize).with(context[:key1]).and_return('remote_value1')
-        expect(Value).to receive(:remotize).with(context[:key2]).and_return('remote_value2')
+  describe '#serialize' do
+    before { allow(EXEL::Value).to receive(:upload) }
 
-        expect(SecureRandom).to receive(:uuid).and_return('uuid')
+    it 'writes the serialized context to a file and upload it' do
+      expect(EXEL::Value).to receive(:remotize).with(context[:key1]).and_return('remote_value1')
+      expect(EXEL::Value).to receive(:remotize).with(context[:key2]).and_return('remote_value2')
 
-        expect(Value).to receive(:remotize) do |file|
-          expect(file.read).to eq(Marshal.dump(Context.new(key1: 'remote_value1', key2: 'remote_value2')))
-          expect(file.path).to include('uuid')
-          'file_uri'
-        end
+      expect(SecureRandom).to receive(:uuid).and_return('uuid')
 
-        expect(context.serialize).to eq('file_uri')
+      expect(EXEL::Value).to receive(:remotize) do |file|
+        expect(file.read).to eq(Marshal.dump(EXEL::Context.new(key1: 'remote_value1', key2: 'remote_value2')))
+        expect(file.path).to include('uuid')
+        'file_uri'
       end
 
-      it 'does not mutate the current context' do
-        allow(Value).to receive(:remotize).and_return('remote_value')
-        original_table = context.dup
-        context.serialize
-        expect(context).to eq(original_table)
-      end
+      expect(context.serialize).to eq('file_uri')
     end
 
-    describe '.deserialize' do
-      it 'deserializes a given uri' do
-        file = StringIO.new(Marshal.dump(context))
-        expect(Value).to receive(:localize).with('uri').and_return(file)
+    it 'does not mutate the current context' do
+      allow(EXEL::Value).to receive(:remotize).and_return('remote_value')
+      original_table = context.dup
+      context.serialize
+      expect(context).to eq(original_table)
+    end
+  end
 
-        expect(Context.deserialize('uri')).to eq(context)
+  describe '.deserialize' do
+    it 'deserializes a given uri' do
+      file = StringIO.new(Marshal.dump(context))
+      expect(EXEL::Value).to receive(:localize).with('uri').and_return(file)
 
-        expect(file).to be_closed
-      end
+      expect(EXEL::Context.deserialize('uri')).to eq(context)
+
+      expect(file).to be_closed
     end
+  end
 
-    shared_examples 'a reader method' do
-      subject(:context) { EXEL::Context.new(key: 'value') }
+  shared_examples 'a reader method' do
+    subject(:context) { EXEL::Context.new(key: 'value') }
 
-      it 'returns the value' do
-        expect(context.send(method, :key)).to eq('value')
-      end
+    it 'returns the value' do
+      expect(context.send(method, :key)).to eq('value')
+    end
 
-      it 'localizes the returned value' do
-        expect(Value).to receive(:localize).with('value').and_return('localized')
-        expect(context.send(method, :key)).to eq('localized')
-      end
+    it 'localizes the returned value' do
+      expect(EXEL::Value).to receive(:localize).with('value').and_return('localized')
+      expect(context.send(method, :key)).to eq('localized')
+    end
 
-      it 'stores the localized value' do
-        allow(Value).to receive(:localize).with('value').and_return('localized')
-        context.send(method, :key)
-        allow(Value).to receive(:localize).with('localized').and_return('localized')
-        context.send(method, :key)
-      end
+    it 'stores the localized value' do
+      allow(EXEL::Value).to receive(:localize).with('value').and_return('localized')
+      context.send(method, :key)
+      allow(EXEL::Value).to receive(:localize).with('localized').and_return('localized')
+      context.send(method, :key)
+    end
 
-      it 'looks up deferred values' do
-        # eq(context) as an argument matcher is necessary to prevent RSpec from calling fetch on the context, leading to
-        # a stack overflow
-        expect(DeferredContextValue).to receive(:resolve).with('value', eq(context)).and_return('resolved')
-        expect(context.send(method, :key)).to eq('resolved')
-      end
+    it 'looks up deferred values' do
+      # eq(context) as an argument matcher is necessary to prevent RSpec from calling fetch on the context, leading to
+      # a stack overflow
+      expect(EXEL::DeferredContextValue).to receive(:resolve).with('value', eq(context)).and_return('resolved')
+      expect(context.send(method, :key)).to eq('resolved')
     end
+  end
 
-    describe '#[]' do
-      it_behaves_like 'a reader method' do
-        let(:method) { :[] }
-      end
+  describe '#[]' do
+    it_behaves_like 'a reader method' do
+      let(:method) { :[] }
     end
+  end
 
-    describe '#fetch' do
-      it 'raises an exception if the key is not found' do
-        expect { context.fetch(:unknown) }.to raise_error(KeyError)
-      end
+  describe '#fetch' do
+    it 'raises an exception if the key is not found' do
+      expect { context.fetch(:unknown) }.to raise_error(KeyError)
+    end
 
-      it_behaves_like 'a reader method' do
-        let(:method) { :fetch }
-      end
+    it_behaves_like 'a reader method' do
+      let(:method) { :fetch }
     end
   end
 end