exel 1.1.0 → 1.2.0

data/lib/exel/processors/split_processor.rb

@@ -4,18 +4,28 @@ require_relative '../processor_helper'
 
 module EXEL
   module Processors
+    # Implements the +split+ instruction. Used to concurrently process a large file by splitting it into small chunks to
+    # be separately processed.
+    #
+    # =Supported Context Options
+    # * +:delete_resource+ Defaults to true, can be set to false to preserve the original resource. Otherwise, it will
+    #   be deleted when splitting is complete
+    # * +:chunk_size+ Set to specify the number of lines that each chunk should contain
     class SplitProcessor
       include EXEL::ProcessorHelper
 
       attr_accessor :file_name, :block
 
+      # Number of lines to include in each chunk. Can be overridden by setting :chunk_size in the context
       DEFAULT_CHUNK_SIZE = 1000
 
+      # The context must contain a CSV File object in context[:resource]
       def initialize(context)
         @buffer = []
         @tempfile_count = 0
         @context = context
         @file = context[:resource]
+        @context[:delete_resource] = true if @context[:delete_resource].nil?
 
         log_prefix_with '[SplitProcessor]'
       end
@@ -73,7 +83,7 @@ module EXEL
       end
 
       def chunk_size
-        DEFAULT_CHUNK_SIZE
+        @context[:chunk_size] || DEFAULT_CHUNK_SIZE
       end
 
       def chunk_filename
@@ -87,7 +97,7 @@ module EXEL
 
       def finish(callback)
         process_line(:eof, callback)
-        File.delete(@file.path)
+        File.delete(@file.path) if @context[:delete_resource]
       end
     end
   end

data/lib/exel/providers/local_file_provider.rb

@@ -1,12 +1,14 @@
 module EXEL
   module Providers
+    # The default remote provider. Doesn't actually upload and download files to and from remote storage, but rather
+    # just works with local files.
     class LocalFileProvider
       def upload(file)
         "file://#{file.path}"
       end
 
       def download(uri)
-        fail 'URI must begin with "file://"' unless uri.start_with? 'file://'
+        raise 'URI must begin with "file://"' unless uri.start_with? 'file://'
         File.open(uri.split('file://').last)
       end
 

data/lib/exel/providers/threaded_async_provider.rb

@@ -1,5 +1,6 @@
 module EXEL
   module Providers
+    # The default remote provider. Provides async execution by running the given EXEL block in a new Thread
     class ThreadedAsyncProvider
       def initialize(context)
         @context = context

data/lib/exel/sequence_node.rb

@@ -1,6 +1,7 @@
 require_relative './ast_node'
 
 module EXEL
+  # A node in the AST that has as its children a sequence of nodes to be run sequentially
   class SequenceNode < ASTNode
     def initialize(*children)
       @instruction = NullInstruction.new

data/lib/exel/value.rb

@@ -1,4 +1,5 @@
 module EXEL
+  # Contains methods to handle remote and local values. Used for {Context} serialization
   module Value
     def self.remotize(value)
       file?(value) ? upload(value) : value

data/lib/exel/version.rb

@@ -1,3 +1,3 @@
 module EXEL
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'.freeze
 end

data/lib/exel.rb

@@ -2,31 +2,50 @@ require 'exel/version'
 require 'exel/logging'
 require 'ostruct'
 
+# Provides methods to configure EXEL
 module EXEL
+  # @return The currently set logger
   def self.logger
     EXEL::Logging.logger
   end
 
+  # Sets the logger to be used.
+  #
+  # @param [Logger] The logger to set. Must comply with the Ruby Logger interface
   def self.logger=(logger)
     EXEL::Logging.logger = logger
   end
 
+  # @return The current configuration
   def self.configuration
     @config ||= OpenStruct.new
   end
 
+  # Yields the configuration object to the given block. Configuration can include:
+  # * +async_provider+ Set an async provider. Defaults to EXEL::Providers::ThreadedAsyncProvider
+  # * +remote_provider+ Set a remote provider. Defaults to EXEL::Providers::LocalFileProvider
+  # * Any configuration required by the async/remote providers
+  #
+  # Typically, async_provider and remote_provider will be automatically set upon requiring those gems.
+  #
+  # Example:
+  #   EXEL.configure do |config|
+  #     config.s3_bucket = 'my_bucket'
+  #   end
   def self.configure
     yield configuration
   end
 
+  # @return The currently configured async provider. Defaults to EXEL::Providers::ThreadedAsyncProvider
   def self.async_provider
     configuration.async_provider || Providers::ThreadedAsyncProvider
   end
 
+  # @return The currently configured remote provider. Defaults to EXEL::Providers::LocalFileProvider
   def self.remote_provider
     configuration.remote_provider || Providers::LocalFileProvider
   end
 
   root = File.expand_path('../..', __FILE__)
-  Dir[File.join(root, 'lib/exel/**/*.rb')].each { |file| require file }
+  Dir[File.join(root, 'lib/exel/**/*.rb')].reject { |file| file.include?('old_context') }.each { |file| require file }
 end

data/spec/exel/ast_node_spec.rb

@@ -10,23 +10,23 @@ module EXEL
 
     describe '#start' do
       context 'when an JobTermination error bubbles up' do
-        it 'should ensure the process fails silently' do
+        it 'ensures the process fails silently' do
           node = TestNode.new(instruction)
           allow(node).to receive(:run).and_raise(EXEL::Error::JobTermination, 'Error')
           expect(EXEL.logger).to receive(:error).with('JobTerminationError: Error')
-          expect { node.start(context) }.to_not raise_error
+          expect { node.start(context) }.not_to raise_error
         end
       end
     end
 
     describe '#run' do
-      it 'should raise an error if not implemented' do
+      it 'raises an error if not implemented' do
         expect { TestNode.new(instruction).run(context) }.to raise_error 'EXEL::TestNode does not implement #process'
       end
     end
 
     describe '#add_child' do
-      it 'should add the given node to its children' do
+      it 'adds the given node to its children' do
         root = ASTNode.new(instruction)
         child_node = ASTNode.new(instruction)
         child_node2 = ASTNode.new(instruction)

data/spec/exel/context_spec.rb

@@ -2,10 +2,13 @@ module EXEL
   describe Context do
     subject(:context) { EXEL::Context.new(key1: '1', key2: 2) }
 
+    it { is_expected.to be_a(Hash) }
+
     describe '#initialize' do
-      it 'should be able to initialize with a hash' do
-        expect(context.table[:key1]).to eq('1')
-        expect(context.table[:key2]).to eq(2)
+      it 'initializes with a hash' do
+        expect(context[:key1]).to eq('1')
+        expect(context[:key2]).to eq(2)
+        expect(context[:key3]).to be_nil
       end
     end
 
@@ -15,7 +18,7 @@ module EXEL
 
         dup = context.deep_dup
         expect(context).to eq(dup)
-        expect(context).to_not be_equal(dup)
+        expect(context).not_to be_equal(dup)
 
         dup[:a][:nested] << 1
         expect(context[:a][:nested]).to be_empty
@@ -25,7 +28,7 @@ module EXEL
     describe '#serialize' do
       before { allow(Value).to receive(:upload) }
 
-      it 'should write the serialized context to a file and upload it' do
+      it 'writes the serialized context to a file and upload it' do
         expect(Value).to receive(:remotize).with(context[:key1]).and_return('remote_value1')
         expect(Value).to receive(:remotize).with(context[:key2]).and_return('remote_value2')
 
@@ -40,15 +43,16 @@ module EXEL
         expect(context.serialize).to eq('file_uri')
       end
 
-      it 'should not mutate the current context' do
-        original_table = context.table.dup
+      it 'does not mutate the current context' do
+        allow(Value).to receive(:remotize).and_return('remote_value')
+        original_table = context.dup
         context.serialize
-        expect(context.table).to eq(original_table)
+        expect(context).to eq(original_table)
       end
     end
 
     describe '.deserialize' do
-      it 'should deserialize a given uri' do
+      it 'deserializes a given uri' do
         file = StringIO.new(Marshal.dump(context))
         expect(Value).to receive(:localize).with('uri').and_return(file)
 
@@ -58,124 +62,46 @@ module EXEL
       end
     end
 
-    describe '#[]' do
+    shared_examples 'a reader method' do
       subject(:context) { EXEL::Context.new(key: 'value') }
 
-      it 'should return the value' do
-        expect(context[:key]).to eq('value')
+      it 'returns the value' do
+        expect(context.send(method, :key)).to eq('value')
       end
 
-      it 'should localize the returned value' do
+      it 'localizes the returned value' do
         expect(Value).to receive(:localize).with('value').and_return('localized')
-        expect(context[:key]).to eq('localized')
+        expect(context.send(method, :key)).to eq('localized')
       end
 
-      it 'should store the localized value' do
+      it 'stores the localized value' do
         allow(Value).to receive(:localize).with('value').and_return('localized')
-        context[:key]
-        expect(context.table[:key]).to eq('localized')
+        context.send(method, :key)
+        allow(Value).to receive(:localize).with('localized').and_return('localized')
+        context.send(method, :key)
       end
 
-      context 'DeferredContextValue object' do
-        context 'at the top level' do
-          it 'should return the lookup value from the context' do
-            deferred_context_value = DeferredContextValue.new[:key]
-            context[:deferred_value] = deferred_context_value
-            expect(context[:deferred_value]).to eq(context[:key])
-          end
-        end
-
-        context 'in an array' do
-          it 'should return the lookup value from the context' do
-            deferred_context_value = DeferredContextValue.new[:key]
-            context[:array] = [1, 2, deferred_context_value]
-            expect(context[:array]).to eq([1, 2, context[:key]])
-          end
-        end
-
-        context 'in a hash' do
-          it 'should return the lookup value from the context' do
-            deferred_context_value = DeferredContextValue.new[:key]
-            context[:hash] = {hash_key: deferred_context_value}
-            expect(context[:hash]).to eq(hash_key: context[:key])
-          end
-        end
-
-        context 'in nested arrays and hashes' do
-          it 'should lookup a deferred context value in a hash nested in an array' do
-            deferred_context_value = DeferredContextValue.new[:key]
-            context[:nested] = [{}, {hash_key: deferred_context_value}]
-            expect(context[:nested]).to eq([{}, {hash_key: context[:key]}])
-          end
-
-          it 'should lookup a deferred context value in an array nested in a hash' do
-            deferred_context_value = DeferredContextValue.new[:key]
-            context[:nested] = {hash_key: [1, deferred_context_value]}
-            expect(context[:nested]).to eq(hash_key: [1, context[:key]])
-          end
-        end
-      end
-    end
-
-    describe '#[]=' do
-      it 'should add the key/value pair to table' do
-        context[:new_key] = 'new_value'
-        expect(context.table[:new_key]).to eq('new_value')
-      end
-    end
-
-    describe '#delete' do
-      it 'should delete the key/value pair from the table' do
-        context[:key] = 'value'
-        context[:key2] = 'value2'
-        context.delete(:key)
-        expect(context.table.keys).to_not include(:key)
-        expect(context.table.keys).to include(:key2)
+      it 'looks up deferred values' do
+        # eq(context) as an argument matcher is necessary to prevent RSpec from calling fetch on the context, leading to
+        # a stack overflow
+        expect(DeferredContextValue).to receive(:resolve).with('value', eq(context)).and_return('resolved')
+        expect(context.send(method, :key)).to eq('resolved')
       end
     end
 
-    describe '#merge!' do
-      it 'should merge given keys and values into the context' do
-        context.table[:overwrite] = 'overwrite'
-        context.table[:existing] = 'existing'
-
-        context.merge!(overwrite: 'changed', new: 'new')
-
-        expect(context.table[:overwrite]).to eq('changed')
-        expect(context.table[:existing]).to eq('existing')
-        expect(context.table[:new]).to eq('new')
-      end
-
-      it 'should return itself' do
-        expect(context.merge!(key: 'value')).to eq(context)
+    describe '#[]' do
+      it_behaves_like 'a reader method' do
+        let(:method) { :[] }
       end
     end
 
-    describe '#==' do
-      it { is_expected.to_not eq(nil) }
-
-      it { is_expected.to eq(context) }
-
-      it { is_expected.to_not eq(42) }
-
-      it { is_expected.to_not eq(Context.new(other_key: 'value')) }
-
-      it { is_expected.to eq(context.dup) }
-    end
-
-    describe 'include?' do
-      subject(:context) { EXEL::Context.new(key1: 1, key2: 2, key3: 3) }
-
-      context 'context contains all key value pairs' do
-        it 'should return true' do
-          expect(context).to include(key1: 1, key2: 2)
-        end
+    describe '#fetch' do
+      it 'raises an exception if the key is not found' do
+        expect { context.fetch(:unknown) }.to raise_error(KeyError)
       end
 
-      context 'context does not contain all key value pairs' do
-        it 'should return true' do
-          expect(context).not_to include(foo: 'bar', key2: 2)
-        end
+      it_behaves_like 'a reader method' do
+        let(:method) { :fetch }
       end
     end
   end

data/spec/exel/deferred_context_value_spec.rb

@@ -1,20 +1,64 @@
 module EXEL
   describe DeferredContextValue do
-    subject(:deferred_context_value) { DeferredContextValue.new }
+    subject(:deferred_value) { DeferredContextValue.new }
+
+    describe '.resolve' do
+      subject(:deferred_value) { DeferredContextValue.new[:key] }
+
+      context 'at the top level' do
+        let(:context) { {key: 'value', deferred_value: deferred_value} }
+
+        it 'returns the lookup value from the context' do
+          expect(DeferredContextValue.resolve(context[:deferred_value], context)).to eq(context[:key])
+        end
+      end
+
+      context 'in an array' do
+        let(:context) { {key: 'value', array: [1, 2, deferred_value]} }
+
+        it 'returns the lookup value from the context' do
+          expect(DeferredContextValue.resolve(context[:array], context)).to eq([1, 2, context[:key]])
+        end
+      end
+
+      context 'in a hash' do
+        let(:context) { {key: 'value', hash: {hash_key: deferred_value}} }
+
+        it 'returns the lookup value from the context' do
+          expect(DeferredContextValue.resolve(context[:hash], context)).to eq(hash_key: context[:key])
+        end
+      end
+
+      context 'in a hash nested in an array' do
+        let(:context) { {key: 'value', nested: [{}, {hash_key: deferred_value}]} }
+
+        it 'looks up a deferred context value in a hash nested in an array' do
+          expect(DeferredContextValue.resolve(context[:nested], context)).to eq([{}, {hash_key: context[:key]}])
+        end
+      end
+
+      context 'in an array nested in a hash' do
+        let(:context) { {key: 'value', nested: {hash_key: [1, deferred_value]}} }
+
+        it 'looks up a deferred context value in an array nested in a hash' do
+          expect(DeferredContextValue.resolve(context[:nested], context)).to eq(hash_key: [1, context[:key]])
+        end
+      end
+    end
 
     describe '#[]' do
-      it 'should store passed key in the keys attribute' do
-        deferred_context_value[:top_level_key]['sub_key']
-        expect(deferred_context_value.keys).to eq([:top_level_key, 'sub_key'])
+      it 'stores the given key in the keys attribute' do
+        deferred_value[:top_level_key]['sub_key']
+        expect(deferred_value.keys).to eq([:top_level_key, 'sub_key'])
       end
     end
 
     describe '#get' do
-      it 'should look up the value of the keys attribute in the passed-in context' do
-        allow(deferred_context_value).to receive(:keys).and_return([:top_level_key, 'sub_key'])
+      it 'looks up the value of the keys attribute in the passed-in context' do
+        allow(deferred_value).to receive(:keys).and_return([:top_level_key, 'sub_key'])
         value = 'example_value'
         context = Context.new(top_level_key: {'sub_key' => value})
-        expect(deferred_context_value.get(context)).to eq(value)
+        expect(deferred_value.get(context)).to eq(value)
       end
     end
   end

data/spec/exel/events_spec.rb

@@ -0,0 +1,89 @@
+module EXEL
+  describe Events do
+    class EventTest
+      include Events
+    end
+
+    subject(:events) { EventTest.new }
+    let(:event_listener) { double(:event_listener) }
+    let(:context) { EXEL::Context.new(Events::LISTENERS_KEY => {event: [event_listener]}) }
+
+    describe '#register_listener' do
+      context 'when no listeners have been defined' do
+        let(:context) { EXEL::Context.new }
+
+        it 'adds a new listener to the context' do
+          events.register_listener(context, :event, event_listener)
+          expect(context[Events::LISTENERS_KEY].fetch(:event)).to contain_exactly(event_listener)
+        end
+      end
+
+      it 'registers multiple listeners for the same event' do
+        new_listener = double(:event_listener2)
+        events.register_listener(context, :event, new_listener)
+        expect(context[Events::LISTENERS_KEY].fetch(:event)).to contain_exactly(event_listener, new_listener)
+      end
+    end
+
+    describe '#trigger' do
+      let(:context) { EXEL::Context.new }
+      let(:data) { {foo: 1} }
+
+      before { allow(events).to receive(:context).and_return(context) }
+
+      context 'when no events have been registered' do
+        it 'does not trigger anything' do
+          expect(event_listener).not_to receive(:event)
+
+          events.trigger(:event, data)
+        end
+      end
+
+      context 'with a single listener registered for the event' do
+        before do
+          events.register_listener(context, :event, event_listener)
+        end
+
+        it 'calls the listener with the context and event data' do
+          expect(event_listener).to receive(:event).with(context, data)
+
+          events.trigger(:event, data)
+        end
+
+        it 'passes an empty hash if no data was given' do
+          expect(event_listener).to receive(:event).with(context, {})
+
+          events.trigger(:event)
+        end
+      end
+
+      context 'with no listeners registered for the event' do
+        before do
+          events.register_listener(context, :other_event, event_listener)
+        end
+
+        it 'does not trigger anything' do
+          expect(event_listener).not_to receive(:event)
+
+          events.trigger(:event, data)
+        end
+      end
+
+      context 'with multiple listeners registered for the event' do
+        let(:event_listener2) { double(:event_listener2) }
+
+        before do
+          events.register_listener(context, :event, event_listener)
+          events.register_listener(context, :event, event_listener2)
+        end
+
+        it 'calls each listener with the context and event data' do
+          expect(event_listener).to receive(:event).with(context, data)
+          expect(event_listener2).to receive(:event).with(context, data)
+
+          events.trigger(:event, data)
+        end
+      end
+    end
+  end
+end

data/spec/exel/instruction_node_spec.rb

@@ -8,13 +8,13 @@ module EXEL
     it { is_expected.to be_kind_of(ASTNode) }
 
     describe '#run' do
-      it 'should only execute the instruction' do
+      it 'only executes the instruction' do
         expect(instruction).to receive(:execute).with(context).once
         node.run(context)
       end
 
-      it 'should not run it`s children' do
-        expect(child).to_not receive(:run)
+      it 'does not run it`s children' do
+        expect(child).not_to receive(:run)
         node.run(context)
       end
     end

data/spec/exel/instruction_spec.rb

@@ -1,20 +1,20 @@
 module EXEL
   describe Instruction do
-    subject(:instruction) { EXEL::Instruction.new('ins_name', processor_class, args) }
+    subject(:instruction) { EXEL::Instruction.new(processor_class, args) }
     let(:processor_class) { double(:processor_class, new: processor_instance) }
     let(:processor_instance) { double(:processor_instance, process: nil) }
     let(:args) { {arg1: 'arg_value1', arg2: {}} }
     let(:context) { {context_key: 'context_value'} }
 
-    describe '#run' do
-      it 'should call process on an instance of the processor class' do
+    describe '#execute' do
+      it 'calls process on an instance of the processor class' do
         expect(processor_class).to receive(:new).and_return(processor_instance)
         expect(processor_instance).to receive(:process)
 
         instruction.execute(context)
       end
 
-      it 'should not pass a copy of the context' do
+      it 'does not pass a copy of the context' do
         allow(processor_class).to receive(:new) do |context_arg|
           expect(context_arg).to be(context)
           processor_instance
@@ -23,13 +23,13 @@ module EXEL
         instruction.execute(context)
       end
 
-      it 'should add args to the context' do
+      it 'adds args to the context' do
         instruction.execute(context)
         expect(context.keys).to include(*args.keys)
       end
 
       context 'with args' do
-        it 'should pass the args to the processor' do
+        it 'passes the args to the processor' do
           expect(processor_class).to receive(:new).with(hash_including(args))
           instruction.execute(context)
         end
@@ -38,7 +38,7 @@ module EXEL
       context 'without args' do
         let(:args) { nil }
 
-        it 'should just pass the context to the processor' do
+        it 'passes only the context to the processor' do
           expect(processor_class).to receive(:new).with(context)
           instruction.execute(context)
         end
@@ -46,9 +46,9 @@ module EXEL
 
       context 'with a subtree' do
         let(:subtree) { double(:subtree) }
-        subject(:instruction) { EXEL::Instruction.new('ins_name', processor_class, args, subtree) }
+        subject(:instruction) { EXEL::Instruction.new(processor_class, args, subtree) }
 
-        it 'should pass the subtree to the processor' do
+        it 'passes the subtree to the processor' do
           expect(processor_instance).to receive(:process).with(subtree)
           instruction.execute(context)
         end