exel 0.9.0 → 1.0.0

data/spec/exel/context_spec.rb

@@ -1,29 +1,25 @@
 module EXEL
   describe Context do
-    subject(:context) { EXEL::Context.new(test1: 'foo', test2: 2) }
+    subject(:context) { EXEL::Context.new(key1: '1', key2: 2) }
 
     describe '#initialize' do
       it 'should be able to initialize with a hash' do
-        expect(context.table[:test1]).to eq('foo')
-        expect(context.table[:test2]).to eq(2)
+        expect(context.table[:key1]).to eq('1')
+        expect(context.table[:key2]).to eq(2)
       end
     end
 
     describe '#serialize' do
-      let(:handler) { instance_double(Handlers::S3Handler, upload: nil) }
-
-      before do
-        allow(Handlers::S3Handler).to receive(:new).and_return(handler) #TODO don't stub new
-      end
+      before { allow(Value).to receive(:upload) }
 
       it 'should write the serialized context to a file and upload it' do
-        expect(Resource).to receive(:remotize).with(context[:test1]).and_return('remote_value1')
-        expect(Resource).to receive(:remotize).with(context[:test2]).and_return('remote_value2')
+        expect(Value).to receive(:remotize).with(context[:key1]).and_return('remote_value1')
+        expect(Value).to receive(:remotize).with(context[:key2]).and_return('remote_value2')
 
         expect(SecureRandom).to receive(:uuid).and_return('uuid')
 
-        expect(handler).to receive(:upload) do |file|
-          expect(file.read).to eq(Marshal.dump(Context.new(test1: 'remote_value1', test2: 'remote_value2')))
+        expect(Value).to receive(:remotize) do |file|
+          expect(file.read).to eq(Marshal.dump(Context.new(key1: 'remote_value1', key2: 'remote_value2')))
           expect(file.path).to include('uuid')
           'file_uri'
         end
@@ -40,26 +36,31 @@ module EXEL
 
     describe '.deserialize' do
       it 'should deserialize a given uri' do
-        uri = 'test_uri'
         file = StringIO.new(Marshal.dump(context))
-        expect_any_instance_of(Handlers::S3Handler).to receive(:download).with(uri).and_return(file)
+        expect(Value).to receive(:localize).with('uri').and_return(file)
 
-        expect(Context.deserialize(uri)).to eq(context)
+        expect(Context.deserialize('uri')).to eq(context)
 
         expect(file).to be_closed
       end
     end
 
     describe '#[]' do
-      subject(:context) { EXEL::Context.new(key: Resource.remotize('value')) }
+      subject(:context) { EXEL::Context.new(key: 'value') }
 
-      it 'should return localized values' do
+      it 'should return the value' do
         expect(context[:key]).to eq('value')
       end
 
+      it 'should localize the returned value' do
+        expect(Value).to receive(:localize).with('value').and_return('localized')
+        expect(context[:key]).to eq('localized')
+      end
+
       it 'should store the localized value' do
+        allow(Value).to receive(:localize).with('value').and_return('localized')
         context[:key]
-        expect(context.table[:key]).to eq('value')
+        expect(context.table[:key]).to eq('localized')
       end
 
       context 'DeferredContextValue object' do
@@ -70,7 +71,8 @@ module EXEL
             expect(context[:deferred_value]).to eq(context[:key])
           end
         end
-        context 'in array' do
+
+        context 'in an array' do
           it 'should return the lookup value from the context' do
             deferred_context_value = DeferredContextValue.new[:key]
             context[:array] = [1, 2, deferred_context_value]
@@ -78,11 +80,11 @@ module EXEL
           end
         end
 
-        context 'in hash' do
+        context 'in a hash' do
           it 'should return the lookup value from the context' do
             deferred_context_value = DeferredContextValue.new[:key]
             context[:hash] = {hash_key: deferred_context_value}
-            expect(context[:hash]).to eq({hash_key: context[:key]})
+            expect(context[:hash]).to eq(hash_key: context[:key])
           end
         end
 
@@ -92,10 +94,11 @@ module EXEL
             context[:nested] = [{}, {hash_key: deferred_context_value}]
             expect(context[:nested]).to eq([{}, {hash_key: context[:key]}])
           end
+
           it 'should lookup a deferred context value in an array nested in a hash' do
             deferred_context_value = DeferredContextValue.new[:key]
             context[:nested] = {hash_key: [1, deferred_context_value]}
-            expect(context[:nested]).to eq({hash_key: [1, context[:key]]})
+            expect(context[:nested]).to eq(hash_key: [1, context[:key]])
           end
         end
       end
@@ -147,5 +150,20 @@ module EXEL
       it { is_expected.to eq(context.dup) }
     end
 
+    describe 'include?' do
+      subject(:context) { EXEL::Context.new(key1: 1, key2: 2, key3: 3) }
+
+      context 'context contains all key value pairs' do
+        it 'should return true' do
+          expect(context).to include(key1: 1, key2: 2)
+        end
+      end
+
+      context 'context does not contain all key value pairs' do
+        it 'should return true' do
+          expect(context).not_to include(foo: 'bar', key2: 2)
+        end
+      end
+    end
   end
 end

data/spec/exel/job_spec.rb

@@ -2,7 +2,7 @@ module EXEL
   describe Job do
     describe '.define' do
       let(:ast) { instance_double(SequenceNode, run: nil, start: nil) }
-      let(:block) { Proc.new {} }
+      let(:block) { proc {} }
 
       after { Job.registry.clear }
 
@@ -37,7 +37,7 @@ module EXEL
 
       context 'with a job name' do
         context 'of a defined job' do
-          let(:block) { Proc.new {} }
+          let(:block) { proc {} }
 
           before do
             allow(Job).to receive(:registry).and_return(test_job: block)
@@ -103,7 +103,7 @@ module EXEL
 
       context 'given DSL code as a proc' do
         it 'should eval the code as a block' do
-          dsl_proc = Proc.new {}
+          dsl_proc = proc {}
           expect(parser).to receive(:instance_eval) do |*_args, &block|
             expect(block).to eq(dsl_proc)
           end
@@ -122,12 +122,12 @@ module EXEL
       end
 
       it 'should return the parsed AST' do
-        expect(Job::Parser.parse(Proc.new {})).to eq(ast)
+        expect(Job::Parser.parse(proc {})).to eq(ast)
       end
     end
 
     describe '#process' do
-      let(:block) { Proc.new {} }
+      let(:block) { proc {} }
 
       before do
         allow(Job::Parser).to receive(:parse).and_return(ast)
@@ -174,8 +174,8 @@ module EXEL
     end
 
     [
-        {method: :async, processor: Processors::AsyncProcessor},
-        {method: :split, processor: Processors::SplitProcessor}
+      {method: :async, processor: Processors::AsyncProcessor},
+      {method: :split, processor: Processors::SplitProcessor}
     ].each do |data|
       describe "##{data[:method]}" do
         before do
@@ -184,7 +184,7 @@ module EXEL
 
         it "should create a #{data[:method]} instruction" do
           expect(Instruction).to receive(:new).with(data[:method].to_s, data[:processor], {arg1: 'arg1_value'}, ast)
-          parser.send(data[:method], {arg1: 'arg1_value'}) {}
+          parser.send(data[:method], arg1: 'arg1_value') {}
         end
 
         it 'should parse the block given' do

data/spec/exel/logging_spec.rb

@@ -21,8 +21,8 @@ module EXEL
 
       it 'should initialize the logger on first read if not already set' do
         EXEL.configure do |config|
-          config[:log_level] = :warn
-          config[:log_filename] = 'log.txt'
+          config.log_level = :warn
+          config.log_filename = 'log.txt'
         end
 
         logger = instance_double(Logger)
@@ -33,4 +33,4 @@ module EXEL
       end
     end
   end
-end
+end

data/spec/exel/processors/async_processor_spec.rb

@@ -1,16 +1,24 @@
 module EXEL
   module Processors
     describe AsyncProcessor do
-      subject(:processor) { AsyncProcessor.new(context) }
+      subject(:processor) { described_class.new(context) }
       let(:context) { EXEL::Context.new }
       let(:block) { instance_double(SequenceNode) }
 
+      before do
+        allow(EXEL).to receive(:async_provider).and_return(EXEL::Providers::DummyAsyncProvider)
+      end
+
+      it 'looks up the async provider on initialization' do
+        expect(processor.provider).to be_an_instance_of(EXEL::Providers::DummyAsyncProvider)
+      end
+
       describe '#process' do
-        it 'should call do_async on the async handler' do
-          expect(processor.handler).to receive(:do_async).with(block)
+        it 'calls do_async on the async provider' do
+          expect(processor.provider).to receive(:do_async).with(block)
           processor.process(block)
         end
       end
     end
   end
-end
+end

data/spec/exel/processors/split_processor_spec.rb

@@ -1,90 +1,90 @@
-  module EXEL
-    module Processors
-      describe SplitProcessor do
-        let(:chunk_file) { instance_double(File) }
-        let(:file) { create_file(1) }
-        let(:context) { Context.new(resource: file) }
-        let(:callback) { instance_double(SequenceNode) }
-        subject(:splitter) { SplitProcessor.new(context) }
-
-        before do
-          allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
-          allow(File).to receive(:delete)
-        end
+module EXEL
+  module Processors
+    describe SplitProcessor do
+      let(:chunk_file) { instance_double(File) }
+      let(:file) { create_file(1) }
+      let(:context) { Context.new(resource: file) }
+      let(:callback) { instance_double(SequenceNode) }
+      subject(:splitter) { SplitProcessor.new(context) }
+
+      before do
+        allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
+        allow(File).to receive(:delete)
+      end
 
-        describe '#process' do
-          let(:file) { create_file(3) }
+      describe '#process' do
+        let(:file) { create_file(3) }
 
-          it 'should process file with 3 lines line by line' do
-            allow(CSV).to receive(:foreach).and_yield("line0").and_yield("line1").and_yield("line2")
+        it 'should process file with 3 lines line by line' do
+          allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
 
-            3.times do |i|
-              expect(splitter).to receive(:process_line).with("line#{i}", callback)
-            end
-            expect(splitter).to receive(:process_line).with(:eof, callback)
+          3.times do |i|
+            expect(splitter).to receive(:process_line).with("line#{i}", callback)
+          end
+          expect(splitter).to receive(:process_line).with(:eof, callback)
 
-            expect(File).to receive(:delete).with(file.path)
+          expect(File).to receive(:delete).with(file.path)
 
-            splitter.process(callback)
-          end
+          splitter.process(callback)
+        end
 
-          it 'should abort parsing the csv file if it is malformed' do
-            allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
-            expect(splitter).to receive(:process_line).with(:eof, callback)
+        it 'should abort parsing the csv file if it is malformed' do
+          allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
+          expect(splitter).to receive(:process_line).with(:eof, callback)
 
-            splitter.process(callback)
-          end
+          splitter.process(callback)
         end
+      end
 
-        describe '#process_line' do
-          [
-              {input: 1, chunks: %W(0\n)},
-              {input: 3, chunks: %W(0\n1\n 2\n)},
-              {input: 4, chunks: %W(0\n1\n 2\n3\n)}
-          ].each do |data|
-            it "should produce #{data[:chunks].size} chunks with #{data[:input]} input lines" do
-              splitter.chunk_size = 2
-
-              data[:chunks].each do |chunk|
-                expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
-                expect(callback).to receive(:run).with(context) do
-                  expect(context[:resource]).to eq(chunk_file)
-                end
+      describe '#process_line' do
+        [
+          {input: 1, chunks: %W(0\n)},
+          {input: 3, chunks: %W(0\n1\n 2\n)},
+          {input: 4, chunks: %W(0\n1\n 2\n3\n)}
+        ].each do |data|
+          it "should produce #{data[:chunks].size} chunks with #{data[:input]} input lines" do
+            splitter.chunk_size = 2
+
+            data[:chunks].each do |chunk|
+              expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
+              expect(callback).to receive(:run).with(context) do
+                expect(context[:resource]).to eq(chunk_file)
               end
-
-              data[:input].times { |i| splitter.process_line([i.to_s], callback) }
-              splitter.process_line(:eof, callback)
             end
+
+            data[:input].times { |i| splitter.process_line([i.to_s], callback) }
+            splitter.process_line(:eof, callback)
           end
         end
+      end
 
-        describe '#generate_chunk' do
-          it 'should create a file with the contents of the given string' do
-            file = splitter.generate_chunk('abc')
-            content = file.read
-            expect(content).to eq('abc')
-          end
+      describe '#generate_chunk' do
+        it 'should create a file with the contents of the given string' do
+          file = splitter.generate_chunk('abc')
+          content = file.read
+          expect(content).to eq('abc')
+        end
 
-          it 'should create a file with a unique name' do
-            3.times do |i|
-              index = i + 1
-              file = splitter.generate_chunk("#{index}")
-              file_name = splitter.filename(file)
-              expect(file_name).to include("text_#{index}_")
-            end
+        it 'should create a file with a unique name' do
+          3.times do |i|
+            index = i + 1
+            file = splitter.generate_chunk("#{index}")
+            file_name = splitter.filename(file)
+            expect(file_name).to include("text_#{index}_")
           end
         end
+      end
 
-        def create_file(lines)
-          content = ''
-
-          lines.times do |i|
-            line = CSV.generate_line(["line#{i}"])
-            content << line
-          end
+      def create_file(lines)
+        content = ''
 
-          StringIO.new content
+        lines.times do |i|
+          line = CSV.generate_line(["line#{i}"])
+          content << line
         end
+
+        StringIO.new content
       end
     end
   end
+end

data/spec/exel/providers/local_async_provider_spec.rb

@@ -0,0 +1,19 @@
+module EXEL
+  module Providers
+    describe ThreadedAsyncProvider do
+      subject { described_class.new(context) }
+      let(:context) { EXEL::Context.new }
+
+      describe '#do_async' do
+        let(:dsl_block) { instance_double(ASTNode) }
+
+        it 'runs the block in a new thread' do
+          expect(dsl_block).to receive(:start).with(context)
+          expect(Thread).to receive(:new).and_yield
+
+          subject.do_async(dsl_block)
+        end
+      end
+    end
+  end
+end

data/spec/exel/providers/local_file_provider_spec.rb

@@ -0,0 +1,36 @@
+module EXEL
+  module Providers
+    describe LocalFileProvider do
+      let(:file) { instance_double(File, path: '/path/to/file') }
+
+      describe '#upload' do
+        it 'returns a file:// URI for the file' do
+          expect(subject.upload(file)).to eq('file:///path/to/file')
+        end
+      end
+
+      describe '#download' do
+        it 'returns the file indicated by the URI' do
+          expect(File).to receive(:open).with('/path/to/file').and_return(file)
+          expect(subject.download('file:///path/to/file')).to eq(file)
+        end
+
+        it 'doesn`t accept URIs for schemes other than file://' do
+          expect { subject.download('s3://') }.to raise_error 'URI must begin with "file://"'
+        end
+      end
+
+      describe '.remote?' do
+        it 'returns true for file:// URIs' do
+          expect(LocalFileProvider.remote?('file:///path/to/file')).to be_truthy
+        end
+
+        it 'returns false for anything else' do
+          expect(LocalFileProvider.remote?('s3://file')).to be_falsey
+          expect(LocalFileProvider.remote?(1)).to be_falsey
+          expect(LocalFileProvider.remote?(nil)).to be_falsey
+        end
+      end
+    end
+  end
+end

data/spec/exel/sequence_node_spec.rb

@@ -1,23 +1,16 @@
 module EXEL
   describe SequenceNode do
-    let(:context) { {} }
+    subject(:node) { described_class.new(instance_double(ASTNode), instance_double(ASTNode)) }
+    let(:context) { instance_double(EXEL::Context) }
 
-    def build_tree
-      @node_2 = instance_double(ASTNode)
-      @node_3 = instance_double(ASTNode)
-      @node_1 = SequenceNode.new(@node_2, @node_3)
-    end
-
-    it { is_expected.to be_kind_of(ASTNode) }
+    it { is_expected.to be_an(ASTNode) }
 
     describe '#run' do
-      before { build_tree }
-
       it 'should run each child node in sequence' do
-        expect(@node_2).to receive(:run).with(context).once.ordered
-        expect(@node_3).to receive(:run).with(context).once.ordered
+        expect(node.children.first).to receive(:run).with(context).once.ordered
+        expect(node.children.last).to receive(:run).with(context).once.ordered
 
-        @node_1.run(context)
+        node.run(context)
       end
     end
   end

data/spec/exel/value_spec.rb

@@ -0,0 +1,51 @@
+module EXEL
+  describe Value do
+    let(:uri) { 's3://test_file.csv' }
+
+    before { allow(EXEL).to receive(:remote_provider).and_return(EXEL::Providers::DummyRemoteProvider) }
+
+    describe '.remotize' do
+      context 'when the value is not a file' do
+        it 'returns the value' do
+          expect(Value.remotize('test')).to eq('test')
+        end
+      end
+
+      [File, Tempfile].each do |file_class|
+        context "when the value is an instance of #{file_class}" do
+          let(:file) { instance_double(file_class) }
+
+          before { allow(file).to receive(:is_a?) { |klass| klass == file_class } }
+
+          it 'uploads the file using the remote provider' do
+            expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file)
+            Value.remotize(file)
+          end
+
+          it 'returns the URI of the uploaded file' do
+            allow_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file).and_return(uri)
+            expect(Value.remotize(file)).to eq(uri)
+          end
+        end
+      end
+    end
+
+    describe '.localize' do
+      context 'with a local value' do
+        it 'returns the value' do
+          expect(Value.localize('test')).to eq('test')
+        end
+      end
+
+      context 'with a remote file' do
+        it 'returns the downloaded file' do
+          expect(EXEL::Providers::DummyRemoteProvider).to receive(:remote?).with(uri).and_return(true)
+          file = double(:file)
+          expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:download).with(uri).and_return(file)
+
+          expect(Value.localize(uri)).to eq(file)
+        end
+      end
+    end
+  end
+end

data/spec/exel_spec.rb

@@ -0,0 +1,41 @@
+describe EXEL do
+  describe '.async_provider' do
+    context 'with no async provider set in the configuration' do
+      it 'defaults to ThreadedAsyncProvider' do
+        expect(EXEL.async_provider).to eq(EXEL::Providers::ThreadedAsyncProvider)
+      end
+    end
+
+    context 'with an async provider set in the configuration' do
+      before do
+        EXEL.configure do |config|
+          config.async_provider = EXEL::Providers::DummyAsyncProvider
+        end
+      end
+
+      it 'returns the configured async provider' do
+        expect(EXEL.async_provider).to eq(EXEL::Providers::DummyAsyncProvider)
+      end
+    end
+  end
+
+  describe '.remote_provider' do
+    context 'with no remote provider set in the configuration' do
+      it 'defaults to NullRemoteProvider' do
+        expect(EXEL.remote_provider).to eq(EXEL::Providers::LocalFileProvider)
+      end
+    end
+
+    context 'with no remote provider set in the configuration' do
+      before do
+        EXEL.configure do |config|
+          config.remote_provider = EXEL::Providers::DummyRemoteProvider
+        end
+      end
+
+      it 'returns the configurated remote provider' do
+        expect(EXEL.remote_provider).to eq(EXEL::Providers::DummyRemoteProvider)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -2,6 +2,25 @@ Dir[File.expand_path('../../lib/**/*.rb', __FILE__)].each { |f| require f }
 
 EXEL.logger = nil
 
-EXEL.configure do |config|
-  config[:aws] = {}
-end
+module EXEL
+  module Providers
+    class DummyAsyncProvider
+      def initialize(_context)
+      end
+
+      def do_async(_block)
+      end
+    end
+
+    class DummyRemoteProvider
+      def upload(_file)
+      end
+
+      def download(_uri)
+      end
+
+      def self.remote?(_uri)
+      end
+    end
+  end
+end