exel 1.2.1 → 1.5.2

data/spec/exel/processors/run_processor_spec.rb

@@ -1,15 +1,14 @@
-module EXEL
-  module Processors
-    describe RunProcessor do
-      subject { RunProcessor.new(context) }
-      let(:context) { EXEL::Context.new(job: :test_job) }
+# frozen_string_literal: true
 
-      describe '#process' do
-        it 'runs the job named in context[:job] with the current context' do
-          expect(EXEL::Job).to receive(:run).with(:test_job, context)
-          subject.process
-        end
-      end
+describe EXEL::Processors::RunProcessor do
+  subject { EXEL::Processors::RunProcessor.new(context) }
+
+  let(:context) { EXEL::Context.new(job: :test_job) }
+
+  describe '#process' do
+    it 'runs the job named in context[:job] with the current context' do
+      expect(EXEL::Job).to receive(:run).with(:test_job, context)
+      subject.process
     end
   end
 end

data/spec/exel/processors/split_processor_spec.rb

@@ -1,96 +1,121 @@
-module EXEL
-  module Processors
-    describe SplitProcessor do
-      let(:chunk_file) { instance_double(File) }
-      let(:file) { create_file(1) }
-      let(:context) { Context.new(resource: file) }
-      let(:callback) { instance_double(SequenceNode) }
-      subject(:splitter) { SplitProcessor.new(context) }
-
-      before do
-        allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
-        allow(File).to receive(:delete)
+# frozen_string_literal: true
+
+describe EXEL::Processors::SplitProcessor do
+  subject(:splitter) { EXEL::Processors::SplitProcessor.new(context) }
+
+  let(:chunk_file) { instance_double(File) }
+  let(:file) { create_file(1) }
+  let(:context) { EXEL::Context.new(resource: file) }
+  let(:callback) { instance_double(EXEL::SequenceNode) }
+
+  before do
+    allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
+    allow(File).to receive(:delete)
+  end
+
+  describe '#process' do
+    let(:file) { create_file(3) }
+
+    it 'processes file with 3 lines line by line' do
+      allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
+
+      3.times do |i|
+        expect(splitter).to receive(:process_line).with("line#{i}", callback)
       end
+      expect(splitter).to receive(:process_line).with(:eof, callback)
 
-      describe '#process' do
-        let(:file) { create_file(3) }
+      expect(File).to receive(:delete).with(file.path)
+      expect(file).to receive(:close)
 
-        it 'processes file with 3 lines line by line' do
-          allow(CSV).to receive(:foreach).and_yield('line0').and_yield('line1').and_yield('line2')
+      splitter.process(callback)
+    end
 
-          3.times do |i|
-            expect(splitter).to receive(:process_line).with("line#{i}", callback)
-          end
-          expect(splitter).to receive(:process_line).with(:eof, callback)
+    it 'aborts parsing the csv file if it is malformed' do
+      allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError.new('message', '1'))
+      expect(splitter).to receive(:process_line).with(:eof, callback)
 
-          expect(File).to receive(:delete).with(file.path)
+      splitter.process(callback)
+    end
 
-          splitter.process(callback)
-        end
+    it 'does not delete the resource file if :delete_resource is set to false in the context' do
+      allow(CSV).to receive(:foreach).and_yield(:eof)
+      expect(File).not_to receive(:delete).with(file.path)
 
-        it 'aborts parsing the csv file if it is malformed' do
-          allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
-          expect(splitter).to receive(:process_line).with(:eof, callback)
+      context[:delete_resource] = false
+      splitter.process(callback)
+    end
 
-          splitter.process(callback)
-        end
+    it 'stops splitting at :max_chunks if it is set in the context' do
+      allow(CSV).to receive(:foreach).and_yield(['line0']).and_yield(['line1']).and_yield(['line2'])
 
-        it 'does not delete the resource file if :delete_resource is set to false in the context' do
-          allow(CSV).to receive(:foreach).and_yield(:eof)
-          expect(File).not_to receive(:delete).with(file.path)
+      chunk_file = create_file(0)
 
-          context[:delete_resource] = false
-          splitter.process(callback)
-        end
-      end
+      allow(Tempfile).to receive(:new).and_return(chunk_file)
+      expect(callback).to receive(:run).once
+      allow_any_instance_of(StringIO).to receive(:path).and_return('test path')
 
-      describe '#process_line' do
-        [
-          {input: 1, chunks: %W(0\n)},
-          {input: 3, chunks: %W(0\n1\n 2\n)},
-          {input: 4, chunks: %W(0\n1\n 2\n3\n)}
-        ].each do |data|
-          it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
-            context[:chunk_size] = 2
-
-            data[:chunks].each do |chunk|
-              expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
-              expect(callback).to receive(:run).with(context) do
-                expect(context[:resource]).to eq(chunk_file)
-              end
-            end
-
-            data[:input].times { |i| splitter.process_line([i.to_s], callback) }
-            splitter.process_line(:eof, callback)
-          end
-        end
-      end
+      expect(File).to receive(:delete).with(file.path)
 
-      describe '#generate_chunk' do
-        it 'creates a file with the contents of the given string' do
-          file = splitter.generate_chunk('abc')
-          content = file.read
-          expect(content).to eq('abc')
-        end
+      context[:chunk_size] = 2
+      context[:max_chunks] = 1
+      splitter.process(callback)
+
+      expect(chunk_file.read).to eq("line0\nline1\n")
+    end
+
+    it 'ensures that the source file gets closed and deleted' do
+      allow(CSV).to receive(:foreach).and_raise(Interrupt)
+
+      expect(File).to receive(:delete).with(file.path)
+      expect(file).to receive(:close)
+
+      begin
+        splitter.process(callback)
+      rescue Interrupt
+        nil
+      end
+    end
+  end
 
-        it 'creates a file with a unique name' do
-          3.times do |i|
-            file = splitter.generate_chunk('content')
-            expect(file.path).to include("text_#{i + 1}_")
+  describe '#process_line' do
+    [
+        {input: 1, chunks: %W(0\n)},
+        {input: 3, chunks: %W(0\n1\n 2\n)},
+        {input: 4, chunks: %W(0\n1\n 2\n3\n)},
+    ].each do |data|
+      it "produces #{data[:chunks].size} chunks with #{data[:input]} input lines" do
+        context[:chunk_size] = 2
+
+        data[:chunks].each do |chunk|
+          expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
+          expect(callback).to receive(:run).with(context) do
+            expect(context[:resource]).to eq(chunk_file)
           end
         end
-      end
 
-      def create_file(lines)
-        content = ''
+        data[:input].times { |i| splitter.process_line([i.to_s], callback) }
+        splitter.process_line(:eof, callback)
+      end
+    end
+  end
 
-        lines.times do |i|
-          line = CSV.generate_line(["line#{i}"])
-          content << line
-        end
+  describe '#generate_chunk' do
+    it 'creates a file with the contents of the given string' do
+      file = splitter.generate_chunk('abc')
+      content = file.read
+      expect(content).to eq('abc')
+    end
 
-        StringIO.new(content)
+    it 'creates a file with a unique name' do
+      3.times do |i|
+        file = splitter.generate_chunk('content')
+        expect(file.path).to include("text_#{i + 1}_")
       end
     end
   end
+
+  def create_file(lines)
+    content = Array.new(lines) { |i| CSV.generate_line(["line#{i}"]) }.join
+    StringIO.new(content)
+  end
 end

data/spec/exel/providers/local_file_provider_spec.rb

@@ -1,36 +1,34 @@
-module EXEL
-  module Providers
-    describe LocalFileProvider do
-      let(:file) { instance_double(File, path: '/path/to/file') }
+# frozen_string_literal: true
 
-      describe '#upload' do
-        it 'returns a file:// URI for the file' do
-          expect(subject.upload(file)).to eq('file:///path/to/file')
-        end
-      end
+describe EXEL::Providers::LocalFileProvider do
+  let(:file) { File.open(File.expand_path('../../../fixtures/sample.csv', __FILE__)) }
 
-      describe '#download' do
-        it 'returns the file indicated by the URI' do
-          expect(File).to receive(:open).with('/path/to/file').and_return(file)
-          expect(subject.download('file:///path/to/file')).to eq(file)
-        end
+  it 'can upload/download a file' do
+    remote_value = subject.upload(file)
+    expect(remote_value.uri.path).to eq(file.path)
 
-        it 'doesn`t accept URIs for schemes other than file://' do
-          expect { subject.download('s3://') }.to raise_error 'URI must begin with "file://"'
-        end
-      end
+    restored_file = subject.download(remote_value)
+    expect(restored_file.path).to eq(file.path)
+  end
+
+  it 'doesn`t accept URIs for schemes other than file://' do
+    expect { subject.download(RemoteValue.new(URI('s3://bucket/file'))) }.to raise_error "Unsupported URI scheme 's3'"
+  end
 
-      describe '.remote?' do
-        it 'returns true for file:// URIs' do
-          expect(LocalFileProvider.remote?('file:///path/to/file')).to be_truthy
-        end
+  describe '.remote?' do
+    it 'returns true for remote values' do
+      expect(EXEL::Providers::LocalFileProvider.remote?(RemoteValue.new(URI('file:///path/to/file')))).to be_truthy
+    end
+
+    it 'returns false for file:// URIs' do
+      expect(EXEL::Providers::LocalFileProvider.remote?('file:///path/to/file')).to be_falsey
+      expect(EXEL::Providers::LocalFileProvider.remote?(URI('file:///path/to/file'))).to be_falsey
+    end
 
-        it 'returns false for anything else' do
-          expect(LocalFileProvider.remote?('s3://file')).to be_falsey
-          expect(LocalFileProvider.remote?(1)).to be_falsey
-          expect(LocalFileProvider.remote?(nil)).to be_falsey
-        end
-      end
+    it 'returns false for anything else' do
+      expect(EXEL::Providers::LocalFileProvider.remote?('s3://file')).to be_falsey
+      expect(EXEL::Providers::LocalFileProvider.remote?(1)).to be_falsey
+      expect(EXEL::Providers::LocalFileProvider.remote?(nil)).to be_falsey
     end
   end
 end

data/spec/exel/providers/threaded_async_provider_spec.rb

@@ -1,53 +1,52 @@
-module EXEL
-  module Providers
-    class ContextMutatingProcessor
-      def initialize(context)
-        @context = context
-      end
+# frozen_string_literal: true
 
-      def process(_block)
-        @context[:array] << @context[:arg]
-      end
-    end
+class ContextMutatingProcessor
+  def initialize(context)
+    @context = context
+  end
 
-    describe ThreadedAsyncProvider do
-      subject { described_class.new(context) }
-      let(:context) { EXEL::Context.new }
+  def process(_block)
+    @context[:array] << @context[:arg]
+  end
+end
 
-      describe '#do_async' do
-        let(:dsl_block) { instance_double(ASTNode) }
+describe EXEL::Providers::ThreadedAsyncProvider do
+  subject { described_class.new(context) }
 
-        it 'runs the block in a new thread' do
-          expect(dsl_block).to receive(:start).with(context)
-          expect(Thread).to receive(:new).and_yield
+  let(:context) { EXEL::Context.new }
 
-          subject.do_async(dsl_block)
-        end
+  describe '#do_async' do
+    let(:dsl_block) { instance_double(EXEL::ASTNode) }
 
-        it 'passes a copy of the context to each thread' do
-          context[:array] = []
-          complete = 0
+    it 'runs the block in a new thread' do
+      expect(dsl_block).to receive(:start).with(context)
+      expect(Thread).to receive(:new).and_yield
 
-          EXEL::Job.define :thread_test do
-            async do
-              process with: ContextMutatingProcessor, arg: 1
-              complete += 1
-            end
-
-            async do
-              process with: ContextMutatingProcessor, arg: 2
-              complete += 1
-            end
-          end
+      subject.do_async(dsl_block)
+    end
 
-          EXEL::Job.run(:thread_test, context)
+    it 'passes a copy of the context to each thread' do
+      context[:array] = []
+      complete = 0
 
-          start_time = Time.now
-          sleep 0.1 while complete < 2 && Time.now - start_time < 2
+      EXEL::Job.define :thread_test do
+        async do
+          process with: ContextMutatingProcessor, arg: 1
+          complete += 1
+        end
 
-          expect(context[:array]).to be_empty
+        async do
+          process with: ContextMutatingProcessor, arg: 2
+          complete += 1
         end
       end
+
+      EXEL::Job.run(:thread_test, context)
+
+      start_time = Time.now
+      sleep 0.1 while complete < 2 && Time.now - start_time < 2
+
+      expect(context[:array]).to be_empty
     end
   end
 end

data/spec/exel/sequence_node_spec.rb

@@ -1,17 +1,18 @@
-module EXEL
-  describe SequenceNode do
-    subject(:node) { described_class.new(instance_double(ASTNode), instance_double(ASTNode)) }
-    let(:context) { instance_double(EXEL::Context) }
+# frozen_string_literal: true
 
-    it { is_expected.to be_an(ASTNode) }
+describe EXEL::SequenceNode do
+  subject(:node) { described_class.new(instance_double(EXEL::ASTNode), instance_double(EXEL::ASTNode)) }
 
-    describe '#run' do
-      it 'runs each child node in sequence' do
-        expect(node.children.first).to receive(:run).with(context).once.ordered
-        expect(node.children.last).to receive(:run).with(context).once.ordered
+  let(:context) { instance_double(EXEL::Context) }
 
-        node.run(context)
-      end
+  it { is_expected.to be_an(EXEL::ASTNode) }
+
+  describe '#run' do
+    it 'runs each child node in sequence' do
+      expect(node.children.first).to receive(:run).with(context).once.ordered
+      expect(node.children.last).to receive(:run).with(context).once.ordered
+
+      node.run(context)
     end
   end
 end

data/spec/exel/value_spec.rb

@@ -1,50 +1,50 @@
-module EXEL
-  describe Value do
-    let(:uri) { 's3://test_file.csv' }
+# frozen_string_literal: true
 
-    before { allow(EXEL).to receive(:remote_provider).and_return(EXEL::Providers::DummyRemoteProvider) }
+describe EXEL::Value do
+  let(:uri) { 's3://test_file.csv' }
 
-    describe '.remotize' do
-      context 'when the value is not a file' do
-        it 'returns the value' do
-          expect(Value.remotize('test')).to eq('test')
-        end
+  before { allow(EXEL).to receive(:remote_provider).and_return(EXEL::Providers::DummyRemoteProvider) }
+
+  describe '.remotize' do
+    context 'when the value is not a file' do
+      it 'returns the value' do
+        expect(EXEL::Value.remotize('test')).to eq('test')
       end
+    end
 
-      [File, Tempfile].each do |file_class|
-        context "when the value is an instance of #{file_class}" do
-          let(:file) { instance_double(file_class) }
+    [File, Tempfile].each do |file_class|
+      context "when the value is an instance of #{file_class}" do
+        let(:file) { instance_double(file_class) }
 
-          before { allow(file).to receive(:is_a?) { |klass| klass == file_class } }
+        before { allow(file).to receive(:is_a?) { |klass| klass == file_class } }
 
-          it 'uploads the file using the remote provider' do
-            expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file)
-            Value.remotize(file)
-          end
+        it 'uploads the file using the remote provider' do
+          expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file)
+          EXEL::Value.remotize(file)
+        end
 
-          it 'returns the URI of the uploaded file' do
-            allow_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file).and_return(uri)
-            expect(Value.remotize(file)).to eq(uri)
-          end
+        it 'returns the URI of the uploaded file' do
+          allow_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:upload).with(file).and_return(uri)
+          expect(EXEL::Value.remotize(file)).to eq(uri)
         end
       end
     end
+  end
 
-    describe '.localize' do
-      context 'with a local value' do
-        it 'returns the value' do
-          expect(Value.localize('test')).to eq('test')
-        end
+  describe '.localize' do
+    context 'with a local value' do
+      it 'returns the value' do
+        expect(EXEL::Value.localize('test')).to eq('test')
       end
+    end
 
-      context 'with a remote file' do
-        it 'returns the downloaded file' do
-          expect(EXEL::Providers::DummyRemoteProvider).to receive(:remote?).with(uri).and_return(true)
-          file = double(:file)
-          expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:download).with(uri).and_return(file)
+    context 'with a remote file' do
+      it 'returns the downloaded file' do
+        expect(EXEL::Providers::DummyRemoteProvider).to receive(:remote?).with(uri).and_return(true)
+        file = double(:file)
+        expect_any_instance_of(EXEL::Providers::DummyRemoteProvider).to receive(:download).with(uri).and_return(file)
 
-          expect(Value.localize(uri)).to eq(file)
-        end
+        expect(EXEL::Value.localize(uri)).to eq(file)
       end
     end
   end