masamune 0.11.0

data/spec/masamune/commands/hadoop_filesystem_spec.rb

@@ -0,0 +1,50 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::HadoopFilesystem do
+  let(:configuration) { {options: options} }
+  let(:options) { [] }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  before do
+    allow(delegate).to receive_message_chain(:configuration, :hadoop_filesystem).and_return(configuration)
+  end
+
+  describe '#command_args' do
+    let(:attrs) { {extra: ['-ls', '/']} }
+
+    subject { instance.command_args }
+
+    it { is_expected.to eq(['hadoop', 'fs', '-ls', '/']) }
+
+    context 'with options' do
+      let(:options) { [{'--conf' => 'hadoop.conf'}] }
+
+      it { is_expected.to eq(['hadoop', 'fs', '--conf', 'hadoop.conf', '-ls', '/']) }
+    end
+  end
+end

data/spec/masamune/commands/hadoop_streaming_spec.rb

@@ -0,0 +1,106 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::HadoopStreaming do
+  let(:filesystem) { Masamune::MockFilesystem.new }
+
+  let(:configuration) { {options: options, input: input_option, output: 'output_dir', mapper: 'mapper.rb', reducer: 'reducer.rb', extra: extra} }
+  let(:options) { [] }
+  let(:input_option) { 'input.txt' }
+  let(:extra) { ['-D', %q(map.output.key.field.separator='\t')] }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  before do
+    allow(delegate).to receive(:filesystem).and_return(filesystem)
+    allow(delegate).to receive(:console).and_return(double)
+    allow(delegate).to receive(:logger).and_return(double)
+    allow(delegate).to receive_message_chain(:configuration, :hadoop_streaming).and_return(configuration)
+  end
+
+  describe '#before_execute' do
+    subject(:input) { instance.input }
+
+    context 'input path with suffix exists' do
+      let(:input_option) { 'dir/input.txt' }
+      before do
+        filesystem.touch!('dir/input.txt')
+        instance.before_execute
+      end
+      it { is_expected.to eq(['dir/input.txt']) }
+    end
+
+    context 'input path hadoop part' do
+      let(:input_option) { 'dir/part_0000' }
+      before do
+        filesystem.touch!('dir/part_0000')
+        instance.before_execute
+      end
+      it { is_expected.to eq(['dir/part_0000']) }
+    end
+
+    context 'input path directory' do
+      let(:input_option) { 'dir' }
+      before do
+        filesystem.touch!('dir')
+        instance.before_execute
+      end
+      it { is_expected.to eq(['dir/*']) }
+    end
+
+    context 'input path does not exist' do
+      before do
+        expect(instance.logger).to receive(:debug).with(/\ARemoving missing input/)
+        instance.before_execute
+      end
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe '#command_args' do
+    let(:pre_command_args) { ['hadoop', 'jar', described_class.default_hadoop_streaming_jar] }
+    let(:post_command_args) { ['-input', 'input.txt', '-mapper', 'mapper.rb', '-file', 'mapper.rb', '-reducer', 'reducer.rb', '-file', 'reducer.rb', '-output', 'output_dir'] }
+
+    subject { instance.command_args }
+
+    it { is_expected.to eq(pre_command_args + extra + post_command_args) }
+
+    context 'with options' do
+      let(:options) { [{'-cacheFile' => 'cache.rb'}] }
+
+      it { is_expected.to eq(pre_command_args + extra + options.map(&:to_a).flatten + post_command_args) }
+    end
+
+    context 'with quote' do
+      let(:attrs) { {quote: true} }
+      let(:quoted_extra) { ['-D', %q(map.output.key.field.separator='"'\\\\t'"')] }
+
+      subject { instance.command_args }
+
+      it { is_expected.to eq(pre_command_args + quoted_extra + post_command_args) }
+    end
+  end
+end

data/spec/masamune/commands/hive_spec.rb

@@ -0,0 +1,117 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::Hive do
+  let(:filesystem) { Masamune::MockFilesystem.new }
+  let(:configuration) { {:options => options} }
+  let(:options) { [] }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  let(:local_file) { File.join(Dir.tmpdir, SecureRandom.hex + '.txt') }
+  let(:mock_tmpdir) { 'mock' }
+  let(:remote_file) { filesystem.path(:tmp_dir, mock_tmpdir, File.basename(local_file)) }
+
+  before do
+    FileUtils.touch(local_file)
+    filesystem.add_path(:tmp_dir, File.join(Dir.tmpdir, SecureRandom.hex))
+    allow(filesystem).to receive(:mktempdir!) { filesystem.path(:tmp_dir, mock_tmpdir) }
+
+    allow(delegate).to receive(:filesystem) { filesystem }
+    allow(delegate).to receive_message_chain(:configuration, :hive).and_return(configuration)
+  end
+
+  describe '#stdin' do
+    context 'with exec' do
+      let(:attrs) { {exec: %q(SELECT * FROM table;)} }
+      subject(:stdin) { instance.stdin }
+
+      it { is_expected.to be_a(StringIO) }
+
+      describe '#string' do
+        subject { stdin.string }
+        it { is_expected.to eq(%q(SELECT * FROM table;)) }
+      end
+    end
+  end
+
+  describe '#command_args' do
+    let(:default_command) { ['hive', '--database', 'default'] }
+
+    subject do
+      instance.command_args
+    end
+
+    it { is_expected.to eq(default_command) }
+
+    context 'with command attrs' do
+      let(:options) { [{'-d' => 'DATABASE=development'}] }
+      it { is_expected.to eq([*default_command, '-d', 'DATABASE=development']) }
+    end
+
+    context 'with file' do
+      let(:attrs) { {file: local_file} }
+      it { is_expected.to eq([*default_command, '-f', remote_file]) }
+    end
+
+    context 'with variables' do
+      let(:attrs) { {file: local_file, variables: {R: 'R2DO', C: 'C3PO'}} }
+      it { is_expected.to eq([*default_command, '-f', remote_file, '-d', 'R=R2DO', '-d', 'C=C3PO']) }
+    end
+
+    context 'with setup files' do
+      before do
+        filesystem.touch!('setup_a.hql', 'setup_b.hql')
+      end
+
+      let(:attrs) { {setup_files: ['setup_a.hql', 'setup_b.hql']} }
+      it { is_expected.to eq([*default_command, '-i', 'setup_a.hql', '-i', 'setup_b.hql']) }
+    end
+
+    context 'with template file' do
+      let(:attrs) { {file: 'zomg.hql.erb'} }
+      before do
+        expect(Masamune::Template).to receive(:render_to_file).with('zomg.hql.erb', {}).and_return('XXX')
+        expect_any_instance_of(Masamune::MockFilesystem).to receive(:copy_file_to_file)
+      end
+      it { is_expected.to eq([*default_command, '-f', filesystem.get_path(:tmp_dir, mock_tmpdir, 'zomg.hql')]) }
+    end
+  end
+
+  describe '#handle_stdout' do
+    let(:buffer) { StringIO.new }
+    let(:delimiter) { "\t" }
+    let(:attrs) { {buffer: buffer, delimiter: delimiter, csv: true} }
+    let(:input_row) { ['A', 'NULL', 'B', 'C', '', 'E'].join(delimiter) }
+    let(:output_row) { ['A', nil, 'B', 'C', nil, 'E'].join(',') }
+
+    before do
+      instance.handle_stdout(input_row, 0)
+    end
+
+    it { expect(buffer.string).to eq(output_row + "\n") }
+  end
+end

data/spec/masamune/commands/postgres_admin_spec.rb

@@ -0,0 +1,69 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::PostgresAdmin do
+  let(:configuration) { {:create_db_path => 'createdb', :drop_db_path => 'dropdb', :hostname => 'localhost', :username => 'postgres'} }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  before do
+    allow(delegate).to receive_message_chain(:configuration, :postgres).and_return({})
+    allow(delegate).to receive_message_chain(:configuration, :postgres_admin).and_return(configuration)
+  end
+
+  describe '#command_args' do
+    subject do
+      instance.command_args
+    end
+
+    context 'action :create with database' do
+      let(:attrs) { {action: :create, database: 'zombo'} }
+      it { is_expected.to eq(['createdb', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
+    end
+
+    context 'action :create without database' do
+      let(:attrs) { {action: :create} }
+      it { expect { subject }.to raise_error ArgumentError, ':database must be given' }
+    end
+
+    context 'action :drop with database' do
+      let(:attrs) { {action: :drop, database: 'zombo'} }
+      it { is_expected.to eq(['dropdb', '--if-exists', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
+    end
+
+    context 'action :drop without database' do
+      let(:attrs) { {action: :drop} }
+      it { expect { subject }.to raise_error ArgumentError, ':database must be given' }
+    end
+
+    context 'action unfuddle with database' do
+      let(:attrs) { {action: :unfuddle, database: 'zombo'} }
+      it { expect { subject }.to raise_error ArgumentError, ':action must be :create or :drop' }
+    end
+  end
+
+  it_should_behave_like Masamune::Commands::PostgresCommon
+end

data/spec/masamune/commands/postgres_spec.rb

@@ -0,0 +1,100 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::Postgres do
+  let(:configuration) { {:path => 'psql', :database => 'postgres', :options => options} }
+  let(:options) { [] }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  before do
+    allow(delegate).to receive_message_chain(:configuration, :postgres).and_return(configuration)
+  end
+
+  describe '#stdin' do
+    context 'with input' do
+      let(:attrs) { {input: %q(SELECT * FROM table;)} }
+      subject(:stdin) { instance.stdin }
+
+      it { is_expected.to be_a(StringIO) }
+
+      describe '#string' do
+        subject { stdin.string }
+        it { is_expected.to eq(%q(SELECT * FROM table;)) }
+      end
+    end
+  end
+
+  describe '#command_args' do
+    let(:default_command) { ['psql', '--host=localhost', '--dbname=postgres', '--username=postgres', '--no-password'] }
+
+    subject do
+      instance.command_args
+    end
+
+    it { is_expected.to eq(default_command) }
+
+    context 'with options' do
+      let(:options) { [{'-A' => nil}] }
+      it { is_expected.to eq([*default_command, '-A']) }
+    end
+
+    context 'with file' do
+      let(:attrs) { {file: 'zomg.hql'} }
+      it { is_expected.to eq([*default_command, '--file=zomg.hql']) }
+    end
+
+    context 'with template file' do
+      let(:attrs) { {file: 'zomg.hql.erb'} }
+      before do
+        expect(Masamune::Template).to receive(:render_to_file).with('zomg.hql.erb', {}).and_return('zomg.hql')
+      end
+      it { is_expected.to eq([*default_command, '--file=zomg.hql']) }
+    end
+
+    context 'with variables and no file' do
+      let(:attrs) { {variables: {R: 'R2D2', C: 'C3PO'}} }
+      it { is_expected.to eq(default_command) }
+    end
+
+    context 'with variables and file' do
+      let(:attrs) { {file: 'zomg.hql', variables: {R: 'R2D2', C: 'C3PO'}} }
+      it { is_expected.to eq([*default_command, '--file=zomg.hql', %q(--set=R='R2D2'), %q(--set=C='C3PO')]) }
+    end
+
+    context 'with csv' do
+      let(:attrs) { {csv: true} }
+      it { is_expected.to eq([*default_command, '--no-align', '--field-separator=,', '--pset=footer']) }
+    end
+
+    context 'with tuple_output' do
+      let(:attrs) { {tuple_output: true} }
+      it { is_expected.to eq([*default_command, '--pset=tuples_only']) }
+    end
+  end
+
+  it_should_behave_like Masamune::Commands::PostgresCommon
+end

data/spec/masamune/commands/retry_with_backoff_spec.rb

@@ -0,0 +1,116 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::RetryWithBackoff do
+  let(:options) { {retries: retries, backoff: 0} }
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, options) }
+
+  before do
+    allow(delegate).to receive(:logger).and_return(double)
+    allow(delegate).to receive(:configuration).and_return(double(retries: 0, backoff: 0))
+  end
+
+  describe '#around_execute' do
+    let(:retries) { 3 }
+
+    context 'when retry command fails with status but eventually succeeds' do
+      before do
+        expect(instance.logger).to receive(:error).with('exited with code: 42').exactly(retries - 1)
+        expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(retries - 1)
+        subject
+      end
+
+      subject do
+        @retry_count = 0
+        instance.around_execute do
+          @retry_count += 1
+          if @retry_count < retries
+            OpenStruct.new(:success? => false, :exitstatus => 42)
+          else
+            OpenStruct.new(:success? => true)
+          end
+        end
+      end
+
+      it 'logs useful debug and error messages' do; end
+      it 'attempts to retry the specified number of times' do
+        expect(@retry_count).to eq(retries)
+      end
+      it 'returns result status' do
+        is_expected.to be_success
+      end
+    end
+
+    context 'when retry command fails with exception but eventually succeeds' do
+      before do
+        expect(instance.logger).to receive(:error).with('wtf').exactly(retries - 1)
+        expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(retries - 1)
+        subject
+      end
+
+      subject do
+        @retry_count = 0
+        instance.around_execute do
+          @retry_count += 1
+          raise 'wtf' if @retry_count < retries
+          OpenStruct.new(:success? => true)
+        end
+      end
+
+      it 'logs useful debug and error messages' do; end
+      it 'attempts to retry the specified number of times' do
+        expect(@retry_count).to eq(retries)
+      end
+      it 'returns result status' do
+        is_expected.to be_success
+      end
+    end
+
+    context 'when retry command eventually fails' do
+      before do
+        expect(instance.logger).to receive(:error).with('wtf').exactly(retries + 1)
+        expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(retries)
+        expect(instance.logger).to receive(:debug).with(/max retries.*bailing/)
+        subject
+      end
+
+      subject do
+        @retry_count = 0
+        instance.around_execute do
+          @retry_count += 1
+          raise 'wtf'
+        end
+      end
+
+      it 'logs useful debug and error messages' do; end
+      it 'attempts to retry the specified number of times' do
+        expect(@retry_count).to eq(retries + 1)
+      end
+      it 'returns failure status' do
+        is_expected.not_to be_success
+      end
+    end
+  end
+end