masamune 0.11.0

data/spec/fixtures/simple.sql.erb

@@ -0,0 +1,28 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+
+
+
+SELECT * FROM <%= table %>;
+
+

data/spec/fixtures/whitespace.sql.erb

@@ -0,0 +1,30 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+
+
+SELECT 1;
+
+
+
+
+SELECT 2;

data/spec/masamune/actions/elastic_mapreduce_spec.rb

@@ -0,0 +1,108 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Actions::ElasticMapreduce do
+  let(:klass) do
+    Class.new do
+      include Masamune::HasEnvironment
+      include Masamune::AfterInitializeCallbacks
+      include Masamune::Actions::ElasticMapreduce
+    end
+  end
+
+  let(:instance) { klass.new }
+  let(:configuration) { {} }
+  let(:extra) { [] }
+
+  before do
+    allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return(configuration)
+    allow(instance).to receive(:extra).and_return(extra)
+  end
+
+  describe '.elastic_mapreduce' do
+    before do
+      mock_command(/\Aelastic-mapreduce/, mock_success)
+    end
+
+    subject { instance.elastic_mapreduce }
+
+    it { is_expected.to be_success }
+  end
+
+  describe '.after_initialize' do
+    let(:options) { {} }
+
+    subject(:after_initialize_invoke) do
+      instance.after_initialize_invoke(options)
+    end
+
+    context 'when configuration is empty' do
+      it { expect { subject }.to_not raise_error }
+    end
+
+    context 'when jobflow not required due to extra options' do
+      let(:configuration) { {enabled: true} }
+      let(:extra) { ['--create', '--name', 'zombo_cluster'] }
+      it { expect { subject }.to_not raise_error }
+    end
+
+    context 'when jobflow is missing' do
+      let(:configuration) { {enabled: true} }
+      it { expect { subject }.to raise_error Thor::RequiredArgumentMissingError, /No value provided for required options '--jobflow'/ }
+    end
+
+    context 'when jobflow does not exist' do
+      let(:configuration) { {enabled: true} }
+      let(:options) { {jobflow: 'j-XYZ'} }
+      before do
+        mock_command(/\Aelastic-mapreduce/, mock_failure)
+      end
+      it { expect { subject }.to raise_error Thor::RequiredArgumentMissingError, /Value 'j-XYZ' for '--jobflow' doesn't exist/ }
+    end
+
+    context 'when jobflow exists' do
+      let(:configuration) { {enabled: true} }
+      let(:options) { {jobflow: 'j-XYZ'} }
+      before do
+        mock_command(/\Aelastic-mapreduce/, mock_success)
+      end
+      it do
+        expect { subject }.to_not raise_error
+        expect(configuration[:jobflow]).to eq('j-XYZ')
+      end
+    end
+
+    context 'when jobflow is symbolic' do
+      let(:configuration) { {enabled: true, jobflows: {'build' => 'j-XYZ'}} }
+      let(:options) { {jobflow: 'build', } }
+      before do
+        mock_command(/\Aelastic-mapreduce/, mock_success)
+      end
+      it do
+        expect { subject }.to_not raise_error
+        expect(configuration[:jobflow]).to eq('j-XYZ')
+      end
+    end
+  end
+end

data/spec/masamune/actions/execute_spec.rb

@@ -0,0 +1,50 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Actions::Execute do
+  let(:klass) do
+    Class.new do
+      include Masamune::HasEnvironment
+      include Masamune::Actions::Execute
+    end
+  end
+
+  let(:instance) { klass.new }
+
+  describe '.execute' do
+    context 'with a simple command' do
+      let(:command) { %w(echo ping) }
+      let(:options) { {fail_fast: true} }
+
+      it { expect { |b| instance.execute(*command, options, &b) }.to yield_with_args('ping', 0) }
+    end
+
+    context 'with a simple command with input' do
+      let(:command) { %w(cat) }
+      let(:options) { {input: 'pong', fail_fast: true} }
+
+      it { expect { |b| instance.execute(*command, options, &b) }.to yield_with_args('pong', 0) }
+    end
+  end
+end

data/spec/masamune/actions/hadoop_filesystem_spec.rb

@@ -0,0 +1,44 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Actions::HadoopFilesystem do
+  let(:klass) do
+    Class.new do
+      include Masamune::HasEnvironment
+      include Masamune::Actions::HadoopFilesystem
+    end
+  end
+
+  let(:instance) { klass.new }
+
+  describe '.hadoop_filesystem' do
+    before do
+      mock_command(/\Ahadoop/, mock_success)
+    end
+
+    subject { instance.hadoop_filesystem }
+
+    it { is_expected.to be_success }
+  end
+end

data/spec/masamune/actions/hadoop_streaming_spec.rb

@@ -0,0 +1,74 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Actions::HadoopStreaming do
+  let(:klass) do
+    Class.new do
+      include Masamune::HasEnvironment
+      include Masamune::Actions::HadoopStreaming
+    end
+  end
+
+  let(:extra) { [] }
+  let(:instance) { klass.new }
+
+  before do
+    instance.environment = Masamune::ExampleGroup
+  end
+
+  describe '.hadoop_streaming' do
+    before do
+      mock_command(/\Ahadoop/, mock_success)
+    end
+
+    subject { instance.hadoop_streaming(extra: extra) }
+
+    it { is_expected.to be_success }
+
+    context 'with jobflow' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({jobflow: 'j-XYZ'})
+        mock_command(/\Ahadoop/, mock_failure)
+        mock_command(/\Aelastic-mapreduce/, mock_success, StringIO.new('ssh fakehost exit'))
+        mock_command(/\Assh fakehost hadoop/, mock_success)
+      end
+
+      it { is_expected.to be_success }
+    end
+
+    context 'with jobflow and extra' do
+      let(:extra) { ['-D', 'EXTRA'] }
+
+      before do
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({jobflow: 'j-XYZ'})
+        mock_command(/\Ahadoop/, mock_failure)
+        mock_command(/\Aelastic-mapreduce/, mock_success, StringIO.new('ssh fakehost exit'))
+        mock_command(/\Assh fakehost -D EXTRA hadoop/, mock_failure)
+        mock_command(/\Assh fakehost hadoop .*? -D EXTRA/, mock_success)
+      end
+
+      it { is_expected.to be_success }
+    end
+  end
+end

data/spec/masamune/actions/hive_spec.rb

@@ -0,0 +1,117 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Actions::Hive do
+  let(:klass) do
+    Class.new do
+      include Masamune::HasEnvironment
+      include Masamune::AfterInitializeCallbacks
+      include Masamune::Actions::Hive
+    end
+  end
+
+  let(:filesystem) { Masamune::MockFilesystem.new }
+  let(:instance) { klass.new }
+  let(:configuration) { {database: 'test'} }
+
+  before do
+    filesystem.add_path(:tmp_dir, File.join(Dir.tmpdir, SecureRandom.hex))
+    allow(instance).to receive(:filesystem) { filesystem }
+    allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({})
+    allow(instance).to receive_message_chain(:configuration, :hive).and_return(configuration)
+    allow(instance).to receive_message_chain(:define_schema, :to_file) { 'schema.hql' }
+    allow_any_instance_of(Masamune::MockFilesystem).to receive(:copy_file_to_dir)
+  end
+
+  describe '.hive' do
+    before do
+      mock_command(/\Ahive/, mock_success)
+    end
+
+    subject { instance.hive }
+
+    it { is_expected.to be_success }
+
+    context 'with jobflow' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({jobflow: 'j-XYZ'})
+        mock_command(/\Ahive/, mock_failure)
+        mock_command(/\Aelastic-mapreduce/, mock_success, StringIO.new('ssh fakehost exit'))
+        mock_command(/\Assh fakehost hive/, mock_success)
+      end
+
+      subject { instance.hive }
+
+      it { is_expected.to be_success }
+    end
+  end
+
+  describe '.after_initialize' do
+    let(:options) { {} }
+    let(:configuration) { {database: 'test'} }
+
+    subject(:after_initialize_invoke) do
+      instance.after_initialize_invoke(options)
+    end
+
+    context 'with default database' do
+      let(:configuration) { {database: 'default'} }
+      before do
+        expect(instance).to receive(:hive).with(exec: an_instance_of(String)).never
+        after_initialize_invoke
+      end
+      it 'should not call hive with create database' do; end
+    end
+
+    context 'with database' do
+      before do
+        expect(instance).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS test;', :database => nil).once.and_return(mock_success)
+        expect(instance).to receive(:hive).with(file: an_instance_of(String)).once.and_return(mock_success)
+        after_initialize_invoke
+      end
+      it 'should call hive with create database' do; end
+    end
+
+    context 'with location' do
+      let(:configuration) { {database: 'test', location: '/tmp'} }
+      before do
+        expect(instance).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS test LOCATION "/tmp";', :database => nil).once.and_return(mock_success)
+        expect(instance).to receive(:hive).with(file: an_instance_of(String)).once.and_return(mock_success)
+        after_initialize_invoke
+      end
+      it 'should call hive with create database' do; end
+    end
+
+    context 'with dryrun' do
+      let(:options) { {dry_run: true} }
+      before do
+        expect(instance).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS test;', :database => nil).once.and_return(mock_success)
+        expect(instance).to receive(:hive).with(exec: 'SHOW TABLES;', safe: true, fail_fast: false).once.and_return(mock_success)
+        expect(instance).to receive(:hive).with(file: an_instance_of(String)).once.and_return(mock_success)
+        after_initialize_invoke
+      end
+      it 'should call hive with show tables' do; end
+    end
+  end
+end