masamune 0.17.15 → 0.18.0

data/lib/masamune/environment.rb

@@ -45,7 +45,7 @@ module Masamune
     end
 
     def configuration
-      @configuration ||= Masamune::Configuration.new(self)
+      @configuration ||= Masamune::Configuration.new(environment: self)
     end
 
     def mutex
@@ -60,13 +60,13 @@ module Masamune
       lock_mode = File::LOCK_EX
       lock_mode |= File::LOCK_NB
       lock_status = lock_file.flock(lock_mode)
-      if lock_status == 0
+      if lock_status && lock_status.zero?
         yield if block_given?
       else
         logger.error "acquire lock attempt failed for '#{lock_name}'"
       end
     ensure
-      if lock_file && lock_status == 0
+      if lock_file && lock_status && lock_status.zero?
         logger.debug("releasing lock '#{lock_name}'")
         lock_file.flock(File::LOCK_UN)
       end
@@ -92,6 +92,7 @@ module Masamune
     end
 
     def log_file_name
+      return unless filesystem.path?(:log_dir)
       @log_file_name ||= filesystem.get_path(:log_dir, log_file_template)
     end
 

data/lib/masamune/method_logger.rb

@@ -37,5 +37,9 @@ module Masamune
         super
       end
     end
+
+    def respond_to_missing?(method_name, include_private = false)
+      @target.respond_to?(method_name) || super
+    end
   end
 end

data/lib/masamune/schema/store.rb

@@ -72,13 +72,25 @@ module Masamune::Schema
       @extra      = []
     end
 
-    def method_missing(method, *_args)
+    def method_missing(method_name, *_args)
+      *attribute_name, attribute_type = method_name.to_s.split('_')
       if type == :files
-        files[method]
-      else
-        *attribute_name, attribute_type = method.to_s.split('_')
-        raise ArgumentError, "unknown attribute type '#{attribute_type}'" unless SUPPORTED_ATTRIBUTES.include?(attribute_type)
+        files[method_name]
+      elsif SUPPORTED_ATTRIBUTES.include?(attribute_type)
         send(attribute_type.pluralize)[attribute_name.join('_')]
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, _include_private = false)
+      *attribute_name, attribute_type = method_name.to_s.split('_')
+      if type == :files
+        files.key?(method_name)
+      elsif SUPPORTED_ATTRIBUTES.include?(attribute_type)
+        send(attribute_type.pluralize).key?(attribute_name.join('_'))
+      else
+        super
       end
     end
 

data/lib/masamune/thor.rb

@@ -78,7 +78,7 @@ module Masamune
       rescue => e
         instance.logger.error("#{e.message} (#{e.class}) backtrace:")
         e.backtrace.each { |x| instance.logger.error(x) }
-        $stderr.puts "For complete debug log see: #{instance.log_file_name}"
+        $stderr.puts "For complete debug log see: #{instance.log_file_name}" if instance.log_file_name
         abort e.message
       end
     end
@@ -146,8 +146,6 @@ module Masamune
 
             if options[:version]
               puts environment.version
-              puts options if options[:verbose]
-              puts config.to_s if options[:verbose]
               exit
             end
           end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.17.15'.freeze
+  VERSION = '0.18.0'.freeze
 end

data/spec/masamune/actions/aws_emr_spec.rb

@@ -33,7 +33,7 @@ describe Masamune::Actions::AwsEmr do
   let(:configuration) { {} }
 
   before do
-    allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return(configuration)
+    allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(configuration)
   end
 
   describe '.aws_emr' do
@@ -47,7 +47,7 @@ describe Masamune::Actions::AwsEmr do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 
@@ -79,7 +79,7 @@ describe Masamune::Actions::AwsEmr do
       end
       it do
         expect { subject }.to_not raise_error
-        expect(instance.configuration.aws_emr[:cluster_id]).to eq('j-XYZ')
+        expect(instance.configuration.commands.aws_emr[:cluster_id]).to eq('j-XYZ')
       end
     end
 
@@ -100,7 +100,7 @@ describe Masamune::Actions::AwsEmr do
       end
       it do
         expect { subject }.to_not raise_error
-        expect(instance.configuration.aws_emr[:cluster_id]).to eq('j-XYZ')
+        expect(instance.configuration.commands.aws_emr[:cluster_id]).to eq('j-XYZ')
       end
     end
   end

data/spec/masamune/actions/hadoop_filesystem_spec.rb

@@ -41,7 +41,7 @@ describe Masamune::Actions::HadoopFilesystem do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :hadoop_filesystem).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_filesystem).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 

data/spec/masamune/actions/hadoop_streaming_spec.rb

@@ -33,8 +33,8 @@ describe Masamune::Actions::HadoopStreaming do
 
   describe '.hadoop_streaming' do
     before do
-      allow(instance).to receive_message_chain(:configuration, :hadoop_streaming).and_return({})
-      allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return({})
+      allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return({})
+      allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return({})
       mock_command(/\Ahadoop/, mock_success)
     end
 
@@ -44,7 +44,7 @@ describe Masamune::Actions::HadoopStreaming do
 
     context 'with cluster_id' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return(cluster_id: 'j-XYZ')
+        allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
         mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost hadoop/, mock_success)
@@ -57,7 +57,7 @@ describe Masamune::Actions::HadoopStreaming do
       let(:extra) { ['-D', 'EXTRA'] }
 
       before do
-        allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return(cluster_id: 'j-XYZ')
+        allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
         mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost -D EXTRA hadoop/, mock_failure)
@@ -69,7 +69,7 @@ describe Masamune::Actions::HadoopStreaming do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :hadoop_streaming).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 

data/spec/masamune/actions/hive_spec.rb

@@ -36,8 +36,8 @@ describe Masamune::Actions::Hive do
   before do
     filesystem.add_path(:tmp_dir, File.join(Dir.tmpdir, SecureRandom.hex))
     allow(instance).to receive(:filesystem) { filesystem }
-    allow(instance).to receive_message_chain(:configuration, :hive).and_return(configuration)
-    allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return({})
+    allow(instance).to receive_message_chain(:configuration, :commands, :hive).and_return(configuration)
+    allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return({})
     allow(instance).to receive_message_chain(:define_schema, :to_file) { 'schema.hql' }
     allow_any_instance_of(Masamune::MockFilesystem).to receive(:copy_file_to_dir)
   end
@@ -53,7 +53,7 @@ describe Masamune::Actions::Hive do
 
     context 'with cluster_id' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :aws_emr).and_return(cluster_id: 'j-XYZ')
+        allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
         mock_command(/\Ahive/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost hive/, mock_success)
@@ -66,7 +66,7 @@ describe Masamune::Actions::Hive do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :hive).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :hive).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 

data/spec/masamune/actions/postgres_spec.rb

@@ -40,7 +40,7 @@ describe Masamune::Actions::Postgres do
     allow(instance).to receive(:filesystem) { filesystem }
     allow(instance).to receive(:catalog) { catalog }
     allow(instance).to receive(:postgres_helper) { postgres_helper }
-    allow(instance).to receive_message_chain(:configuration, :postgres).and_return(configuration)
+    allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(configuration)
     allow(instance).to receive_message_chain(:configuration, :with_quiet).and_yield
     allow(instance).to receive_message_chain(:define_schema, :to_file) { 'catalog.psql' }
   end
@@ -56,7 +56,7 @@ describe Masamune::Actions::Postgres do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :postgres).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 

data/spec/masamune/actions/s3cmd_spec.rb

@@ -41,7 +41,7 @@ describe Masamune::Actions::S3Cmd do
 
     context 'with retries and backoff' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :s3cmd).and_return(retries: 1, backoff: 10)
+        allow(instance).to receive_message_chain(:configuration, :commands, :s3cmd).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
       end
 

data/spec/masamune/commands/aws_emr_spec.rb

@@ -29,7 +29,7 @@ describe Masamune::Commands::AwsEmr do
 
   before do
     allow(delegate).to receive(:logger).and_return(double)
-    allow(delegate).to receive_message_chain(:configuration, :aws_emr).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(configuration)
   end
 
   describe '#command_env' do

data/spec/masamune/commands/hadoop_filesystem_spec.rb

@@ -29,7 +29,7 @@ describe Masamune::Commands::HadoopFilesystem do
   let(:instance) { described_class.new(delegate, attrs) }
 
   before do
-    allow(delegate).to receive_message_chain(:configuration, :hadoop_filesystem).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :hadoop_filesystem).and_return(configuration)
   end
 
   describe '#command_args' do

data/spec/masamune/commands/hadoop_streaming_spec.rb

@@ -36,7 +36,7 @@ describe Masamune::Commands::HadoopStreaming do
     allow(delegate).to receive(:filesystem).and_return(filesystem)
     allow(delegate).to receive(:console).and_return(double)
     allow(delegate).to receive(:logger).and_return(double)
-    allow(delegate).to receive_message_chain(:configuration, :hadoop_streaming).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return(configuration)
   end
 
   describe '#before_execute' do

data/spec/masamune/commands/hive_spec.rb

@@ -41,7 +41,7 @@ describe Masamune::Commands::Hive do
     allow(delegate).to receive(:filesystem) { filesystem }
     allow(delegate).to receive(:logger).and_return(double)
     allow(delegate).to receive(:console).and_return(double)
-    allow(delegate).to receive_message_chain(:configuration, :hive).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :hive).and_return(configuration)
   end
 
   describe '#command_args' do

data/spec/masamune/commands/postgres_admin_spec.rb

@@ -22,14 +22,15 @@
 
 describe Masamune::Commands::PostgresAdmin do
   let(:configuration) { { create_db_path: 'createdb', drop_db_path: 'dropdb', hostname: 'localhost', username: 'postgres' } }
+  let(:postgres_configuration) { {} }
   let(:attrs) { {} }
 
   let(:delegate) { double }
   let(:instance) { described_class.new(delegate, attrs) }
 
   before do
-    allow(delegate).to receive_message_chain(:configuration, :postgres).and_return({})
-    allow(delegate).to receive_message_chain(:configuration, :postgres_admin).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :postgres).and_return(postgres_configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :postgres_admin).and_return(configuration)
   end
 
   describe '#command_args' do
@@ -42,6 +43,18 @@ describe Masamune::Commands::PostgresAdmin do
       it { is_expected.to eq(['createdb', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
     end
 
+    context 'action :create with database with postgres database configuration (string)' do
+      let(:postgres_configuration) { { 'database' => 'test' } }
+      let(:attrs) { { action: :create, database: 'zombo' } }
+      it { is_expected.to eq(['createdb', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
+    end
+
+    context 'action :create with database with postgres database configuration (symbol)' do
+      let(:postgres_configuration) { { database: 'test' } }
+      let(:attrs) { { action: :create, database: 'zombo' } }
+      it { is_expected.to eq(['createdb', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
+    end
+
     context 'action :create without database' do
       let(:attrs) { { action: :create } }
       it { expect { subject }.to raise_error ArgumentError, ':database must be given' }

data/spec/masamune/commands/postgres_spec.rb

@@ -31,7 +31,7 @@ describe Masamune::Commands::Postgres do
   before do
     allow(delegate).to receive(:logger).and_return(double)
     allow(delegate).to receive(:console).and_return(double)
-    allow(delegate).to receive_message_chain(:configuration, :postgres).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :postgres).and_return(configuration)
   end
 
   describe '#stdin' do

data/spec/masamune/commands/s3cmd_spec.rb

@@ -29,7 +29,7 @@ describe Masamune::Commands::S3Cmd do
   let(:instance) { described_class.new(delegate, attrs) }
 
   before do
-    allow(delegate).to receive_message_chain(:configuration, :s3cmd).and_return(configuration)
+    allow(delegate).to receive_message_chain(:configuration, :commands, :s3cmd).and_return(configuration)
   end
 
   describe '#command_args' do

data/spec/masamune/configuration_spec.rb

@@ -22,7 +22,7 @@
 
 describe Masamune::Configuration do
   let(:environment) { Masamune::Environment.new }
-  let(:instance) { described_class.new(environment) }
+  let(:instance) { described_class.new(environment: environment) }
 
   describe '.default_config_file' do
     subject { described_class.default_config_file }
@@ -34,6 +34,23 @@ describe Masamune::Configuration do
     it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
   end
 
+  describe '#params' do
+    subject { instance.params }
+    it { is_expected.to be_a(Hash) }
+  end
+
+  describe '#commands' do
+    subject { instance.commands }
+    it { is_expected.to be_a(Hashie::Mash) }
+  end
+
+  described_class.default_commands.each do |command_sym|
+    describe "#commands.#{command_sym}" do
+      subject { instance.commands.send(command_sym) }
+      it { is_expected.to be_a(Hashie::Mash) }
+    end
+  end
+
   describe '#as_options' do
     subject { instance.as_options }
     it { is_expected.to eq([]) }
@@ -45,4 +62,82 @@ describe Masamune::Configuration do
       it { is_expected.to eq(['--debug', '--dry-run']) }
     end
   end
+
+  describe '#load' do
+    let(:yaml_file) do
+      Tempfile.create('masamune').tap do |tmp|
+        tmp.write(yaml)
+        tmp.close
+      end.path
+    end
+
+    subject(:result) { instance.load(yaml_file) }
+
+    context 'with Hash params' do
+      let(:yaml) do
+        <<-YAML.strip_heredoc
+        ---
+          params:
+            key_one: value_one
+            key_two: value_two
+        YAML
+      end
+
+      it do
+        expect(result.params[:key_one]).to eq('value_one')
+        expect(result.params[:key_two]).to eq('value_two')
+      end
+    end
+
+    context 'with Array params' do
+      let(:yaml) do
+        <<-YAML.strip_heredoc
+        ---
+          params:
+            - one
+            - two
+        YAML
+      end
+
+      it do
+        expect { result }.to raise_error(ArgumentError, 'params section must only contain key value pairs')
+      end
+    end
+
+    context 'with Hash paths' do
+      let(:yaml) do
+        <<-YAML.strip_heredoc
+        ---
+          paths:
+            - foo_dir: ['/tmp/foo', {mkdir: true}]
+            - bar_dir: '/tmp/bar'
+        YAML
+      end
+
+      it do
+        expect(result.filesystem.paths[:foo_dir]).to eq(['/tmp/foo', { mkdir: true }])
+        expect(result.filesystem.paths[:bar_dir]).to eq(['/tmp/bar', {}])
+      end
+    end
+
+    context 'with Hash commands' do
+      let(:yaml) do
+        <<-YAML.strip_heredoc
+        ---
+          commands:
+            aws_emr:
+              path: /opt/aws/bin/emr
+              config_file: /etc/aws/emr_config
+            hive:
+              database: 'zombo'
+        YAML
+      end
+
+      it do
+        expect(result.commands.aws_emr.path).to eq('/opt/aws/bin/emr')
+        expect(result.commands.aws_emr.config_file).to eq('/etc/aws/emr_config')
+        expect(result.commands.hive.database).to eq('zombo')
+      end
+    end
+  end
 end