masamune 0.18.1 → 0.18.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00d8f2eadd5197f9d6bc29ea6ad781184a0e6cc0
-  data.tar.gz: 2916c960c93dba409c0f8cf5f9dc0aab40362c93
+  metadata.gz: fb31759e76d58c026aeee2bcf241611e9821da4f
+  data.tar.gz: fb82f3c5215527397b742bd0b8108f9f8d58b4c5
 SHA512:
-  metadata.gz: 1f214ac8bc7f480c60308ca49c856df221f344b77231cff5b0983ad9b55079c4c3c1ccf8bc2179355b87c2c1626641a76a6138e95489cd858f7e7eb6b919f187
-  data.tar.gz: b63e53edf5bcd55c059ad38f302baec66c6f0ab7595e216a978888d31c4532cf694e1595f752b55a5d224264cb25d7e507d4f46c1db204a0373d447e70d10883
+  metadata.gz: a8c29df50f07a4d7bbc56b66940bbb1b4352ad8d2489740741cabe056803d196375c75c217d28be4d0feabf257f3a1f84788594163467b36885ae6565aa4e1b2
+  data.tar.gz: 4e0b4decd60fe42953ccd0ac92c7c4077137e4c85db42e238893e769b86c5f159a94f505b13cc0bc681a20d0b819562afb9c121e6f435e06e9010667c51b9c7b

data/lib/masamune/actions/invoke_parallel.rb

@@ -35,13 +35,16 @@ module Masamune::Actions
     end
 
     def invoke_parallel(*task_group)
-      opts = task_group.last.is_a?(Hash) ? task_group.pop.dup : {}
-      max_tasks = [opts.delete(:max_tasks), task_group.count].min
+      per_task_opts = task_group.last.is_a?(Array) ? task_group.pop : [{}]
+      all_task_opts = task_group.last.is_a?(Hash) ? task_group.pop : {}
+      max_tasks = [all_task_opts.delete(:max_tasks), task_group.count].min
       console("Setting max_tasks to #{max_tasks}")
-      bail_fast task_group, opts if opts[:version]
-      Parallel.each(task_group, in_processes: max_tasks) do |task_name|
+      bail_fast task_group, all_task_opts if all_task_opts[:version]
+      task_group_by_task_opts = task_group.product(per_task_opts)
+      Parallel.each(task_group_by_task_opts, in_processes: max_tasks) do |task_name, task_opts|
+        task_env = task_opts[:env] || {}
         begin
-          execute(thor_wrapper, task_name, *task_args(opts), interactive: true, detach: false)
+          execute(thor_wrapper, task_name, *task_args(all_task_opts.merge(task_opts)), interactive: true, detach: false, env: task_env)
         rescue SystemExit # rubocop:disable Lint/HandleExceptions
         end
       end
@@ -55,12 +58,13 @@ module Masamune::Actions
 
     def bail_fast(task_group, opts = {})
       task_name = task_group.first
-      execute($PROGRAM_NAME, task_name, *task_args(opts))
+      task_env = task_opts[:env] || {}
+      execute($PROGRAM_NAME, task_name, *task_args(opts), env: task_env)
       exit
     end
 
     def task_args(opts = {})
-      opts.map do |k, v|
+      opts.except(:env).map do |k, v|
         case v
         when true
           "--#{k.to_s.tr('_', '-')}"

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.18.1'.freeze
+  VERSION = '0.18.2'.freeze
 end

data/spec/masamune/actions/aws_emr_spec.rb

@@ -37,18 +37,29 @@ describe Masamune::Actions::AwsEmr do
   end
 
   describe '.aws_emr' do
-    before do
-      mock_command(/\Aaws emr/, mock_success)
+    subject(:action) { instance.aws_emr }
+
+    context 'when success' do
+      before do
+        mock_command(/\Aaws emr/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.aws_emr }
+    context 'when failure' do
+      before do
+        mock_command(/\Aaws emr/, mock_failure)
+      end
 
-    it { is_expected.to be_success }
+      it { expect { action }.to raise_error RuntimeError, 'fail_fast: aws emr ssh' }
+    end
 
     context 'with retries and backoff' do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+        mock_command(/\Aaws emr/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/hadoop_filesystem_spec.rb

@@ -31,18 +31,29 @@ describe Masamune::Actions::HadoopFilesystem do
   let(:instance) { klass.new }
 
   describe '.hadoop_filesystem' do
-    before do
-      mock_command(/\Ahadoop/, mock_success)
+    subject(:action) { instance.hadoop_filesystem }
+
+    context 'when success' do
+      before do
+        mock_command(/\Ahadoop/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.hadoop_filesystem }
+    context 'when failure' do
+      before do
+        mock_command(/\Ahadoop/, mock_failure)
+      end
 
-    it { is_expected.to be_success }
+      it { is_expected.not_to be_success }
+    end
 
     context 'with retries and backoff' do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_filesystem).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+        mock_command(/\Ahadoop/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/hadoop_streaming_spec.rb

@@ -31,21 +31,33 @@ describe Masamune::Actions::HadoopStreaming do
   let(:extra) { [] }
   let(:instance) { klass.new }
 
+  before do
+    allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return({})
+    allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return({})
+  end
+
   describe '.hadoop_streaming' do
-    before do
-      allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return({})
-      allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return({})
-      mock_command(/\Ahadoop/, mock_success)
+    subject(:action) { instance.hadoop_streaming(extra: extra) }
+
+    context 'when success' do
+      before do
+        mock_command(/\Ahadoop/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.hadoop_streaming(extra: extra) }
+    context 'when failure' do
+      before do
+        mock_command(/\Ahadoop/, mock_failure)
+      end
 
-    it { is_expected.to be_success }
+      it { expect { action }.to raise_error RuntimeError, /\Afail_fast: hadoop/ }
+    end
 
     context 'with cluster_id' do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
-        mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost hadoop/, mock_success)
       end
@@ -58,9 +70,7 @@ describe Masamune::Actions::HadoopStreaming do
 
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
-        mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
-        mock_command(/\Assh fakehost -D EXTRA hadoop/, mock_failure)
         mock_command(/\Assh fakehost hadoop .*? -D EXTRA/, mock_success)
       end
 
@@ -71,6 +81,7 @@ describe Masamune::Actions::HadoopStreaming do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+        mock_command(/\Ahadoop/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/hive_spec.rb

@@ -43,18 +43,27 @@ describe Masamune::Actions::Hive do
   end
 
   describe '.hive' do
-    before do
-      mock_command(/\Ahive/, mock_success)
+    subject(:action) { instance.hive }
+
+    context 'when success' do
+      before do
+        mock_command(/\Ahive/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.hive }
+    context 'when failure' do
+      before do
+        mock_command(/\Ahive/, mock_failure)
+      end
 
-    it { is_expected.to be_success }
+      it { is_expected.not_to be_success }
+    end
 
     context 'with cluster_id' do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(cluster_id: 'j-XYZ')
-        mock_command(/\Ahive/, mock_failure)
         mock_command(/\Aaws emr/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost hive/, mock_success)
       end
@@ -68,6 +77,7 @@ describe Masamune::Actions::Hive do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :hive).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+        mock_command(/\Ahive/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/invoke_parallel_spec.rb

@@ -31,13 +31,54 @@ describe Masamune::Actions::InvokeParallel do
   let(:instance) { klass.new }
 
   describe '.invoke_parallel' do
-    context 'with a simple thor command' do
+    context 'with a single thor command' do
       before do
         mock_command(/\Athor list/, mock_success)
       end
 
       subject do
-        instance.invoke_parallel('list', max_tasks: 1)
+        instance.invoke_parallel('list', max_tasks: 0)
+      end
+
+      it { expect { subject }.to_not raise_error }
+    end
+
+    context 'with a single thor command and multiple arguments' do
+      before do
+        mock_command(/\Athor list --a/, mock_success)
+        mock_command(/\Athor list --b/, mock_success)
+      end
+
+      subject do
+        instance.invoke_parallel('list', { max_tasks: 0 }, [{ a: true, b: false }, { a: false, b: true }])
+      end
+
+      it { expect { subject }.to_not raise_error }
+    end
+
+    context 'with a single thor command and multiple environments' do
+      before do
+        mock_command(/\AMASAMUNE_ENV=test_1 thor list/, mock_success)
+        mock_command(/\AMASAMUNE_ENV=test_2 thor list/, mock_success)
+      end
+
+      subject do
+        instance.invoke_parallel('list', { max_tasks: 0 }, [{ env: { 'MASAMUNE_ENV' => 'test_1' } }, { env: { 'MASAMUNE_ENV' => 'test_2' } }])
+      end
+
+      it { expect { subject }.to_not raise_error }
+    end
+
+    context 'with a multiple thor command and multiple environments' do
+      before do
+        mock_command(/\AMASAMUNE_ENV=test_1 thor list/, mock_success)
+        mock_command(/\AMASAMUNE_ENV=test_2 thor list/, mock_success)
+        mock_command(/\AMASAMUNE_ENV=test_1 thor help/, mock_success)
+        mock_command(/\AMASAMUNE_ENV=test_2 thor help/, mock_success)
+      end
+
+      subject do
+        instance.invoke_parallel('list', 'help', { max_tasks: 0 }, [{ env: { 'MASAMUNE_ENV' => 'test_1' } }, { env: { 'MASAMUNE_ENV' => 'test_2' } }])
       end
 
       it { expect { subject }.to_not raise_error }

data/spec/masamune/actions/postgres_admin_spec.rb

@@ -37,7 +37,7 @@ describe Masamune::Actions::PostgresAdmin do
       let(:action) { :create }
 
       before do
-        mock_command(/\Acreatedb/, mock_success)
+        mock_command(/\APGOPTIONS=.* createdb/, mock_success)
       end
 
       it { is_expected.to be_success }
@@ -47,7 +47,7 @@ describe Masamune::Actions::PostgresAdmin do
       let(:action) { :drop }
 
       before do
-        mock_command(/\Adropdb/, mock_success)
+        mock_command(/\APGOPTIONS=.* dropdb/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/postgres_spec.rb

@@ -46,18 +46,29 @@ describe Masamune::Actions::Postgres do
   end
 
   describe '.postgres' do
-    before do
-      mock_command(/\Apsql/, mock_success)
+    subject(:action) { instance.postgres }
+
+    context 'when success' do
+      before do
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.postgres }
+    context 'when failure' do
+      before do
+        mock_command(/\APGOPTIONS=.* psql/, mock_failure)
+      end
 
-    it { is_expected.to be_success }
+      it { is_expected.not_to be_success }
+    end
 
     context 'with retries and backoff' do
       before do
         allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(retries: 1, backoff: 10)
         expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
       end
 
       it { is_expected.to be_success }

data/spec/masamune/actions/transform_spec.rb

@@ -83,7 +83,7 @@ describe Masamune::Actions::Transform do
     context 'without :map' do
       before do
         expect_any_instance_of(Masamune::Schema::Map).to_not receive(:apply)
-        mock_command(/\Apsql/, mock_success)
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
       end
 
       it { is_expected.to be_success }
@@ -104,7 +104,7 @@ describe Masamune::Actions::Transform do
         end
 
         expect_any_instance_of(Masamune::Schema::Map).to receive(:apply).and_call_original
-        mock_command(/\Apsql/, mock_success)
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
       end
 
       it { is_expected.to be_success }
@@ -113,7 +113,7 @@ describe Masamune::Actions::Transform do
 
   describe '.relabel_dimension' do
     before do
-      mock_command(/\Apsql/, mock_success)
+      mock_command(/\APGOPTIONS=.* psql/, mock_success)
     end
 
     subject { instance.relabel_dimension(postgres.user_dimension, options) }
@@ -123,7 +123,7 @@ describe Masamune::Actions::Transform do
 
   describe '.consolidate_dimension' do
     before do
-      mock_command(/\Apsql/, mock_success)
+      mock_command(/\APGOPTIONS=.* psql/, mock_success)
     end
 
     subject { instance.consolidate_dimension(postgres.user_dimension, options) }
@@ -137,7 +137,7 @@ describe Masamune::Actions::Transform do
     context 'without :map' do
       before do
         expect_any_instance_of(Masamune::Schema::Map).to_not receive(:apply)
-        mock_command(/\Apsql/, mock_success)
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
       end
 
       subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
@@ -152,7 +152,7 @@ describe Masamune::Actions::Transform do
         end
 
         expect_any_instance_of(Masamune::Schema::Map).to receive(:apply).and_call_original
-        mock_command(/\Apsql/, mock_success)
+        mock_command(/\APGOPTIONS=.* psql/, mock_success)
       end
 
       subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
@@ -165,7 +165,7 @@ describe Masamune::Actions::Transform do
     let(:date) { DateTime.civil(2014, 8) }
 
     before do
-      mock_command(/\Apsql/, mock_success)
+      mock_command(/\APGOPTIONS=.* psql/, mock_success)
     end
 
     subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date, options) }

data/spec/support/masamune/mock_command.rb

@@ -46,11 +46,21 @@ module Masamune::MockCommand
       def reset!
         @patterns = {}
       end
+
+      def match(pattern)
+        # nop
+      end
+    end
+
+    def command_env_and_args
+      command_env = @delegate.respond_to?(:command_env) ? @delegate.command_env.map { |key, val| "#{key}=#{val}" } : []
+      (command_env + @delegate.command_args).join(' ')
     end
 
     def around_execute(&block)
       self.class.patterns.each do |pattern, (value, io)|
-        next unless @delegate.command_args.join(' ') =~ pattern
+        next unless command_env_and_args =~ pattern
+        CommandMatcher.match(pattern)
         until io.eof?
           line = io.gets
           line_no ||= 0
@@ -90,6 +100,7 @@ module Masamune::MockCommand
   end
 
   def mock_command(pattern, value = nil, io = StringIO.new, &block)
+    expect(CommandMatcher).to receive(:match).with(pattern)
     CommandMatcher.add_pattern(pattern, block_given? ? block.to_proc : value, io, &block)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.18.1
+  version: 0.18.2
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-10 00:00:00.000000000 Z
+date: 2016-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor