masamune 0.13.4 → 0.13.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b1cfbf2086afb91e515f3350cdefd9edc38b5c5
-  data.tar.gz: 7fd27c38a43dbfaa2b4a75db79207c9da2d412ca
+  metadata.gz: 77626a3f01664f620bc080cac5edf5118d5ab70b
+  data.tar.gz: f32fca25218d4d6c8e990a0f2456ae8fa82c7b7b
 SHA512:
-  metadata.gz: 665f24b4261a0e9f05367d031de278cf3561e9993b004339feafb0022da22d43ed54be8682b9706e9ffb420ec0ec72bc9167bb54cf3b2e3f68065b05db91dcdb
-  data.tar.gz: 7ab3d42d09761b855abd75210d96c841709bba94bbe591c90844d3199328ffaa84f405708a9b7df198bd14a08c501e21f9fd0dff347f97f84478a63f7c47dc61
+  metadata.gz: 1010e89677ee57f4dc5b18fe5992f11233f24ada91eac3132eec0f5a288ac1bfb2981f2ce645603a2a63d4d1b01dc28d48163ba58cbab8832481d5004dbb633a
+  data.tar.gz: 0cdcf98d169c4b674d494098c25c00f8cec223a035f69936f103531527b24ea2872444daf5ac84ed8a9375795414c52cab6dfbb48eb419dbe816363df0e74194

data/lib/masamune/actions/elastic_mapreduce.rb

@@ -32,7 +32,7 @@ module Masamune::Actions
 
       command = Masamune::Commands::Interactive.new(environment, :interactive => opts.fetch(:interactive, false))
       command = Masamune::Commands::ElasticMapReduce.new(command, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.elastic_mapreduce.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute

data/lib/masamune/actions/hadoop_filesystem.rb

@@ -30,7 +30,7 @@ module Masamune::Actions
       opts.merge!(block: block.to_proc) if block_given?
 
       command = Masamune::Commands::HadoopFilesystem.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_filesystem.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute

data/lib/masamune/actions/hadoop_streaming.rb

@@ -32,7 +32,7 @@ module Masamune::Actions
       end
 
       command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
-      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_streaming.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute

data/lib/masamune/actions/hive.rb

@@ -36,7 +36,7 @@ module Masamune::Actions
 
       command = Masamune::Commands::Hive.new(environment, opts)
       command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
-      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hive.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute

data/lib/masamune/actions/postgres.rb

@@ -36,6 +36,7 @@ module Masamune::Actions
       opts.merge!(block: block.to_proc) if block_given?
 
       command = Masamune::Commands::Postgres.new(environment, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.postgres.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute
@@ -50,14 +51,14 @@ module Masamune::Actions
     def load_postgres_setup_files
       configuration.postgres[:setup_files].each do |file|
         configuration.with_quiet do
-          postgres(file: file)
+          postgres(file: file, retries: 0)
         end
       end if configuration.postgres.has_key?(:setup_files)
     end
 
     def load_postgres_schema
       transform = define_schema(catalog, :postgres)
-      postgres(file: transform.to_file)
+      postgres(file: transform.to_file, retries: 0)
     rescue => e
       logger.error(e)
       logger.error("Could not load schema")

data/lib/masamune/actions/s3cmd.rb

@@ -31,7 +31,7 @@ module Masamune::Actions
       opts.merge!(block: block.to_proc) if block_given?
 
       command = Masamune::Commands::S3Cmd.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.s3cmd.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute

data/lib/masamune/helpers/postgres.rb

@@ -39,7 +39,7 @@ module Masamune::Helpers
     end
 
     def database_exists?
-      @database_exists ||= postgres(exec: 'SELECT version();', fail_fast: false).success?
+      @database_exists ||= postgres(exec: 'SELECT version();', fail_fast: false, retries: 0).success?
     end
 
     def table_exists?(table)
@@ -64,7 +64,7 @@ module Masamune::Helpers
 
     def update_tables
       return unless @cache.empty?
-      postgres(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true) do |line|
+      postgres(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true, retries: 0) do |line|
         table = line.strip
         next if table =~ /\Apg_/
         @cache[table] ||= nil
@@ -73,7 +73,7 @@ module Masamune::Helpers
 
     def update_table_last_modified_at(table, column)
       return if @cache[table].present?
-      postgres(exec: "SELECT MAX(#{column}) FROM #{table};", tuple_output: true) do |line|
+      postgres(exec: "SELECT MAX(#{column}) FROM #{table};", tuple_output: true, retries: 0) do |line|
         begin
           @cache[table] = Time.parse(line.strip).at_beginning_of_minute.utc
         rescue ArgumentError

data/lib/masamune/tasks/postgres_thor.rb

@@ -38,9 +38,11 @@ module Masamune::Tasks
     method_option :exec, :aliases => '-e', :desc => 'SQL from command line'
     method_option :output, :aliases => '-o', :desc => 'Save SQL output to file'
     method_option :csv, :type => :boolean, :desc => 'Report SQL output in CSV format', :default => false
+    method_option :retry, :type => :boolean, :desc => 'Retry SQL query in event of failure', :default => false
     def psql_exec
       postgres_options = options.dup.with_indifferent_access
       postgres_options.merge!(print: true)
+      postgres_options.merge!(retries: 0) unless options[:retry]
       postgres(postgres_options)
     end
     default_task :psql_exec

data/lib/masamune/transform/define_schema.psql.erb

@@ -23,8 +23,6 @@
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 CREATE EXTENSION IF NOT EXISTS hstore;
 
-SET TIMEZONE TO 'UTC';
-
 CREATE OR REPLACE FUNCTION json_to_hstore(JSON)
 RETURNS HSTORE AS $$
   SELECT HSTORE(RTRIM(LTRIM(REPLACE($1::TEXT, ':', '=>'), '{'), '}'))

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.13.4'
+  VERSION = '0.13.5'
 end

data/spec/masamune/actions/elastic_mapreduce_spec.rb

@@ -48,6 +48,15 @@ describe Masamune::Actions::ElasticMapreduce do
     subject { instance.elastic_mapreduce }
 
     it { is_expected.to be_success }
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 
   describe '.after_initialize' do

data/spec/masamune/actions/hadoop_filesystem_spec.rb

@@ -40,5 +40,14 @@ describe Masamune::Actions::HadoopFilesystem do
     subject { instance.hadoop_filesystem }
 
     it { is_expected.to be_success }
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :hadoop_filesystem).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 end

data/spec/masamune/actions/hadoop_streaming_spec.rb

@@ -39,6 +39,8 @@ describe Masamune::Actions::HadoopStreaming do
 
   describe '.hadoop_streaming' do
     before do
+      allow(instance).to receive_message_chain(:configuration, :hadoop_streaming).and_return({})
+      allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({})
       mock_command(/\Ahadoop/, mock_success)
     end
 
@@ -48,7 +50,7 @@ describe Masamune::Actions::HadoopStreaming do
 
     context 'with jobflow' do
       before do
-        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({jobflow: 'j-XYZ'})
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return(jobflow: 'j-XYZ')
         mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aelastic-mapreduce/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost hadoop/, mock_success)
@@ -61,7 +63,7 @@ describe Masamune::Actions::HadoopStreaming do
       let(:extra) { ['-D', 'EXTRA'] }
 
       before do
-        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return({jobflow: 'j-XYZ'})
+        allow(instance).to receive_message_chain(:configuration, :elastic_mapreduce).and_return(jobflow: 'j-XYZ')
         mock_command(/\Ahadoop/, mock_failure)
         mock_command(/\Aelastic-mapreduce/, mock_success, StringIO.new('ssh fakehost exit'))
         mock_command(/\Assh fakehost -D EXTRA hadoop/, mock_failure)
@@ -70,5 +72,14 @@ describe Masamune::Actions::HadoopStreaming do
 
       it { is_expected.to be_success }
     end
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :hadoop_streaming).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 end

data/spec/masamune/actions/hive_spec.rb

@@ -65,6 +65,15 @@ describe Masamune::Actions::Hive do
 
       it { is_expected.to be_success }
     end
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :hive).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 
   describe '.after_initialize' do

data/spec/masamune/actions/postgres_spec.rb

@@ -55,6 +55,15 @@ describe Masamune::Actions::Postgres do
     subject { instance.postgres }
 
     it { is_expected.to be_success }
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :postgres).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 
   describe '.after_initialize' do
@@ -81,7 +90,7 @@ describe Masamune::Actions::Postgres do
       before do
         expect(postgres_helper).to receive(:database_exists?).and_return(false)
         expect(instance).to receive(:postgres_admin).with(action: :create, database: 'test', safe: true).once
-        expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
+        expect(instance).to receive(:postgres).with(file: 'catalog.psql', retries: 0).once
         after_initialize_invoke
       end
       it 'should call posgres_admin once' do; end
@@ -91,7 +100,7 @@ describe Masamune::Actions::Postgres do
       before do
         expect(postgres_helper).to receive(:database_exists?).and_return(true)
         expect(instance).to receive(:postgres_admin).never
-        expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
+        expect(instance).to receive(:postgres).with(file: 'catalog.psql', retries: 0).once
         after_initialize_invoke
       end
       it 'should not call postgres_admin' do; end
@@ -101,8 +110,8 @@ describe Masamune::Actions::Postgres do
       let(:setup_files) { ['setup.psql'] }
       before do
         expect(postgres_helper).to receive(:database_exists?).and_return(true)
-        expect(instance).to receive(:postgres).with(file: setup_files.first).once
-        expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
+        expect(instance).to receive(:postgres).with(file: setup_files.first, retries: 0).once
+        expect(instance).to receive(:postgres).with(file: 'catalog.psql', retries: 0).once
         after_initialize_invoke
       end
       it 'should call postgres with setup_files' do; end
@@ -124,7 +133,7 @@ describe Masamune::Actions::Postgres do
       before do
         filesystem.touch!('schema_1.psql', 'schema_2.psql')
         expect(postgres_helper).to receive(:database_exists?).and_return(true)
-        expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
+        expect(instance).to receive(:postgres).with(file: 'catalog.psql', retries: 0).once
         after_initialize_invoke
       end
       it 'should call postgres with schema_files' do; end
@@ -135,7 +144,7 @@ describe Masamune::Actions::Postgres do
       before do
         filesystem.touch!('schema.rb')
         expect(postgres_helper).to receive(:database_exists?).and_return(true)
-        expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
+        expect(instance).to receive(:postgres).with(file: 'catalog.psql', retries: 0).once
         after_initialize_invoke
       end
       it 'should call postgres with schema_files' do; end

data/spec/masamune/actions/s3cmd_spec.rb

@@ -40,5 +40,14 @@ describe Masamune::Actions::S3Cmd do
     subject { instance.s3cmd 'ls', 's3://fake-bucket' }
 
     it { is_expected.to be_success }
+
+    context 'with retries and backoff' do
+      before do
+        allow(instance).to receive_message_chain(:configuration, :s3cmd).and_return(retries: 1, backoff: 10)
+        expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(retries: 1, backoff: 10)).once.and_call_original
+      end
+
+      it { is_expected.to be_success }
+    end
   end
 end

data/spec/masamune/helpers/postgres_spec.rb

@@ -26,10 +26,30 @@ describe Masamune::Helpers::Postgres do
   let(:environment) { double }
   let(:instance) { described_class.new(environment) }
 
+  describe '#database_exists' do
+    let(:mock_status) { }
+
+    before do
+      expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT version();', fail_fast: false, retries: 0)).and_return(mock_status)
+    end
+
+    subject { instance.database_exists? }
+
+    context 'when database exists' do
+      let(:mock_status) { mock_success }
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when database does not exist' do
+      let(:mock_status) { mock_failure }
+      it { is_expected.to eq(false) }
+    end
+  end
+
   describe '#table_exists' do
     before do
       expect(instance).to receive(:database_exists?).and_return(true)
-      expect(instance).to receive(:postgres).with(hash_including(:exec, :tuple_output)).and_yield('  foo').and_yield('  bar').and_yield('  baz')
+      expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true, retries: 0)).and_yield('  foo').and_yield('  bar').and_yield('  baz')
     end
 
     subject { instance.table_exists?(table) }
@@ -56,7 +76,7 @@ describe Masamune::Helpers::Postgres do
     context 'with last_modified_at option' do
       before do
         expect(instance).to receive(:table_exists?).and_return(true)
-        expect(instance).to receive(:postgres).with(hash_including(:exec, :tuple_output)).and_yield(output)
+        expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT MAX(last_modified_at) FROM foo;', tuple_output: true, retries: 0)).and_yield(output)
       end
 
       let(:options) { { last_modified_at: 'last_modified_at' } }

data/spec/masamune/tasks/hive_thor_spec.rb

@@ -34,6 +34,15 @@ describe Masamune::Tasks::HiveThor do
     it_behaves_like 'command usage'
   end
 
+  context 'without arguments' do
+    let(:options) { [] }
+
+    it do
+      expect_any_instance_of(described_class).to receive(:hive).with(hash_including(retries: 0)).once.and_return(mock_success)
+      cli_invocation
+    end
+  end
+
   context 'with --file and --initialize' do
     let(:options) { ['--file=zombo.hql', '--initialize'] }
     it do
@@ -75,4 +84,12 @@ describe Masamune::Tasks::HiveThor do
       cli_invocation
     end
   end
+
+  context 'with --retry' do
+    let(:options) { ['--retry'] }
+    it do
+      expect_any_instance_of(described_class).to receive(:hive).with(hash_excluding(retries: 0)).once.and_return(mock_success)
+      cli_invocation
+    end
+  end
 end

data/spec/masamune/tasks/postgres_thor_spec.rb

@@ -31,10 +31,19 @@ describe Masamune::Tasks::PostgresThor do
     it_behaves_like 'command usage'
   end
 
+  context 'without arguments' do
+    let(:options) { [] }
+
+    it do
+      expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(retries: 0)).once.and_return(mock_success)
+      cli_invocation
+    end
+  end
+
   context 'with --file and --initialize' do
     let(:options) { ['--file=zombo.hql', '--initialize'] }
     it do
-      expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String)).once.and_return(mock_success)
+      expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String), retries: 0).once.and_return(mock_success)
       expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(file: 'zombo.hql')).once.and_return(mock_success)
       cli_invocation
     end
@@ -47,4 +56,12 @@ describe Masamune::Tasks::PostgresThor do
       cli_invocation
     end
   end
+
+  context 'with --retry' do
+    let(:options) { ['--retry'] }
+    it do
+      expect_any_instance_of(described_class).to receive(:postgres).with(hash_excluding(retries: 0)).once.and_return(mock_success)
+      cli_invocation
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.13.4
+  version: 0.13.5
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor