masamune 0.18.10 → 0.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/masamune/actions/aws_emr.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_streaming.rb +1 -1
- data/lib/masamune/actions/hive.rb +1 -1
- data/lib/masamune/actions/postgres.rb +3 -3
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +5 -5
- data/lib/masamune/configuration.rb +1 -1
- data/lib/masamune/helpers/postgres.rb +3 -3
- data/lib/masamune/tasks/hive_thor.rb +1 -1
- data/lib/masamune/tasks/postgres_thor.rb +1 -1
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +3 -3
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/actions/hadoop_streaming_spec.rb +3 -3
- data/spec/masamune/actions/hive_spec.rb +3 -3
- data/spec/masamune/actions/postgres_admin_spec.rb +10 -10
- data/spec/masamune/actions/postgres_spec.rb +9 -9
- data/spec/masamune/actions/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/retry_with_backoff_spec.rb +14 -14
- data/spec/masamune/filesystem_spec.rb +1 -1
- data/spec/masamune/helpers/postgres_spec.rb +3 -3
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/tasks/postgres_thor_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/shared_example_group.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3dd3fb8ccd6de6212ecdf97971916b26c2761ebc
|
|
4
|
+
data.tar.gz: 7b9c608f895b6b5abc8b32c097754fa720d04cde
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2567cc86397502ebf47886c61262bbe4d2e08188ec3c049e477efd91c36008df8fecb333c47a9c6743d7b3dd0633e9ced8a5a34832a706a46ba1714c8ebb0483
|
|
7
|
+
data.tar.gz: 045e5e14031ef288f7a85758d6cf317f5814d0d0a9ec5d1ef0dddf8a7a894d3ab1d11af7283fe8948f53e695e2915ab73d15e5360c946d1e7aa2ac735c7ed798
|
|
@@ -28,7 +28,7 @@ module Masamune::Actions
|
|
|
28
28
|
opts = opts.to_hash.symbolize_keys
|
|
29
29
|
|
|
30
30
|
command = Masamune::Commands::AwsEmr.new(environment, opts)
|
|
31
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.aws_emr.slice(:
|
|
31
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.aws_emr.slice(:max_retries, :backoff).merge(opts))
|
|
32
32
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
33
33
|
|
|
34
34
|
command.interactive? ? command.replace : command.execute
|
|
@@ -30,7 +30,7 @@ module Masamune::Actions
|
|
|
30
30
|
opts[:block] = block.to_proc if block_given?
|
|
31
31
|
|
|
32
32
|
command = Masamune::Commands::HadoopFilesystem.new(environment, opts)
|
|
33
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_filesystem.slice(:
|
|
33
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_filesystem.slice(:max_retries, :backoff).merge(opts))
|
|
34
34
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
35
35
|
|
|
36
36
|
command.execute
|
|
@@ -27,7 +27,7 @@ module Masamune::Actions
|
|
|
27
27
|
|
|
28
28
|
command = Masamune::Commands::HadoopStreaming.new(environment, aws_emr_options(opts))
|
|
29
29
|
command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.commands.aws_emr[:cluster_id]
|
|
30
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_streaming.slice(:
|
|
30
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_streaming.slice(:max_retries, :backoff).merge(opts))
|
|
31
31
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
32
32
|
|
|
33
33
|
command.execute
|
|
@@ -34,7 +34,7 @@ module Masamune::Actions
|
|
|
34
34
|
|
|
35
35
|
command = Masamune::Commands::Hive.new(environment, opts)
|
|
36
36
|
command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.commands.aws_emr[:cluster_id]
|
|
37
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hive.slice(:
|
|
37
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hive.slice(:max_retries, :backoff).merge(opts))
|
|
38
38
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
39
39
|
|
|
40
40
|
command.interactive? ? command.replace : command.execute
|
|
@@ -35,7 +35,7 @@ module Masamune::Actions
|
|
|
35
35
|
opts[:block] = block.to_proc if block_given?
|
|
36
36
|
|
|
37
37
|
command = Masamune::Commands::Postgres.new(environment, opts)
|
|
38
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.postgres.slice(:
|
|
38
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.postgres.slice(:max_retries, :backoff).merge(opts))
|
|
39
39
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
40
40
|
|
|
41
41
|
command.interactive? ? command.replace : command.execute
|
|
@@ -51,14 +51,14 @@ module Masamune::Actions
|
|
|
51
51
|
return unless configuration.commands.postgres.key?(:setup_files)
|
|
52
52
|
configuration.commands.postgres[:setup_files].each do |file|
|
|
53
53
|
configuration.with_quiet do
|
|
54
|
-
postgres(file: file,
|
|
54
|
+
postgres(file: file, max_retries: 0)
|
|
55
55
|
end
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
def load_postgres_schema
|
|
60
60
|
transform = define_schema(catalog, :postgres)
|
|
61
|
-
postgres(file: transform.to_file,
|
|
61
|
+
postgres(file: transform.to_file, max_retries: 0)
|
|
62
62
|
rescue => e
|
|
63
63
|
logger.error(e)
|
|
64
64
|
logger.error('Could not load schema')
|
|
@@ -35,7 +35,7 @@ module Masamune::Actions
|
|
|
35
35
|
private
|
|
36
36
|
|
|
37
37
|
def postgres_admin_retry_with_backoff_options
|
|
38
|
-
configuration.commands.postgres.merge(configuration.commands.postgres_admin).slice(:
|
|
38
|
+
configuration.commands.postgres.merge(configuration.commands.postgres_admin).slice(:max_retries, :backoff)
|
|
39
39
|
end
|
|
40
40
|
end
|
|
41
41
|
end
|
|
@@ -33,7 +33,7 @@ module Masamune::Actions
|
|
|
33
33
|
opts[:block] = block.to_proc if block_given?
|
|
34
34
|
|
|
35
35
|
command = Masamune::Commands::S3Cmd.new(environment, opts)
|
|
36
|
-
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.s3cmd.slice(:
|
|
36
|
+
command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.s3cmd.slice(:max_retries, :backoff).merge(opts))
|
|
37
37
|
command = Masamune::Commands::Shell.new(command, opts)
|
|
38
38
|
|
|
39
39
|
command.execute
|
|
@@ -28,8 +28,8 @@ module Masamune::Commands
|
|
|
28
28
|
|
|
29
29
|
def initialize(delegate, attrs = {})
|
|
30
30
|
super delegate
|
|
31
|
-
@delegate
|
|
32
|
-
@
|
|
31
|
+
@delegate = delegate
|
|
32
|
+
@max_retries = attrs.fetch(:max_retries, configuration.max_retries)
|
|
33
33
|
@backoff = attrs.fetch(:backoff, configuration.backoff)
|
|
34
34
|
@retry_count = 0
|
|
35
35
|
end
|
|
@@ -48,11 +48,11 @@ module Masamune::Commands
|
|
|
48
48
|
logger.error(e.to_s)
|
|
49
49
|
sleep @backoff
|
|
50
50
|
@retry_count += 1
|
|
51
|
-
if @retry_count > @
|
|
52
|
-
logger.debug("max retries (#{@
|
|
51
|
+
if @retry_count > @max_retries
|
|
52
|
+
logger.debug("max retries (#{@max_retries}) attempted, bailing")
|
|
53
53
|
OpenStruct.new(success?: false, exitstatus: MAX_RETRY_EXIT_STATUS)
|
|
54
54
|
else
|
|
55
|
-
logger.debug("retrying (#{@retry_count}/#{@
|
|
55
|
+
logger.debug("retrying (#{@retry_count}/#{@max_retries})")
|
|
56
56
|
retry
|
|
57
57
|
end
|
|
58
58
|
end
|
|
@@ -56,7 +56,7 @@ class Masamune::Configuration < Hashie::Dash
|
|
|
56
56
|
property :debug, default: false
|
|
57
57
|
property :dry_run, default: false
|
|
58
58
|
property :lock
|
|
59
|
-
property :
|
|
59
|
+
property :max_retries, default: 3
|
|
60
60
|
property :backoff, default: 5
|
|
61
61
|
property :params, default: Hashie::Mash.new
|
|
62
62
|
property :commands, default: Hashie::Mash.new { |h, k| h[k] = Hashie::Mash.new }
|
|
@@ -39,7 +39,7 @@ module Masamune::Helpers
|
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
def database_exists?
|
|
42
|
-
@database_exists ||= postgres(exec: 'SELECT version();', fail_fast: false,
|
|
42
|
+
@database_exists ||= postgres(exec: 'SELECT version();', fail_fast: false, max_retries: 0).success?
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def table_exists?(table)
|
|
@@ -64,7 +64,7 @@ module Masamune::Helpers
|
|
|
64
64
|
|
|
65
65
|
def update_tables
|
|
66
66
|
return unless @cache.empty?
|
|
67
|
-
postgres(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true,
|
|
67
|
+
postgres(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true, max_retries: 0) do |line|
|
|
68
68
|
table = line.strip
|
|
69
69
|
next if table.start_with?('pg_')
|
|
70
70
|
@cache[table] ||= nil
|
|
@@ -73,7 +73,7 @@ module Masamune::Helpers
|
|
|
73
73
|
|
|
74
74
|
def update_table_last_modified_at(table, column)
|
|
75
75
|
return if @cache[table].present?
|
|
76
|
-
postgres(exec: "SELECT MAX(#{column}) FROM #{table};", tuple_output: true,
|
|
76
|
+
postgres(exec: "SELECT MAX(#{column}) FROM #{table};", tuple_output: true, max_retries: 0) do |line|
|
|
77
77
|
last_modified_at = line.strip
|
|
78
78
|
@cache[table] = parse_date_time(last_modified_at) unless last_modified_at.blank?
|
|
79
79
|
end
|
|
@@ -46,7 +46,7 @@ module Masamune::Tasks
|
|
|
46
46
|
def hive_exec
|
|
47
47
|
hive_options = options.dup.with_indifferent_access
|
|
48
48
|
hive_options[:print] = true
|
|
49
|
-
hive_options[:
|
|
49
|
+
hive_options[:max_retries] = 0 unless options[:retry]
|
|
50
50
|
hive_options[:file] = File.expand_path(options[:file]) if options[:file]
|
|
51
51
|
hive_options[:output] = File.expand_path(options[:output]) if options[:output]
|
|
52
52
|
hive(hive_options)
|
|
@@ -42,7 +42,7 @@ module Masamune::Tasks
|
|
|
42
42
|
def psql_exec
|
|
43
43
|
postgres_options = options.dup.with_indifferent_access
|
|
44
44
|
postgres_options[:print] = true
|
|
45
|
-
postgres_options[:
|
|
45
|
+
postgres_options[:max_retries] = 0 unless options[:retry]
|
|
46
46
|
postgres(postgres_options)
|
|
47
47
|
end
|
|
48
48
|
default_task :psql_exec
|
data/lib/masamune/version.rb
CHANGED
|
@@ -55,10 +55,10 @@ describe Masamune::Actions::AwsEmr do
|
|
|
55
55
|
it { expect { action }.to raise_error RuntimeError, 'fail_fast: aws emr ssh' }
|
|
56
56
|
end
|
|
57
57
|
|
|
58
|
-
context 'with
|
|
58
|
+
context 'with max_retries and backoff' do
|
|
59
59
|
before do
|
|
60
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(
|
|
61
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
60
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :aws_emr).and_return(max_retries: 1, backoff: 10)
|
|
61
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
62
62
|
mock_command(/\Aaws emr/, mock_success)
|
|
63
63
|
end
|
|
64
64
|
|
|
@@ -49,10 +49,10 @@ describe Masamune::Actions::HadoopFilesystem do
|
|
|
49
49
|
it { is_expected.not_to be_success }
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
-
context 'with
|
|
52
|
+
context 'with max_retries and backoff' do
|
|
53
53
|
before do
|
|
54
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_filesystem).and_return(
|
|
55
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
54
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_filesystem).and_return(max_retries: 1, backoff: 10)
|
|
55
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
56
56
|
mock_command(/\Ahadoop/, mock_success)
|
|
57
57
|
end
|
|
58
58
|
|
|
@@ -77,10 +77,10 @@ describe Masamune::Actions::HadoopStreaming do
|
|
|
77
77
|
it { is_expected.to be_success }
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
-
context 'with
|
|
80
|
+
context 'with max_retries and backoff' do
|
|
81
81
|
before do
|
|
82
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return(
|
|
83
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
82
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :hadoop_streaming).and_return(max_retries: 1, backoff: 10)
|
|
83
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
84
84
|
mock_command(/\Ahadoop/, mock_success)
|
|
85
85
|
end
|
|
86
86
|
|
|
@@ -73,10 +73,10 @@ describe Masamune::Actions::Hive do
|
|
|
73
73
|
it { is_expected.to be_success }
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
-
context 'with
|
|
76
|
+
context 'with max_retries and backoff' do
|
|
77
77
|
before do
|
|
78
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :hive).and_return(
|
|
79
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
78
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :hive).and_return(max_retries: 1, backoff: 10)
|
|
79
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
80
80
|
mock_command(/\Ahive/, mock_success)
|
|
81
81
|
end
|
|
82
82
|
|
|
@@ -30,22 +30,22 @@ describe Masamune::Actions::PostgresAdmin do
|
|
|
30
30
|
|
|
31
31
|
let(:instance) { klass.new }
|
|
32
32
|
|
|
33
|
-
shared_context '
|
|
34
|
-
context 'with
|
|
33
|
+
shared_context 'max_retries and backoff' do
|
|
34
|
+
context 'with max_retries and backoff configured via postgres_admin command' do
|
|
35
35
|
before do
|
|
36
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(
|
|
37
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :postgres_admin).and_return(
|
|
38
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
36
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(max_retries: 1, backoff: 10)
|
|
37
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :postgres_admin).and_return(max_retries: 3, backoff: 1)
|
|
38
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 3, backoff: 1)).once.and_call_original
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
it { is_expected.to be_success }
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
-
context 'with
|
|
44
|
+
context 'with max_retries and backoff configured via postgres command' do
|
|
45
45
|
before do
|
|
46
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(
|
|
46
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(max_retries: 1, backoff: 10)
|
|
47
47
|
allow(instance).to receive_message_chain(:configuration, :commands, :postgres_admin).and_return({})
|
|
48
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
48
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
it { is_expected.to be_success }
|
|
@@ -64,7 +64,7 @@ describe Masamune::Actions::PostgresAdmin do
|
|
|
64
64
|
|
|
65
65
|
it { is_expected.to be_success }
|
|
66
66
|
|
|
67
|
-
include_context '
|
|
67
|
+
include_context 'max_retries and backoff'
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
context 'with :action :drop' do
|
|
@@ -76,7 +76,7 @@ describe Masamune::Actions::PostgresAdmin do
|
|
|
76
76
|
|
|
77
77
|
it { is_expected.to be_success }
|
|
78
78
|
|
|
79
|
-
include_context '
|
|
79
|
+
include_context 'max_retries and backoff'
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
end
|
|
@@ -64,10 +64,10 @@ describe Masamune::Actions::Postgres do
|
|
|
64
64
|
it { is_expected.not_to be_success }
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
-
context 'with
|
|
67
|
+
context 'with max_retries and backoff' do
|
|
68
68
|
before do
|
|
69
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(
|
|
70
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
69
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :postgres).and_return(max_retries: 1, backoff: 10)
|
|
70
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
71
71
|
mock_command(/\APGOPTIONS=.* psql/, mock_success)
|
|
72
72
|
end
|
|
73
73
|
|
|
@@ -100,7 +100,7 @@ describe Masamune::Actions::Postgres do
|
|
|
100
100
|
before do
|
|
101
101
|
expect(postgres_helper).to receive(:database_exists?).and_return(false)
|
|
102
102
|
expect(instance).to receive(:postgres_admin).with(action: :create, database: 'test', safe: true).once
|
|
103
|
-
expect(instance).to receive(:postgres).with(file: 'catalog.psql',
|
|
103
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql', max_retries: 0).once
|
|
104
104
|
after_initialize_invoke
|
|
105
105
|
end
|
|
106
106
|
it 'should call posgres_admin once' do
|
|
@@ -111,7 +111,7 @@ describe Masamune::Actions::Postgres do
|
|
|
111
111
|
before do
|
|
112
112
|
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
|
113
113
|
expect(instance).to receive(:postgres_admin).never
|
|
114
|
-
expect(instance).to receive(:postgres).with(file: 'catalog.psql',
|
|
114
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql', max_retries: 0).once
|
|
115
115
|
after_initialize_invoke
|
|
116
116
|
end
|
|
117
117
|
it 'should not call postgres_admin' do
|
|
@@ -122,8 +122,8 @@ describe Masamune::Actions::Postgres do
|
|
|
122
122
|
let(:setup_files) { ['setup.psql'] }
|
|
123
123
|
before do
|
|
124
124
|
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
|
125
|
-
expect(instance).to receive(:postgres).with(file: setup_files.first,
|
|
126
|
-
expect(instance).to receive(:postgres).with(file: 'catalog.psql',
|
|
125
|
+
expect(instance).to receive(:postgres).with(file: setup_files.first, max_retries: 0).once
|
|
126
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql', max_retries: 0).once
|
|
127
127
|
after_initialize_invoke
|
|
128
128
|
end
|
|
129
129
|
it 'should call postgres with setup_files' do
|
|
@@ -147,7 +147,7 @@ describe Masamune::Actions::Postgres do
|
|
|
147
147
|
before do
|
|
148
148
|
filesystem.touch!('schema_1.psql', 'schema_2.psql')
|
|
149
149
|
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
|
150
|
-
expect(instance).to receive(:postgres).with(file: 'catalog.psql',
|
|
150
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql', max_retries: 0).once
|
|
151
151
|
after_initialize_invoke
|
|
152
152
|
end
|
|
153
153
|
it 'should call postgres with schema_files' do
|
|
@@ -159,7 +159,7 @@ describe Masamune::Actions::Postgres do
|
|
|
159
159
|
before do
|
|
160
160
|
filesystem.touch!('schema.rb')
|
|
161
161
|
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
|
162
|
-
expect(instance).to receive(:postgres).with(file: 'catalog.psql',
|
|
162
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql', max_retries: 0).once
|
|
163
163
|
after_initialize_invoke
|
|
164
164
|
end
|
|
165
165
|
it 'should call postgres with schema_files' do
|
|
@@ -39,10 +39,10 @@ describe Masamune::Actions::S3Cmd do
|
|
|
39
39
|
|
|
40
40
|
it { is_expected.to be_success }
|
|
41
41
|
|
|
42
|
-
context 'with
|
|
42
|
+
context 'with max_retries and backoff' do
|
|
43
43
|
before do
|
|
44
|
-
allow(instance).to receive_message_chain(:configuration, :commands, :s3cmd).and_return(
|
|
45
|
-
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(
|
|
44
|
+
allow(instance).to receive_message_chain(:configuration, :commands, :s3cmd).and_return(max_retries: 1, backoff: 10)
|
|
45
|
+
expect(Masamune::Commands::RetryWithBackoff).to receive(:new).with(anything, hash_including(max_retries: 1, backoff: 10)).once.and_call_original
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
it { is_expected.to be_success }
|
|
@@ -21,22 +21,22 @@
|
|
|
21
21
|
# THE SOFTWARE.
|
|
22
22
|
|
|
23
23
|
describe Masamune::Commands::RetryWithBackoff do
|
|
24
|
-
let(:options) { {
|
|
24
|
+
let(:options) { { max_retries: max_retries, backoff: 0 } }
|
|
25
25
|
let(:delegate) { double }
|
|
26
26
|
let(:instance) { described_class.new(delegate, options) }
|
|
27
27
|
|
|
28
28
|
before do
|
|
29
29
|
allow(delegate).to receive(:logger).and_return(double)
|
|
30
|
-
allow(delegate).to receive(:configuration).and_return(double(
|
|
30
|
+
allow(delegate).to receive(:configuration).and_return(double(max_retries: 0, backoff: 0))
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
describe '#around_execute' do
|
|
34
|
-
let(:
|
|
34
|
+
let(:max_retries) { 3 }
|
|
35
35
|
|
|
36
36
|
context 'when retry command fails with status but eventually succeeds' do
|
|
37
37
|
before do
|
|
38
|
-
expect(instance.logger).to receive(:error).with('exited with code: 42').exactly(
|
|
39
|
-
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(
|
|
38
|
+
expect(instance.logger).to receive(:error).with('exited with code: 42').exactly(max_retries - 1)
|
|
39
|
+
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(max_retries - 1)
|
|
40
40
|
subject
|
|
41
41
|
end
|
|
42
42
|
|
|
@@ -44,7 +44,7 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
44
44
|
@retry_count = 0
|
|
45
45
|
instance.around_execute do
|
|
46
46
|
@retry_count += 1
|
|
47
|
-
if @retry_count <
|
|
47
|
+
if @retry_count < max_retries
|
|
48
48
|
OpenStruct.new(success?: false, exitstatus: 42)
|
|
49
49
|
else
|
|
50
50
|
OpenStruct.new(success?: true)
|
|
@@ -55,7 +55,7 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
55
55
|
it 'logs useful debug and error messages' do
|
|
56
56
|
end
|
|
57
57
|
it 'attempts to retry the specified number of times' do
|
|
58
|
-
expect(@retry_count).to eq(
|
|
58
|
+
expect(@retry_count).to eq(max_retries)
|
|
59
59
|
end
|
|
60
60
|
it 'returns result status' do
|
|
61
61
|
is_expected.to be_success
|
|
@@ -64,8 +64,8 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
64
64
|
|
|
65
65
|
context 'when retry command fails with exception but eventually succeeds' do
|
|
66
66
|
before do
|
|
67
|
-
expect(instance.logger).to receive(:error).with('wtf').exactly(
|
|
68
|
-
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(
|
|
67
|
+
expect(instance.logger).to receive(:error).with('wtf').exactly(max_retries - 1)
|
|
68
|
+
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(max_retries - 1)
|
|
69
69
|
subject
|
|
70
70
|
end
|
|
71
71
|
|
|
@@ -73,7 +73,7 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
73
73
|
@retry_count = 0
|
|
74
74
|
instance.around_execute do
|
|
75
75
|
@retry_count += 1
|
|
76
|
-
raise 'wtf' if @retry_count <
|
|
76
|
+
raise 'wtf' if @retry_count < max_retries
|
|
77
77
|
OpenStruct.new(success?: true)
|
|
78
78
|
end
|
|
79
79
|
end
|
|
@@ -81,7 +81,7 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
81
81
|
it 'logs useful debug and error messages' do
|
|
82
82
|
end
|
|
83
83
|
it 'attempts to retry the specified number of times' do
|
|
84
|
-
expect(@retry_count).to eq(
|
|
84
|
+
expect(@retry_count).to eq(max_retries)
|
|
85
85
|
end
|
|
86
86
|
it 'returns result status' do
|
|
87
87
|
is_expected.to be_success
|
|
@@ -90,8 +90,8 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
90
90
|
|
|
91
91
|
context 'when retry command eventually fails' do
|
|
92
92
|
before do
|
|
93
|
-
expect(instance.logger).to receive(:error).with('wtf').exactly(
|
|
94
|
-
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(
|
|
93
|
+
expect(instance.logger).to receive(:error).with('wtf').exactly(max_retries + 1)
|
|
94
|
+
expect(instance.logger).to receive(:debug).with(/retrying.*/).exactly(max_retries)
|
|
95
95
|
expect(instance.logger).to receive(:debug).with(/max retries.*bailing/)
|
|
96
96
|
subject
|
|
97
97
|
end
|
|
@@ -107,7 +107,7 @@ describe Masamune::Commands::RetryWithBackoff do
|
|
|
107
107
|
it 'logs useful debug and error messages' do
|
|
108
108
|
end
|
|
109
109
|
it 'attempts to retry the specified number of times' do
|
|
110
|
-
expect(@retry_count).to eq(
|
|
110
|
+
expect(@retry_count).to eq(max_retries + 1)
|
|
111
111
|
end
|
|
112
112
|
it 'returns failure status' do
|
|
113
113
|
is_expected.not_to be_success
|
|
@@ -36,7 +36,7 @@ shared_examples_for 'Filesystem' do
|
|
|
36
36
|
let(:old_file) { File.join(old_dir, SecureRandom.hex + '.txt') }
|
|
37
37
|
|
|
38
38
|
before do
|
|
39
|
-
filesystem.configuration.
|
|
39
|
+
filesystem.configuration.max_retries = 0
|
|
40
40
|
FileUtils.mkdir_p(old_dir)
|
|
41
41
|
FileUtils.touch(old_file)
|
|
42
42
|
end
|
|
@@ -28,7 +28,7 @@ describe Masamune::Helpers::Postgres do
|
|
|
28
28
|
let(:mock_status) {}
|
|
29
29
|
|
|
30
30
|
before do
|
|
31
|
-
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT version();', fail_fast: false,
|
|
31
|
+
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT version();', fail_fast: false, max_retries: 0)).and_return(mock_status)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
subject { instance.database_exists? }
|
|
@@ -47,7 +47,7 @@ describe Masamune::Helpers::Postgres do
|
|
|
47
47
|
describe '#table_exists' do
|
|
48
48
|
before do
|
|
49
49
|
expect(instance).to receive(:database_exists?).and_return(true)
|
|
50
|
-
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true,
|
|
50
|
+
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true, max_retries: 0)).and_yield(' foo').and_yield(' bar').and_yield(' baz')
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
subject { instance.table_exists?(table) }
|
|
@@ -74,7 +74,7 @@ describe Masamune::Helpers::Postgres do
|
|
|
74
74
|
context 'with last_modified_at option' do
|
|
75
75
|
before do
|
|
76
76
|
expect(instance).to receive(:table_exists?).and_return(true)
|
|
77
|
-
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT MAX(last_modified_at) FROM foo;', tuple_output: true,
|
|
77
|
+
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT MAX(last_modified_at) FROM foo;', tuple_output: true, max_retries: 0)).and_yield(output).and_yield('')
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
let(:options) { { last_modified_at: 'last_modified_at' } }
|
|
@@ -35,7 +35,7 @@ describe Masamune::Tasks::HiveThor do
|
|
|
35
35
|
let(:options) { [] }
|
|
36
36
|
|
|
37
37
|
it do
|
|
38
|
-
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(
|
|
38
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(max_retries: 0)).once.and_return(mock_success)
|
|
39
39
|
execute_command
|
|
40
40
|
end
|
|
41
41
|
end
|
|
@@ -85,7 +85,7 @@ describe Masamune::Tasks::HiveThor do
|
|
|
85
85
|
context 'with --retry' do
|
|
86
86
|
let(:options) { ['--retry'] }
|
|
87
87
|
it do
|
|
88
|
-
expect_any_instance_of(described_class).to receive(:hive).with(hash_excluding(
|
|
88
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_excluding(max_retries: 0)).once.and_return(mock_success)
|
|
89
89
|
execute_command
|
|
90
90
|
end
|
|
91
91
|
end
|
|
@@ -32,7 +32,7 @@ describe Masamune::Tasks::PostgresThor do
|
|
|
32
32
|
let(:options) { [] }
|
|
33
33
|
|
|
34
34
|
it do
|
|
35
|
-
expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(
|
|
35
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(max_retries: 0)).once.and_return(mock_success)
|
|
36
36
|
execute_command
|
|
37
37
|
end
|
|
38
38
|
end
|
|
@@ -40,7 +40,7 @@ describe Masamune::Tasks::PostgresThor do
|
|
|
40
40
|
context 'with --file and --initialize' do
|
|
41
41
|
let(:options) { ['--file=zombo.hql', '--initialize'] }
|
|
42
42
|
it do
|
|
43
|
-
expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String),
|
|
43
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String), max_retries: 0).once.and_return(mock_success)
|
|
44
44
|
expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(file: 'zombo.hql')).once.and_return(mock_success)
|
|
45
45
|
execute_command
|
|
46
46
|
end
|
|
@@ -57,7 +57,7 @@ describe Masamune::Tasks::PostgresThor do
|
|
|
57
57
|
context 'with --retry' do
|
|
58
58
|
let(:options) { ['--retry'] }
|
|
59
59
|
it do
|
|
60
|
-
expect_any_instance_of(described_class).to receive(:postgres).with(hash_excluding(
|
|
60
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(hash_excluding(max_retries: 0)).once.and_return(mock_success)
|
|
61
61
|
execute_command
|
|
62
62
|
end
|
|
63
63
|
end
|
data/spec/spec_helper.rb
CHANGED
|
@@ -32,7 +32,7 @@ ENV['MASAMUNE_ENV'] = 'test'
|
|
|
32
32
|
Masamune::ExampleGroup.configure do |config|
|
|
33
33
|
config.quiet = ENV['MASAMUNE_DEBUG'] ? false : true
|
|
34
34
|
config.debug = ENV['MASAMUNE_DEBUG'] ? true : false
|
|
35
|
-
config.
|
|
35
|
+
config.max_retries = 0
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
RSpec.configure do |config|
|
|
@@ -78,7 +78,7 @@ module Masamune::SharedExampleGroup
|
|
|
78
78
|
if configuration.commands.postgres[:clean]
|
|
79
79
|
postgres_admin(action: :drop, database: configuration.commands.postgres[:database])
|
|
80
80
|
postgres_admin(action: :create, database: configuration.commands.postgres[:database])
|
|
81
|
-
postgres(file: define_schema(catalog, :postgres).to_file,
|
|
81
|
+
postgres(file: define_schema(catalog, :postgres).to_file, max_retries: 0)
|
|
82
82
|
end
|
|
83
83
|
filesystem.paths.each do |_, (path, options)|
|
|
84
84
|
filesystem.remove_dir(path) if options[:clean]
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: masamune
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.18.
|
|
4
|
+
version: 0.18.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Andrews
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-03-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|