cheftacular 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,9 +39,11 @@ class Cheftacular
39
39
  if @config['helper'].is_initialization_command?(ARGV[0])
40
40
  @options['command'] = ARGV[0] #this is normally set by parse_context but that is not run for initialization commands
41
41
  else
42
+ @config['stateless_action'].cheftacular_config('sync') unless @config['helper'].running_on_chef_node?
43
+
42
44
  @config['stateless_action'].initialize_data_bag_contents(@options['env']) #ensure basic structure are always maintained before each run
43
45
 
44
- @config['parser'].parse_application_context if @config['cheftacular']['mode'] == 'application'
46
+ @config['parser'].parse_application_context if @config['helper'].running_in_mode?('application')
45
47
 
46
48
  @config['parser'].parse_context
47
49
 
@@ -50,7 +52,7 @@ class Cheftacular
50
52
  @config['auditor'].audit_run if @config['cheftacular']['auditing']
51
53
  end
52
54
 
53
- @config['stateless_action'].send('check_cheftacular_yml_keys')
55
+ @config['stateless_action'].check_cheftacular_yml_keys unless @config['helper'].is_initialization_command?(ARGV[0])
54
56
 
55
57
  @config['action'].send(@options['command']) if @config['helper'].is_command?(@options['command'])
56
58
 
@@ -58,6 +60,8 @@ class Cheftacular
58
60
 
59
61
  @config['stateless_action'].send('help') if @config['helper'].is_not_command_or_stateless_command?(@options['command'])
60
62
 
63
+ @config['queue_master'].work_off_slack_queue unless @config['helper'].is_initialization_command?(@options['command'])
64
+
61
65
  @config['helper'].output_run_stats
62
66
 
63
67
  exit #explicitly call this in case some celluoid workers are still hanging around
@@ -14,6 +14,10 @@ class Cheftacular
14
14
  end
15
15
  end
16
16
 
17
+ def write_environment_config_cache
18
+ File.open( current_environment_config_cache_file_path, "w") { |f| f.write("set for #{ Time.now.strftime("%Y%m%d") }") }
19
+ end
20
+
17
21
  def check_nodes_file_cache nodes=[]
18
22
  Dir.entries(current_nodes_file_cache_path).each do |location|
19
23
  next if is_junk_filename?(location)
@@ -32,6 +36,10 @@ class Cheftacular
32
36
  current_file_path 'audit-check.txt'
33
37
  end
34
38
 
39
+ def current_environment_config_cache_file_path
40
+ current_file_path 'environment_config-check.txt'
41
+ end
42
+
35
43
  def is_junk_filename? filename
36
44
  filename =~ /.DS_Store|.com.apple.timemachine.supported|README.*/ || filename == '.' || filename == '..' && File.directory?(filename)
37
45
  end
@@ -68,12 +76,17 @@ class Cheftacular
68
76
 
69
77
  Dir.entries(base_dir).each do |entry|
70
78
  next if is_junk_filename?(entry)
79
+ next if File.file?("#{ base_dir }/#{ entry }") && entry == 'local_cheftacular_cache' && mode != 'all'
71
80
 
72
81
  case mode
73
82
  when 'old'
74
83
  FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }") && !entry.include?(Time.now.strftime("%Y%m%d"))
75
- when 'current'
84
+ when 'current-nodes'
76
85
  check_current_day_entry = true
86
+ when 'all'
87
+ FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }")
88
+
89
+ FileUtils.rm_rf("#{ base_dir }/#{ entry }") if File.exists?("#{ base_dir }/#{ entry }") && File.directory?("#{ base_dir }/#{ entry }")
77
90
  when 'current-audit-only'
78
91
  FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }") && entry.include?(Time.now.strftime("%Y%m%d"))
79
92
  end
@@ -102,21 +115,29 @@ class Cheftacular
102
115
  current_file_path "chef_repo_cheftacular_cache"
103
116
  end
104
117
 
118
+ def local_cheftacular_file_cache_path
119
+ current_file_path "local_cheftacular_cache", false
120
+ end
121
+
105
122
  def write_chef_repo_cheftacular_cache_file hash
106
123
  File.open( current_chef_repo_cheftacular_file_cache_path, "w") { |f| f.write(hash) }
107
124
  end
108
125
 
126
+ def write_local_cheftacular_cache_file hash_string
127
+ File.open( local_cheftacular_file_cache_path, 'w') { |f| f.write(hash_string) }
128
+ end
129
+
109
130
  def write_chef_repo_cheftacular_yml_file file_location
110
131
  File.open( file_location, "w") { |f| f.write(@config['helper'].compile_chef_repo_cheftacular_yml_as_hash.to_yaml) }
111
132
  end
112
133
 
113
- def write_config_cheftacular_yml_file filename='cheftacular.yml'
114
- File.open( File.join(@config['locs']['chef-repo'], "config", filename), "w") { |f| f.write(File.read(File.join(@config['locs']['examples'], "cheftacular.yml"))) }
134
+ def write_config_cheftacular_yml_file to_be_created_filename='cheftacular.yml', example_filename='cheftacular.yml'
135
+ File.open( File.join(@config['locs']['chef-repo'], "config", to_be_created_filename), "w") { |f| f.write(File.read(File.join(@config['locs']['examples'], example_filename))) }
115
136
  end
116
137
 
117
138
  private
118
- def current_file_path file_name
119
- File.join( @config['locs']['app-root'], 'tmp', @config['helper'].declassify, "#{ Time.now.strftime("%Y%m%d") }-#{ file_name }")
139
+ def current_file_path file_name, use_timestamp=true
140
+ File.join( @config['locs']['app-root'], 'tmp', @config['helper'].declassify, ( use_timestamp ? "#{ Time.now.strftime("%Y%m%d") }-#{ file_name }" : file_name ))
120
141
  end
121
142
  end
122
143
  end
@@ -120,6 +120,13 @@ class Cheftacular
120
120
  ret_hash
121
121
  end
122
122
 
123
+ def get_db_primary_node
124
+ nodes = get_true_node_objects true
125
+ target_db_role = get_current_repo_config['db_primary_host_role']
126
+
127
+ @config['parser'].exclude_nodes( nodes, [{ unless: "role[#{ target_db_role }]" }, { if: { not_env: @options['env'] } }], true)
128
+ end
129
+
123
130
  def get_split_branch_hash ret={}
124
131
  @config['cheftacular']['repositories'].each_pair do |name, repo_hash|
125
132
  ret[repo_hash['name']] = repo_hash if repo_hash.has_key?('has_split_branches') && repo_hash['has_split_branches']
@@ -263,6 +263,35 @@ class Cheftacular
263
263
  end
264
264
  end
265
265
 
266
+ class Hash
267
+ def deep_diff(compare_hash, remove_if_nil_on_original=false)
268
+ original_hash = self
269
+
270
+ (original_hash.keys | compare_hash.keys).inject({}) do |diff_hash, key|
271
+ if original_hash[key] != compare_hash[key]
272
+ if original_hash[key].respond_to?(:deep_diff) && compare_hash[key].respond_to?(:deep_diff)
273
+ diff_hash[key] = original_hash[key].deep_diff(compare_hash[key], remove_if_nil_on_original)
274
+ else
275
+ if remove_if_nil_on_original
276
+ diff_hash[key] = []
277
+ diff_hash[key] << original_hash[key] if original_hash.has_key?(key)
278
+ diff_hash[key] << compare_hash[key] if original_hash.has_key?(key)
279
+ diff_hash.delete(key) if diff_hash[key].empty?
280
+ else
281
+ diff_hash[key] = [original_hash[key], compare_hash[key]]
282
+ end
283
+ end
284
+ end
285
+
286
+ diff_hash
287
+ end
288
+ end
289
+
290
+ def compact
291
+ self.select { |_, value| !value.nil? }
292
+ end
293
+ end
294
+
266
295
  class String
267
296
  def scrub_pretty_text
268
297
  self.gsub("",'').gsub(/\[0m|\[1m|\[32m|\[35m|\[36m/,'')
@@ -4,12 +4,16 @@ class Cheftacular
4
4
  def initialize options, config
5
5
  @options, @config = options, config
6
6
 
7
+ initialize_queues
8
+
7
9
  initialize_yaml_configuration
8
10
 
9
11
  initialize_default_cheftacular_options
10
12
 
11
13
  initialize_locations
12
14
 
15
+ initialize_data_bag_cheftacular_hash if !@config['helper'].is_initialization_command?(ARGV[0]) && !@config['helper'].running_on_chef_node?
16
+
13
17
  initialize_monkeypatches unless @config['helper'].running_on_chef_node?
14
18
 
15
19
  initialize_arguments
@@ -231,13 +235,45 @@ class Cheftacular
231
235
  end.parse!
232
236
  end
233
237
 
238
+ def initialize_queues
239
+ @config['slack_queue'] ||= []
240
+ end
241
+
234
242
  def initialize_yaml_configuration
235
243
  @config['cheftacular'] = @config['helper'].get_cheftacular_yml_as_hash
236
244
  end
237
245
 
246
+ def initialize_data_bag_cheftacular_hash
247
+ initialize_ridley
248
+
249
+ @config['ChefDataBag'] ||= Cheftacular::ChefDataBag.new(@options, @config)
250
+
251
+ @config['ChefDataBag'].init_bag('default', 'cheftacular', false)
252
+
253
+ diff_hash = @config['cheftacular'].deep_diff(@config['default']['cheftacular_bag_hash'], true).except('mode', 'default_repository').compact
254
+
255
+ diff_hash.each_pair do |key, value|
256
+ diff_hash.delete(key) if value.empty? || value.nil?
257
+ end
258
+
259
+ if @config['helper'].running_in_mode?('devops') && !diff_hash.empty?
260
+ puts "Difference detected between local cheftacular.yml and data bag cheftacular.yml! Displaying..."
261
+
262
+ ap diff_hash
263
+ elsif @config['helper'].running_in_mode?('application') && @config['default']['cheftacular_bag_hash']['slack']['webhook'] && !diff_hash.empty?
264
+ @config['slack_queue'] << diff_hash.awesome_inspect({plain: true, indent: 2}).prepend('```').insert(-1, '```')
265
+ end
266
+
267
+ @config['cheftacular'] = if @config['default']['cheftacular_bag_hash']['sync_application_cheftacular_yml']
268
+ @config['default']['cheftacular_bag_hash'].deep_merge(@config['cheftacular'])
269
+ else
270
+ @config['cheftacular'].deep_merge(@config['default']['cheftacular_bag_hash'].except('default_repository', 'mode'))
271
+ end
272
+ end
273
+
238
274
  def initialize_default_cheftacular_options
239
275
  @options['env'] = @config['cheftacular']['default_environment'] if @config['cheftacular'].has_key?('default_environment')
240
- @options['repository'] = @config['cheftacular']['default_repository'] if @config['cheftacular'].has_key?('default_repository')
276
+ @options['repository'] = @config['cheftacular']['default_repository'] if @config['cheftacular'].has_key?('default_repository')
241
277
  end
242
278
 
243
279
  def initialize_monkeypatches
@@ -287,7 +323,7 @@ class Cheftacular
287
323
  locs['chef'] = File.expand_path("~/.chef") unless locs['chef']
288
324
  locs['cookbooks'] = File.expand_path("#{ locs['chef-repo'] }/cookbooks")
289
325
  locs['berks'] = File.expand_path('~/.berkshelf/cookbooks')
290
- locs['wrapper-cookbooks'] = @config['cheftacular']['wrapper_cookbooks']
326
+ locs['wrapper-cookbooks'] = @config['cheftacular']['wrapper_cookbooks'] unless @config['helper'].running_in_mode?('application')
291
327
  locs['ssh'] = File.expand_path('~/.ssh')
292
328
  locs['chef-log'] = File.expand_path("#{ locs['root']}/log") unless locs['chef-log']
293
329
  locs['app-tmp'] = File.expand_path("#{ locs['app-root']}/tmp")
@@ -330,6 +366,10 @@ class Cheftacular
330
366
 
331
367
  @config['ChefDataBag'].init_bag('default', 'authentication') if bags_to_load.empty? || bags_to_load.include?('authentication')
332
368
 
369
+ @config['ChefDataBag'].init_bag('default', 'cheftacular', false) if bags_to_load.include?('cheftacular')
370
+
371
+ @config['ChefDataBag'].init_bag('default', 'environment_config', false) if bags_to_load.empty? || bags_to_load.include?('environment_config')
372
+
333
373
  @config['helper'].completion_rate?(38, 'initializer') if in_initializer
334
374
 
335
375
  @config['ChefDataBag'].init_bag(env, 'addresses', false) if bags_to_load.empty? || bags_to_load.include?('addresses')
@@ -437,6 +477,7 @@ class Cheftacular
437
477
  @config['error'] = Cheftacular::Error.new(@options, @config)
438
478
  @config['dummy_sshkit'] = SSHKit::Backend::Netssh.new(SSHKit::Host.new('127.0.0.1'))
439
479
  @config['DNS'] = Cheftacular::DNS.new(@options, @config)
480
+ @config['queue_master'] = Cheftacular::QueueMaster.new(@options, @config)
440
481
  @config['cloud_provider'] = Cheftacular::CloudProvider.new(@options, @config)
441
482
  end
442
483
 
@@ -15,11 +15,7 @@ class Cheftacular
15
15
 
16
16
  @options['command'] = ARGV[0] unless @options['command']
17
17
 
18
- unless @config['helper'].is_stateless_command?(@options['command'])
19
- parse_repository(@options['repository'])
20
-
21
- parse_role(@options['role'])
22
- end
18
+ parse_repository(@options['repository'])
23
19
 
24
20
  parse_node_name(@options['node_name']) if @options['node_name']
25
21
 
@@ -45,7 +41,7 @@ class Cheftacular
45
41
  @options['command'] = ARGV[0] unless @config['helper'].is_not_command_or_stateless_command?(ARGV[0])
46
42
  end
47
43
 
48
- return if !@options['codebase'].nil? && !@options['role'].nil? && !@options['command'].nil?
44
+ return if !@options['repository'].nil? && !@options['role'].nil? && !@options['command'].nil?
49
45
  return if !@options['command'].nil? && @config['helper'].is_stateless_command?(ARGV[0])
50
46
  end
51
47
 
@@ -107,10 +103,10 @@ class Cheftacular
107
103
  raise "Cannot set or unset target_revision without a role" unless @options['role']
108
104
 
109
105
  if @options['target_revision']
110
- @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_codebase_from_role_name(@options['role'])] = @options['target_revision']
106
+ @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_repository_from_role_name(@options['role'])] = @options['target_revision']
111
107
 
112
108
  elsif @options['unset_revision']
113
- @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_codebase_from_role_name(@options['role'])] = "<use_default>"
109
+ @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_repository_from_role_name(@options['role'])] = "<use_default>"
114
110
 
115
111
  end
116
112
 
@@ -0,0 +1,16 @@
1
+
2
+ class Cheftacular
3
+ class QueueMaster
4
+ def initialize options, config
5
+ @options, @config = options, config
6
+ end
7
+
8
+ def work_off_slack_queue
9
+ return true if @config['slack_queue'].empty?
10
+
11
+ @config['slack_queue'].each do |message|
12
+ @config['stateless_action'].slack(message)
13
+ end
14
+ end
15
+ end
16
+ end
@@ -3,9 +3,20 @@ class Cheftacular
3
3
  class StatelessActionDocumentation
4
4
  def backups
5
5
  @config['documentation']['stateless_action'] << [
6
- "`cft backup [activate|deactivate]` this command " +
6
+ "`cft backups [activate|deactivate|load|run]` this command " +
7
7
  "sets the fetch_backups and restore_backups flags in your config data bag for an environment. " +
8
- "These can be used to give application developers a way to trigger / untrigger restores in an environment"
8
+ "These can be used to give application developers a way to trigger / untrigger restores in an environment",
9
+
10
+ [
11
+ " 1. `activate` will turn on automated backup running (turns on the flag for the env in the config bag).",
12
+
13
+ " 2. `deactivate` will turn off automated backup running.",
14
+
15
+ " 3. `load` will fetch the latest backup from the production primary **if it doesn't already exist on " +
16
+ "the server** and run the _backup loading command_ to load this backup into the env.",
17
+
18
+ " 4. `run` will simply just run the _backup loading command_ to load the latest backup onto the server."
19
+ ]
9
20
  ]
10
21
 
11
22
  @config['documentation']['application'] << @config['documentation']['stateless_action'].last
@@ -13,103 +24,203 @@ class Cheftacular
13
24
  end
14
25
 
15
26
  class StatelessAction
16
- def backups force_fetch_and_restore=false
17
- fetch_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']
18
- restore_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups']
27
+ def backups command=''
28
+ command = ARGV[1] if command.blank?
29
+
30
+ raise "Unsupported command (#{ command }) for cft backups" unless command =~ /activate|deactivate|load|run/
31
+
32
+ self.send("backups_#{ command }")
33
+ end
34
+
35
+ private
19
36
 
20
- puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups was set to #{ fetch_backup ? 'on' : 'off' } and restoring backups was set to #{ restore_backup ? 'on' : 'off' }"
37
+ def backups_activate restore_backup=true, fetch_backup=true
38
+ backups_toggle_setting(restore_backup, fetch_backup)
39
+ end
40
+
41
+ def backups_deactivate restore_backup=false, fetch_backup=false
42
+ backups_toggle_setting(restore_backup, fetch_backup)
43
+ end
44
+
45
+ def backups_load status_hash={}
46
+ old_role, old_env, backup_env = @options['role'], @options['env'], @config['cheftacular']['backup_config']['global_backup_environ']
21
47
 
22
- case ARGV[1]
23
- when 'activate' then restore_backup, fetch_backup = true, true
24
- when 'deactivate' then restore_backup, fetch_backup = false, false
48
+ if backup_env != @options['env']
49
+ @config['initializer'].initialize_data_bags_for_environment(backup_env, false, ['addresses', 'server_passwords'])
50
+ @config['initializer'].initialize_passwords backup_env
25
51
  end
26
52
 
27
- puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups is now set to #{ fetch_backup ? 'on' : 'off' } and restoring backups is now set to #{ restore_backup ? 'on' : 'off' }"
53
+ @options['role'] = @config['cheftacular']['backup_config']['global_backup_role_name']
54
+ @options['env'] = backup_env
28
55
 
29
- @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups'] = fetch_backup
30
- @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups'] = restore_backup
56
+ puts "Deploying to backup master to force refresh of the ssh keys..."
57
+ #@config['action'].deploy
31
58
 
32
- @config['ChefDataBag'].save_config_bag
59
+ @options['role'] = old_role
60
+ @options['env'] = old_env
33
61
 
34
- if force_fetch_and_restore
35
- nodes = @config['getter'].get_true_node_objects true
62
+ target_db_primary = @config['getter'].get_db_primary_node
36
63
 
37
- db_primary_nodes = @config['parser'].exclude_nodes( nodes, [{ unless: 'role[db_primary]' }, { if: { not_env: 'production' } }])
64
+ args_config = [
65
+ { unless: "role[#{ @config['cheftacular']['backup_config']['global_backup_role_name'] }]" },
66
+ { if: { not_env: backup_env } }
67
+ ]
38
68
 
39
- backup_slave_local_ip = @config['cheftacular']['backup_server']
69
+ backup_master = @config['parser'].exclude_nodes( nodes, args_config, true)
70
+ backup_master_local_ip = @config['getter'].get_address_hash(backup_master.first.name)['priv']
40
71
 
41
- if backup_slave_local_ip == 'first_production_slave'
42
- backup_slave = @config['parser'].exclude_nodes( nodes, [{ unless: 'role[db_slave]' }, { if: { not_env: 'production' } }], true)
72
+ options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
43
73
 
44
- backup_slave_local_ip = @config['getter'].get_address_hash(backup_slave.first.hostname)['priv']
45
- end
74
+ on ( backup_master.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
75
+ n = get_node_from_address(nodes, host.hostname)
46
76
 
47
- options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = set_local_instance_vars
77
+ puts("Beginning latest db_fetch_and_check for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
48
78
 
49
- on ( db_primary_nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
50
- n = get_node_from_address(nodes, host.hostname)
79
+ status_hash['latest_backup'] = start_db_check_and_fetch( n.name, n.public_ipaddress, options, locs, cheftacular, passwords)
80
+ end
51
81
 
52
- puts("Beginning db fetch_and_restore for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
82
+ return false unless status_hash['latest_backup']['file_check']
53
83
 
54
- start_db_fetch_and_restore( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, backup_slave_local_ip)
55
- end
84
+ on ( target_db_primary.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
85
+ n = get_node_from_address(nodes, host.hostname)
86
+
87
+ puts("Beginning db_backup_fetch for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
88
+
89
+ start_db_backup_fetch( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, backup_master_local_ip, status_hash['latest_backup'])
90
+ end
56
91
 
57
- @config['action'].migrate
58
- else
59
- puts "Triggering deploy on databases to refresh backup setting..."
92
+ backups_run(nodes)
93
+ end
94
+
95
+ def backups_run
96
+ target_db_primary = @config['getter'].get_db_primary_node
97
+ applications_as_string = @config['getter'].get_repo_names_for_repositories.keys.join(',')
98
+ env_pg_pass = @config[@options['env']]['chef_passwords_bag_hash']['pg_pass']
99
+
100
+ options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
101
+
102
+ on ( target_db_primary.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
103
+ n = get_node_from_address(nodes, host.hostname)
60
104
 
61
- @options['role'] = 'db_primary'
105
+ puts("Beginning db_backup_run for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
62
106
 
63
- @config['action'].deploy
107
+ start_db_backup_run( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, applications_as_string, env_pg_pass )
64
108
  end
65
109
  end
110
+
111
+ def backups_toggle_setting restore_backup, fetch_backup
112
+ initial_fetch_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']
113
+ initial_restore_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups']
114
+
115
+ puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups was set to " +
116
+ "#{ initial_fetch_backup ? 'on' : 'off' } and restoring backups was set to #{ initial_restore_backup ? 'on' : 'off' }"
117
+
118
+ puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups is now set to " +
119
+ "#{ fetch_backup ? 'on' : 'off' } and restoring backups is now set to #{ restore_backup ? 'on' : 'off' }"
120
+
121
+ @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups'] = fetch_backup
122
+ @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups'] = restore_backup
123
+
124
+ @config['ChefDataBag'].save_config_bag
125
+
126
+ puts "Triggering deploy on databases to refresh backup setting..."
127
+
128
+ @options['role'] = 'db_primary'
129
+
130
+ @config['action'].deploy
131
+ end
66
132
  end
67
133
  end
68
134
 
69
135
  module SSHKit
70
136
  module Backend
71
137
  class Netssh
72
- def start_db_fetch_and_restore name, ip_address, options, locs, cheftacular, passwords, global_backup_ip, out=[]
73
- log_loc, timestamp = set_log_loc_and_timestamp(locs)
138
+ def start_db_check_and_fetch name, ip_address, options, locs, cheftacular, passwords, out=[], return_hash={ 'file_check' => false }
139
+ base_dir = cheftacular['backup_config']['global_backup_path']
74
140
 
75
- puts("Generating pg_restore log file for #{ name } (#{ ip_address }) at #{ log_loc }/#{ name }-fetch-and-restore-#{ timestamp }.txt") unless options['quiet']
141
+ if !sudo_test( passwords[ip_address], base_dir ) #true if dir exists
142
+ puts "#{ name } (#{ ip_address }) cannot run #{ __method__ } as there is no directory at #{ base_dir }!"
76
143
 
77
- cheftacular['db_primary_backup_database_stacks'].each do |target_database_stack|
78
- compile_database_backups_hash(cheftacular, target_database_stack, global_backup_ip).each_pair do |app_name, app_hash|
144
+ return return_hash
145
+ end
79
146
 
80
- out << sudo_capture( passwords[ip_address], 'scp', "#{ cheftacular['deploy_user'] }@#{ app_hash['backup_server'] }:#{ location }", backup_dir )
147
+ target_dir = case cheftacular['backup_filesystem']
148
+ when 'backup_gem'
149
+ backup_gem_dir_sort passwords[ip_address], cheftacular, base_dir
150
+ when 'raw'
151
+ File.join( base_dir, sudo_capture( passwords[ip_address], :ls, base_dir ).split(' ').last )
152
+ else
153
+ raise "#{ __method__ } does not currently support the #{ cheftacular['backup_filesystem'] } backup strategy at this time"
154
+ end
155
+
156
+ return_hash['file_check'] = true
157
+ return_hash['file_path'] = [target_dir].flatten.last
158
+ return_hash['file_dir'] = case cheftacular['backup_filesystem']
159
+ when 'backup_gem' then target_dir.first
160
+ when 'raw' then base_dir
161
+ end
162
+
163
+ return_hash
164
+ end
81
165
 
82
- puts(out.last) if options['output'] || options['verbose']
166
+ def start_db_backup_fetch name, ip_address, options, locs, cheftacular, passwords, backup_master_local_ip, backup_hash, out=[]
167
+ if sudo_test( passwords[ip_address], backup_path ) #true if dir exists
168
+ puts "#{ name } (#{ ip_address }) already has the backup at #{ backup_path }, skipping #{ __method__ }..."
83
169
 
84
- commands = [
85
- "pg_restore --verbose --clean --no-acl --no-owner -j 4 -h localhost -U #{ cheftacular['deploy_user'] } -d #{ app_hash['repo_name'] }_#{ options['env'] } #{ app_hash['restore_backup_file_name'] }",
86
- "service postgresql restart"
87
- ]
170
+ return true
171
+ end
88
172
 
89
- commands.each do |command|
90
- out << sudo_capture( passwords[ip_address], command )
173
+ sudo_execute( passwords[ip_address], :mkdir, '-p', backup_hash['file_dir'] )
91
174
 
92
- puts(out.last) if options['output'] || options['verbose']
93
- end
94
- end
95
- end
175
+ sudo_execute( passwords[ip_address], :chown, "#{ cheftacular['deploy_user'] }:#{ cheftacular['deploy_user'] }", backup_hash['file_dir'] )
176
+
177
+ sudo_execute( passwords[ip_address], :chmod, cheftacular['backup_config']['backup_dir_mode'], backup_hash['file_dir'] )
96
178
 
97
- #TODO fetch and restore other database types
179
+ execute( :scp, "#{ cheftacular['deploy_user'] }@#{ backup_master_local_ip }:#{ backup_hash['file_path'] }", backup_hash['file_dir'] )
98
180
 
99
- ::File.open("#{ log_loc }/#{ name }-fetch-and-restore-#{ timestamp }.txt", "w") { |f| f.write(out.join("\n").scrub_pretty_text) } unless options['no_logs']
181
+ puts "Finished transferring #{ backup_hash['file_path'] } to #{ name }(#{ ip_address })..."
182
+ end
100
183
 
101
- puts "Succeeded fetch and restore of #{ name } (#{ ip_address })"
184
+ def start_db_backup_run name, ip_address, options, locs, cheftacular, passwords, applications_as_string, env_pg_pass
185
+ puts "Beginning backup run on #{ name } (#{ ip_address }), this command may take a while to complete..."
186
+ case cheftacular['backup_filesystem']
187
+ when 'backup_gem'
188
+ #'ruby /root/backup_management.rb /mnt/postgresbackups/backups ENVIRONMENT APPLICATIONS PG_PASS > /root/restore.log 2>&1'
189
+ command = cheftacular['backup_config']['backup_load_command']
190
+ command = command.gsub('ENVIRONMENT', options['env']).gsub('APPLICATIONS', applications_as_string).gsub('PG_PASS', env_pg_pass)
102
191
 
103
- [out.join("\n"), timestamp]
192
+ sudo_execute( passwords[ip_address], command )
193
+ when 'raw'
194
+ end
195
+
196
+ puts "Finished executing backup command on #{ name } (#{ ip_address })"
104
197
  end
105
198
 
106
- def compile_database_backups_hash cheftacular, target_database_stack, global_backup_ip, ret={}
107
- cheftacular['repositories'].each_pair do |app_name, app_hash|
108
- ret[app_name] = app_hash if app_hash['restore_backup_file_name'] && app_hash['database'] == target_database_stack
109
- ret[app_name]['backup_server'] = global_backup_ip unless ret[app_name]['backup_server']
199
+ def backup_gem_dir_sort password, cheftacular, base_dir
200
+ timestamp_dirs, check_dirs, target_dir = [], [], ''
201
+
202
+ dirs = sudo_capture( password, :ls, base_dir )
203
+
204
+ dirs.split(' ').each do |timestamp_dir|
205
+ next if timestamp_dir == '.' || timestamp_dir == '..'
206
+
207
+ timestamp_dirs << timestamp_dir
208
+ end
209
+
210
+ timestamp_dirs.each do |dir|
211
+ if check_dirs.empty?
212
+ check_dirs << dir
213
+ target_dir = dir
214
+ else
215
+ check_dirs.each do |cdir|
216
+ target_dir = dir if Date.parse(dir) >= Date.parse(target_dir)
217
+ end
218
+ end
110
219
  end
111
220
 
112
- ret
221
+ target_file = sudo_capture( password, :ls, File.join(base_dir, target_dir) ).split(' ').last
222
+
223
+ [ File.join(base_dir, target_dir), File.join( target_dir, target_file )]
113
224
  end
114
225
  end
115
226
  end
@@ -13,6 +13,15 @@ class Cheftacular
13
13
  def check_cheftacular_yml_keys out=[], exit_on_missing=false, warn_on_missing=false
14
14
  base_message = "Your cheftacular.yml is missing the key KEY, its default value is being set to DEFAULT for this run."
15
15
 
16
+ #############################2.7.0################################################
17
+
18
+ unless @config['cheftacular'].has_key?('backup_config')
19
+ #backup_config:global_backup_role_name
20
+ base_message.gsub('KEY', 'backup_config').gsub('DEFAULT', 'nil')
21
+
22
+ warn_on_missing = true
23
+ end
24
+
16
25
  #############################2.6.0################################################
17
26
  unless @config['cheftacular'].has_key?('route_dns_changes_via')
18
27
  puts base_message.gsub('KEY', 'route_dns_changes_via').gsub('DEFAULT', @options['preferred_cloud'])