cheftacular 2.6.0 → 2.7.0

data/lib/cheftacular/cheftacular.rb

@@ -39,9 +39,11 @@ class Cheftacular
     if @config['helper'].is_initialization_command?(ARGV[0])
       @options['command'] = ARGV[0] #this is normally set by parse_context but that is not run for initialization commands
     else
+      @config['stateless_action'].cheftacular_config('sync') unless @config['helper'].running_on_chef_node?
+
       @config['stateless_action'].initialize_data_bag_contents(@options['env']) #ensure basic structure are always maintained before each run
 
-      @config['parser'].parse_application_context if @config['cheftacular']['mode'] == 'application'
+      @config['parser'].parse_application_context if @config['helper'].running_in_mode?('application')
 
       @config['parser'].parse_context
 
@@ -50,7 +52,7 @@ class Cheftacular
       @config['auditor'].audit_run if @config['cheftacular']['auditing']
     end
 
-    @config['stateless_action'].send('check_cheftacular_yml_keys')
+    @config['stateless_action'].check_cheftacular_yml_keys unless @config['helper'].is_initialization_command?(ARGV[0])
 
     @config['action'].send(@options['command']) if @config['helper'].is_command?(@options['command'])
 
@@ -58,6 +60,8 @@ class Cheftacular
 
     @config['stateless_action'].send('help') if @config['helper'].is_not_command_or_stateless_command?(@options['command'])
 
+    @config['queue_master'].work_off_slack_queue unless @config['helper'].is_initialization_command?(@options['command'])
+
     @config['helper'].output_run_stats
 
     exit #explicitly call this in case some celluoid workers are still hanging around

data/lib/cheftacular/file_system.rb

@@ -14,6 +14,10 @@ class Cheftacular
       end
     end
 
+    def write_environment_config_cache
+      File.open( current_environment_config_cache_file_path, "w") { |f| f.write("set for #{ Time.now.strftime("%Y%m%d") }") }
+    end
+
     def check_nodes_file_cache nodes=[]
       Dir.entries(current_nodes_file_cache_path).each do |location|
         next if is_junk_filename?(location)
@@ -32,6 +36,10 @@ class Cheftacular
       current_file_path 'audit-check.txt'
     end
 
+    def current_environment_config_cache_file_path
+      current_file_path 'environment_config-check.txt'
+    end
+
     def is_junk_filename? filename
       filename =~ /.DS_Store|.com.apple.timemachine.supported|README.*/ || filename == '.' || filename == '..' && File.directory?(filename)
     end
@@ -68,12 +76,17 @@ class Cheftacular
 
       Dir.entries(base_dir).each do |entry|
         next if is_junk_filename?(entry)
+        next if File.file?("#{ base_dir }/#{ entry }") && entry == 'local_cheftacular_cache' && mode != 'all'
 
         case mode
         when 'old'
           FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }") && !entry.include?(Time.now.strftime("%Y%m%d"))
-        when 'current'
+        when 'current-nodes'
           check_current_day_entry = true
+        when 'all'
+          FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }")
+
+          FileUtils.rm_rf("#{ base_dir }/#{ entry }") if File.exists?("#{ base_dir }/#{ entry }") && File.directory?("#{ base_dir }/#{ entry }")
         when 'current-audit-only'
           FileUtils.rm("#{ base_dir }/#{ entry }") if File.file?("#{ base_dir }/#{ entry }") && entry.include?(Time.now.strftime("%Y%m%d"))
         end
@@ -102,21 +115,29 @@ class Cheftacular
       current_file_path "chef_repo_cheftacular_cache"
     end
 
+    def local_cheftacular_file_cache_path
+      current_file_path "local_cheftacular_cache", false
+    end
+
     def write_chef_repo_cheftacular_cache_file hash
       File.open( current_chef_repo_cheftacular_file_cache_path, "w") { |f| f.write(hash) }
     end
 
+    def write_local_cheftacular_cache_file hash_string
+      File.open( local_cheftacular_file_cache_path, 'w') { |f| f.write(hash_string) }
+    end
+
     def write_chef_repo_cheftacular_yml_file file_location
       File.open( file_location, "w") { |f| f.write(@config['helper'].compile_chef_repo_cheftacular_yml_as_hash.to_yaml) }
     end
 
-    def write_config_cheftacular_yml_file filename='cheftacular.yml'
-      File.open( File.join(@config['locs']['chef-repo'], "config", filename), "w") { |f| f.write(File.read(File.join(@config['locs']['examples'], "cheftacular.yml"))) }
+    def write_config_cheftacular_yml_file to_be_created_filename='cheftacular.yml', example_filename='cheftacular.yml'
+      File.open( File.join(@config['locs']['chef-repo'], "config", to_be_created_filename), "w") { |f| f.write(File.read(File.join(@config['locs']['examples'], example_filename))) }
     end
 
     private
-    def current_file_path file_name
-      File.join( @config['locs']['app-root'], 'tmp', @config['helper'].declassify, "#{ Time.now.strftime("%Y%m%d") }-#{ file_name }")
+    def current_file_path file_name, use_timestamp=true
+      File.join( @config['locs']['app-root'], 'tmp', @config['helper'].declassify, ( use_timestamp ? "#{ Time.now.strftime("%Y%m%d") }-#{ file_name }" : file_name ))
     end
   end
 end

data/lib/cheftacular/getter.rb

@@ -120,6 +120,13 @@ class Cheftacular
       ret_hash
     end
 
+    def get_db_primary_node
+      nodes          = get_true_node_objects true
+      target_db_role = get_current_repo_config['db_primary_host_role']
+      
+      @config['parser'].exclude_nodes( nodes, [{ unless: "role[#{ target_db_role }]" }, { if: { not_env: @options['env'] } }], true)
+    end
+
     def get_split_branch_hash ret={}
       @config['cheftacular']['repositories'].each_pair do |name, repo_hash|
         ret[repo_hash['name']] = repo_hash if repo_hash.has_key?('has_split_branches') && repo_hash['has_split_branches']

data/lib/cheftacular/helper.rb

@@ -263,6 +263,35 @@ class Cheftacular
   end
 end
 
+class Hash
+  def deep_diff(compare_hash, remove_if_nil_on_original=false)
+    original_hash = self
+
+    (original_hash.keys | compare_hash.keys).inject({}) do |diff_hash, key|
+      if original_hash[key] != compare_hash[key]
+        if original_hash[key].respond_to?(:deep_diff) && compare_hash[key].respond_to?(:deep_diff)
+          diff_hash[key] = original_hash[key].deep_diff(compare_hash[key], remove_if_nil_on_original)
+        else
+          if remove_if_nil_on_original
+            diff_hash[key]  = []
+            diff_hash[key] << original_hash[key] if original_hash.has_key?(key)
+            diff_hash[key] << compare_hash[key]  if original_hash.has_key?(key)
+            diff_hash.delete(key)                if diff_hash[key].empty?
+          else
+            diff_hash[key] = [original_hash[key], compare_hash[key]]
+          end
+        end
+      end
+
+      diff_hash
+    end
+  end
+
+  def compact
+    self.select { |_, value| !value.nil? }
+  end
+end
+
 class String
   def scrub_pretty_text
     self.gsub("",'').gsub(/\[0m|\[1m|\[32m|\[35m|\[36m/,'')

data/lib/cheftacular/initializer.rb

@@ -4,12 +4,16 @@ class Cheftacular
     def initialize options, config
       @options, @config  = options, config
 
+      initialize_queues
+
       initialize_yaml_configuration
 
       initialize_default_cheftacular_options
 
       initialize_locations
 
+      initialize_data_bag_cheftacular_hash if !@config['helper'].is_initialization_command?(ARGV[0]) && !@config['helper'].running_on_chef_node?
+
       initialize_monkeypatches unless @config['helper'].running_on_chef_node?
 
       initialize_arguments
@@ -231,13 +235,45 @@ class Cheftacular
       end.parse!
     end
 
+    def initialize_queues
+      @config['slack_queue'] ||= []
+    end
+
     def initialize_yaml_configuration
       @config['cheftacular'] = @config['helper'].get_cheftacular_yml_as_hash
     end
 
+    def initialize_data_bag_cheftacular_hash
+      initialize_ridley
+
+      @config['ChefDataBag'] ||= Cheftacular::ChefDataBag.new(@options, @config)
+
+      @config['ChefDataBag'].init_bag('default', 'cheftacular', false)
+
+      diff_hash = @config['cheftacular'].deep_diff(@config['default']['cheftacular_bag_hash'], true).except('mode', 'default_repository').compact
+
+      diff_hash.each_pair do |key, value|
+        diff_hash.delete(key) if value.empty? || value.nil?
+      end
+
+      if @config['helper'].running_in_mode?('devops') && !diff_hash.empty?
+        puts "Difference detected between local cheftacular.yml and data bag cheftacular.yml! Displaying..."
+
+        ap diff_hash
+      elsif @config['helper'].running_in_mode?('application') && @config['default']['cheftacular_bag_hash']['slack']['webhook'] && !diff_hash.empty?
+        @config['slack_queue'] << diff_hash.awesome_inspect({plain: true, indent: 2}).prepend('```').insert(-1, '```')
+      end
+
+      @config['cheftacular'] = if @config['default']['cheftacular_bag_hash']['sync_application_cheftacular_yml']
+                                 @config['default']['cheftacular_bag_hash'].deep_merge(@config['cheftacular'])
+                               else
+                                 @config['cheftacular'].deep_merge(@config['default']['cheftacular_bag_hash'].except('default_repository', 'mode'))
+                               end
+    end
+
     def initialize_default_cheftacular_options
       @options['env']        = @config['cheftacular']['default_environment'] if @config['cheftacular'].has_key?('default_environment')
-      @options['repository'] = @config['cheftacular']['default_repository'] if @config['cheftacular'].has_key?('default_repository')
+      @options['repository'] = @config['cheftacular']['default_repository']  if @config['cheftacular'].has_key?('default_repository')
     end
 
     def initialize_monkeypatches
@@ -287,7 +323,7 @@ class Cheftacular
       locs['chef']                  = File.expand_path("~/.chef")                          unless locs['chef']
       locs['cookbooks']             = File.expand_path("#{ locs['chef-repo'] }/cookbooks")
       locs['berks']                 = File.expand_path('~/.berkshelf/cookbooks')
-      locs['wrapper-cookbooks']     = @config['cheftacular']['wrapper_cookbooks']
+      locs['wrapper-cookbooks']     = @config['cheftacular']['wrapper_cookbooks']          unless @config['helper'].running_in_mode?('application')
       locs['ssh']                   = File.expand_path('~/.ssh')
       locs['chef-log']              = File.expand_path("#{ locs['root']}/log")             unless locs['chef-log']
       locs['app-tmp']               = File.expand_path("#{ locs['app-root']}/tmp")
@@ -330,6 +366,10 @@ class Cheftacular
 
       @config['ChefDataBag'].init_bag('default', 'authentication') if bags_to_load.empty? || bags_to_load.include?('authentication')
 
+      @config['ChefDataBag'].init_bag('default', 'cheftacular', false) if bags_to_load.include?('cheftacular')
+
+      @config['ChefDataBag'].init_bag('default', 'environment_config', false) if bags_to_load.empty? || bags_to_load.include?('environment_config')
+
       @config['helper'].completion_rate?(38, 'initializer') if in_initializer
 
       @config['ChefDataBag'].init_bag(env, 'addresses', false) if bags_to_load.empty? || bags_to_load.include?('addresses')
@@ -437,6 +477,7 @@ class Cheftacular
       @config['error']                          = Cheftacular::Error.new(@options, @config)
       @config['dummy_sshkit']                   = SSHKit::Backend::Netssh.new(SSHKit::Host.new('127.0.0.1'))
       @config['DNS']                            = Cheftacular::DNS.new(@options, @config)
+      @config['queue_master']                   = Cheftacular::QueueMaster.new(@options, @config)
       @config['cloud_provider']                 = Cheftacular::CloudProvider.new(@options, @config)
     end
 

data/lib/cheftacular/parser.rb

@@ -15,11 +15,7 @@ class Cheftacular
 
       @options['command'] = ARGV[0] unless @options['command']
 
-      unless @config['helper'].is_stateless_command?(@options['command'])
-        parse_repository(@options['repository'])
-
-        parse_role(@options['role'])
-      end
+      parse_repository(@options['repository'])
 
       parse_node_name(@options['node_name']) if @options['node_name']
 
@@ -45,7 +41,7 @@ class Cheftacular
         @options['command'] = ARGV[0] unless @config['helper'].is_not_command_or_stateless_command?(ARGV[0])
       end
 
-      return if !@options['codebase'].nil? && !@options['role'].nil? && !@options['command'].nil?
+      return if !@options['repository'].nil? && !@options['role'].nil? && !@options['command'].nil?
       return if !@options['command'].nil? && @config['helper'].is_stateless_command?(ARGV[0])
     end
 
@@ -107,10 +103,10 @@ class Cheftacular
       raise "Cannot set or unset target_revision without a role" unless @options['role']
 
       if @options['target_revision']
-        @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_codebase_from_role_name(@options['role'])] = @options['target_revision']
+        @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_repository_from_role_name(@options['role'])] = @options['target_revision']
 
       elsif @options['unset_revision']
-        @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_codebase_from_role_name(@options['role'])] = "<use_default>"
+        @config[@options['env']]['config_bag_hash'][@options['sub_env']]['app_revisions'][@config['getter'].get_repository_from_role_name(@options['role'])] = "<use_default>"
 
       end
 

data/lib/cheftacular/queue_master.rb

@@ -0,0 +1,16 @@
+
+class Cheftacular
+  class QueueMaster
+    def initialize options, config
+      @options, @config  = options, config
+    end
+
+    def work_off_slack_queue
+      return true if @config['slack_queue'].empty?
+      
+      @config['slack_queue'].each do |message|
+        @config['stateless_action'].slack(message)
+      end
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/backups.rb

@@ -3,9 +3,20 @@ class Cheftacular
   class StatelessActionDocumentation
     def backups
       @config['documentation']['stateless_action'] <<  [
-        "`cft backup [activate|deactivate]` this command " +
+        "`cft backups [activate|deactivate|load|run]` this command " +
         "sets the fetch_backups and restore_backups flags in your config data bag for an environment. " +
-        "These can be used to give application developers a way to trigger / untrigger restores in an environment"
+        "These can be used to give application developers a way to trigger / untrigger restores in an environment",
+      
+        [
+          "    1. `activate` will turn on automated backup running (turns on the flag for the env in the config bag).",
+
+          "    2. `deactivate` will turn off automated backup running.",
+
+          "    3. `load` will fetch the latest backup from the production primary **if it doesn't already exist on " +
+          "the server** and run the _backup loading command_ to load this backup into the env.",
+
+          "    4. `run` will simply just run the _backup loading command_ to load the latest backup onto the server."
+        ]
       ]
 
       @config['documentation']['application'] << @config['documentation']['stateless_action'].last
@@ -13,103 +24,203 @@ class Cheftacular
   end
 
   class StatelessAction
-    def backups force_fetch_and_restore=false
-      fetch_backup   = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']
-      restore_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups']
+    def backups command=''
+      command = ARGV[1] if command.blank?
+
+      raise "Unsupported command (#{ command }) for cft backups" unless command =~ /activate|deactivate|load|run/
+
+      self.send("backups_#{ command }")
+    end
+
+    private
 
-      puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups was set to #{ fetch_backup ? 'on' : 'off' } and restoring backups was set to #{ restore_backup ? 'on' : 'off' }"
+    def backups_activate restore_backup=true, fetch_backup=true
+      backups_toggle_setting(restore_backup, fetch_backup)
+    end
+
+    def backups_deactivate restore_backup=false, fetch_backup=false
+      backups_toggle_setting(restore_backup, fetch_backup)
+    end
+
+    def backups_load status_hash={}
+      old_role, old_env, backup_env = @options['role'], @options['env'], @config['cheftacular']['backup_config']['global_backup_environ']
 
-      case ARGV[1]
-      when 'activate'   then restore_backup, fetch_backup = true, true
-      when 'deactivate' then restore_backup, fetch_backup = false, false
+      if backup_env != @options['env']
+        @config['initializer'].initialize_data_bags_for_environment(backup_env, false, ['addresses', 'server_passwords'])
+        @config['initializer'].initialize_passwords backup_env
       end
 
-      puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups is now set to #{ fetch_backup ? 'on' : 'off' } and restoring backups is now set to #{ restore_backup ? 'on' : 'off' }"
+      @options['role'] = @config['cheftacular']['backup_config']['global_backup_role_name']
+      @options['env']  = backup_env
 
-      @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']   = fetch_backup
-      @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups'] = restore_backup
+      puts "Deploying to backup master to force refresh of the ssh keys..." 
+      #@config['action'].deploy
 
-      @config['ChefDataBag'].save_config_bag
+      @options['role'] = old_role
+      @options['env']  = old_env
 
-      if force_fetch_and_restore
-        nodes = @config['getter'].get_true_node_objects true
+      target_db_primary = @config['getter'].get_db_primary_node
 
-        db_primary_nodes = @config['parser'].exclude_nodes( nodes, [{ unless: 'role[db_primary]' }, { if: { not_env: 'production' } }])
+      args_config = [
+        { unless: "role[#{ @config['cheftacular']['backup_config']['global_backup_role_name'] }]" },
+        { if: { not_env: backup_env } }
+      ]
 
-        backup_slave_local_ip = @config['cheftacular']['backup_server']
+      backup_master          = @config['parser'].exclude_nodes( nodes, args_config, true)
+      backup_master_local_ip = @config['getter'].get_address_hash(backup_master.first.name)['priv']
 
-        if backup_slave_local_ip == 'first_production_slave'
-          backup_slave = @config['parser'].exclude_nodes( nodes, [{ unless: 'role[db_slave]' }, { if: { not_env: 'production' } }], true)
+      options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
 
-          backup_slave_local_ip = @config['getter'].get_address_hash(backup_slave.first.hostname)['priv']
-        end
+      on ( backup_master.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
+        n = get_node_from_address(nodes, host.hostname)
 
-        options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = set_local_instance_vars
+        puts("Beginning latest db_fetch_and_check for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
 
-        on ( db_primary_nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
-          n = get_node_from_address(nodes, host.hostname)
+        status_hash['latest_backup'] = start_db_check_and_fetch( n.name, n.public_ipaddress, options, locs, cheftacular, passwords)
+      end
 
-          puts("Beginning db fetch_and_restore for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
+      return false unless status_hash['latest_backup']['file_check']
 
-          start_db_fetch_and_restore( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, backup_slave_local_ip)
-        end
+      on ( target_db_primary.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
+        n = get_node_from_address(nodes, host.hostname)
+
+        puts("Beginning db_backup_fetch for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
+
+        start_db_backup_fetch( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, backup_master_local_ip, status_hash['latest_backup'])
+      end
 
-        @config['action'].migrate
-      else
-        puts "Triggering deploy on databases to refresh backup setting..."
+      backups_run(nodes)
+    end
+
+    def backups_run
+      target_db_primary      = @config['getter'].get_db_primary_node
+      applications_as_string = @config['getter'].get_repo_names_for_repositories.keys.join(',')
+      env_pg_pass            = @config[@options['env']]['chef_passwords_bag_hash']['pg_pass']
+
+      options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
+
+      on ( target_db_primary.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ) do |host|
+        n = get_node_from_address(nodes, host.hostname)
 
-        @options['role'] = 'db_primary'
+        puts("Beginning db_backup_run for #{ n.name } (#{ n.public_ipaddress }) for env #{ options['env'] }") unless options['quiet']
 
-        @config['action'].deploy
+        start_db_backup_run( n.name, n.public_ipaddress, options, locs, cheftacular, passwords, applications_as_string, env_pg_pass )
       end
     end
+
+    def backups_toggle_setting restore_backup, fetch_backup
+      initial_fetch_backup   = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']
+      initial_restore_backup = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups']
+
+      puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups was set to " +
+        "#{ initial_fetch_backup ? 'on' : 'off' } and restoring backups was set to #{ initial_restore_backup ? 'on' : 'off' }"
+
+      puts "For #{ @options['env'] } (sub-env: #{ @options['sub_env'] }) fetch backups is now set to " +
+        "#{ fetch_backup ? 'on' : 'off' } and restoring backups is now set to #{ restore_backup ? 'on' : 'off' }"
+
+      @config[@options['env']]['config_bag_hash'][@options['sub_env']]['fetch_backups']   = fetch_backup
+      @config[@options['env']]['config_bag_hash'][@options['sub_env']]['restore_backups'] = restore_backup
+
+      @config['ChefDataBag'].save_config_bag
+
+      puts "Triggering deploy on databases to refresh backup setting..."
+
+      @options['role'] = 'db_primary'
+
+      @config['action'].deploy
+    end
   end
 end
 
 module SSHKit
   module Backend
     class Netssh
-      def start_db_fetch_and_restore name, ip_address, options, locs, cheftacular, passwords, global_backup_ip, out=[]
-        log_loc, timestamp = set_log_loc_and_timestamp(locs)
+      def start_db_check_and_fetch name, ip_address, options, locs, cheftacular, passwords, out=[], return_hash={ 'file_check' => false }
+        base_dir = cheftacular['backup_config']['global_backup_path']
 
-        puts("Generating pg_restore log file for #{ name } (#{ ip_address }) at #{ log_loc }/#{ name }-fetch-and-restore-#{ timestamp }.txt") unless options['quiet']
+        if !sudo_test( passwords[ip_address], base_dir ) #true if dir exists
+          puts "#{ name } (#{ ip_address }) cannot run #{ __method__ } as there is no directory at #{ base_dir }!"
 
-        cheftacular['db_primary_backup_database_stacks'].each do |target_database_stack|
-          compile_database_backups_hash(cheftacular, target_database_stack, global_backup_ip).each_pair do |app_name, app_hash|
+          return return_hash
+        end
 
-            out << sudo_capture( passwords[ip_address], 'scp', "#{ cheftacular['deploy_user'] }@#{ app_hash['backup_server'] }:#{ location }", backup_dir )
+        target_dir = case cheftacular['backup_filesystem']
+                     when 'backup_gem'
+                       backup_gem_dir_sort passwords[ip_address], cheftacular, base_dir
+                     when 'raw'
+                       File.join( base_dir, sudo_capture( passwords[ip_address], :ls, base_dir ).split(' ').last )
+                     else
+                       raise "#{ __method__ } does not currently support the #{ cheftacular['backup_filesystem'] } backup strategy at this time"
+                     end
+
+        return_hash['file_check'] = true
+        return_hash['file_path']  = [target_dir].flatten.last
+        return_hash['file_dir']   = case cheftacular['backup_filesystem']
+                                    when 'backup_gem' then target_dir.first
+                                    when 'raw'        then base_dir
+                                    end
+
+        return_hash
+      end
 
-            puts(out.last) if options['output'] || options['verbose']
+      def start_db_backup_fetch name, ip_address, options, locs, cheftacular, passwords, backup_master_local_ip, backup_hash, out=[]
+        if sudo_test( passwords[ip_address], backup_path ) #true if dir exists
+          puts "#{ name } (#{ ip_address }) already has the backup at #{ backup_path }, skipping #{ __method__ }..."
 
-            commands = [
-              "pg_restore --verbose --clean --no-acl --no-owner -j 4 -h localhost -U #{ cheftacular['deploy_user'] } -d #{ app_hash['repo_name'] }_#{ options['env'] } #{ app_hash['restore_backup_file_name'] }",
-              "service postgresql restart"
-            ]
+          return true
+        end
 
-            commands.each do |command|
-              out << sudo_capture( passwords[ip_address], command )
+        sudo_execute( passwords[ip_address], :mkdir, '-p', backup_hash['file_dir'] )
 
-              puts(out.last) if options['output'] || options['verbose']
-            end
-          end
-        end
+        sudo_execute( passwords[ip_address], :chown, "#{ cheftacular['deploy_user'] }:#{ cheftacular['deploy_user'] }", backup_hash['file_dir'] )
+
+        sudo_execute( passwords[ip_address], :chmod, cheftacular['backup_config']['backup_dir_mode'], backup_hash['file_dir'] )
 
-        #TODO fetch and restore other database types
+        execute( :scp, "#{ cheftacular['deploy_user'] }@#{ backup_master_local_ip }:#{ backup_hash['file_path'] }", backup_hash['file_dir'] )
 
-        ::File.open("#{ log_loc }/#{ name }-fetch-and-restore-#{ timestamp }.txt", "w") { |f| f.write(out.join("\n").scrub_pretty_text) } unless options['no_logs']
+        puts "Finished transferring #{ backup_hash['file_path'] } to #{ name }(#{ ip_address })..."
+      end
 
-        puts "Succeeded fetch and restore of #{ name } (#{ ip_address })"
+      def start_db_backup_run name, ip_address, options, locs, cheftacular, passwords, applications_as_string, env_pg_pass
+        puts "Beginning backup run on #{ name } (#{ ip_address }), this command may take a while to complete..."
+        case cheftacular['backup_filesystem']
+        when 'backup_gem'
+          #'ruby /root/backup_management.rb /mnt/postgresbackups/backups ENVIRONMENT APPLICATIONS PG_PASS > /root/restore.log 2>&1'
+          command = cheftacular['backup_config']['backup_load_command']
+          command = command.gsub('ENVIRONMENT', options['env']).gsub('APPLICATIONS', applications_as_string).gsub('PG_PASS', env_pg_pass)
 
-        [out.join("\n"), timestamp]
+          sudo_execute( passwords[ip_address], command )
+        when 'raw'
+        end
+
+        puts "Finished executing backup command on #{ name } (#{ ip_address })"
       end
 
-      def compile_database_backups_hash cheftacular, target_database_stack, global_backup_ip, ret={}
-        cheftacular['repositories'].each_pair do |app_name, app_hash|
-          ret[app_name] = app_hash if app_hash['restore_backup_file_name'] && app_hash['database'] == target_database_stack
-          ret[app_name]['backup_server'] = global_backup_ip unless ret[app_name]['backup_server']
+      def backup_gem_dir_sort password, cheftacular, base_dir
+        timestamp_dirs, check_dirs, target_dir = [], [], ''
+
+        dirs = sudo_capture( password, :ls, base_dir )
+
+        dirs.split(' ').each do |timestamp_dir|
+          next if timestamp_dir == '.' || timestamp_dir == '..'
+
+          timestamp_dirs << timestamp_dir
+        end
+
+        timestamp_dirs.each do |dir|
+          if check_dirs.empty?
+            check_dirs << dir
+            target_dir  = dir
+          else
+            check_dirs.each do |cdir|
+              target_dir = dir if Date.parse(dir) >= Date.parse(target_dir)
+            end
+          end
         end
 
-        ret
+        target_file = sudo_capture( password, :ls, File.join(base_dir, target_dir) ).split(' ').last
+
+        [ File.join(base_dir, target_dir), File.join( target_dir, target_file )]
       end
     end
   end

data/lib/cheftacular/stateless_actions/check_cheftacular_yml_keys.rb

@@ -13,6 +13,15 @@ class Cheftacular
     def check_cheftacular_yml_keys out=[], exit_on_missing=false, warn_on_missing=false
       base_message = "Your cheftacular.yml is missing the key KEY, its default value is being set to DEFAULT for this run."
 
+      #############################2.7.0################################################
+
+      unless @config['cheftacular'].has_key?('backup_config')
+        #backup_config:global_backup_role_name
+        base_message.gsub('KEY', 'backup_config').gsub('DEFAULT', 'nil')
+
+        warn_on_missing = true
+      end
+
       #############################2.6.0################################################
       unless @config['cheftacular'].has_key?('route_dns_changes_via')
         puts base_message.gsub('KEY', 'route_dns_changes_via').gsub('DEFAULT', @options['preferred_cloud'])