cheftacular 2.10.2 → 2.11.0

data/lib/cheftacular/stateless_actions/bootstrappers/{centos_bootstrap.rb → centos_bootstrap_from_queue.rb}

@@ -1,6 +1,6 @@
 class Cheftacular
   class StatelessAction
-    def redhat_bootstrap out=[]
+    def redhat_bootstrap_from_queue out=[]
       raise "Not yet implemented!"
     end
   end

data/lib/cheftacular/stateless_actions/bootstrappers/{coreos_bootstrap.rb → coreos_bootstrap_from_queue.rb}

@@ -1,6 +1,6 @@
 class Cheftacular
   class StatelessAction
-    def coreos_bootstrap out=[]
+    def coreos_bootstrap_from_queue out=[]
       raise "Not yet implemented!"
     end
   end

data/lib/cheftacular/stateless_actions/bootstrappers/{fedora_bootstrap.rb → fedora_bootstrap_from_queue.rb}

@@ -1,6 +1,6 @@
 class Cheftacular
   class StatelessAction
-    def fedora_bootstrap out=[]
+    def fedora_bootstrap_from_queue out=[]
       raise "Not yet implemented!"
     end
   end

data/lib/cheftacular/stateless_actions/bootstrappers/{redhat_bootstrap.rb → redhat_bootstrap_from_queue.rb}

@@ -1,6 +1,6 @@
 class Cheftacular
   class StatelessAction
-    def redhat_bootstrap out=[]
+    def redhat_bootstrap_from_queue out=[]
       raise "Not yet implemented!"
     end
   end

data/lib/cheftacular/stateless_actions/bootstrappers/ubuntu_bootstrap_from_queue.rb

@@ -0,0 +1,203 @@
+class Cheftacular
+  class StatelessActionDocumentation
+    def ubuntu_bootstrap_from_queue
+      @config['documentation']['stateless_action'][__method__] ||= {}
+      @config['documentation']['stateless_action'][__method__]['long_description'] = [
+        "`cft ubuntu_bootstrap_from_queue` This command will bring a fresh server to a state " +
+        "where chef-client can be run on it via `cft chef-bootstrap`. It should be noted that it is in "+
+        "this step where a server's randomized deploy_user sudo password is generated."
+      ]
+
+      @config['documentation']['stateless_action'][__method__]['short_description'] = '[Not Directly Callable]'
+    end
+  end
+
+  class StatelessAction
+    def ubuntu_bootstrap_from_queue threads=[], execution_hash_array=[]
+      raise "This action is not meant to be called directly!" if !@options['in_scaling'] && !@options['in_single_server_creation']
+
+      if `which sshpass`.empty?
+        raise "sshpass not installed! Please run brew install https://raw.github.com/eugeneoden/homebrew/eca9de1/Library/Formula/sshpass.rb (or get it from your repo for linux)"
+      end
+
+      @config['bootstrap_timestamp'] ||= Time.now.strftime("%Y%m%d%H%M%S")
+
+      @config['queue_master'].generate_passwords_for_each_server_hash_in_queue
+
+      execution_hash_array << compile_root_execute_hash
+      execution_hash_array << compile_deploy_execute_hash
+      execution_hash_array << compile_rvm_execute_hash           if @config['cheftacular']['install_rvm_on_boot']
+      execution_hash_array << compile_install_rvm_sh_file_hashes if @config['cheftacular']['install_rvm_on_boot']
+      execution_hash_array  = execution_hash_array.flatten(1)
+
+      @config['server_creation_queue'].each do |server_hash|
+        puts("#{ server_name_output(server_hash) }_Starting initial setup for server...") if @options['in_scaling']
+
+        threads << Thread.new { execute_execution_hash_array(server_hash, execution_hash_array) }
+      end
+
+      threads.each { |thread| thread.join }
+
+      @config['server_creation_queue'].each do |server_hash|
+        @config[@options['env']]['server_passwords_bag_hash']["#{ server_hash['address'] }-root-pass"]   = server_hash['client_pass']
+        @config[@options['env']]['server_passwords_bag_hash']["#{ server_hash['address'] }-deploy-pass"] = server_hash['deploy_password']
+        @config[@options['env']]['server_passwords_bag_hash']["#{ server_hash['address'] }-name"]        = server_hash['node_name']
+      end
+
+      @config['ChefDataBag'].save_server_passwords_bag unless @config['in_server_creation_queue']
+    end
+
+    private
+
+    def compile_root_execute_hash
+      deploy_user = @config['cheftacular']['deploy_user']
+
+      root_commands = [
+        "cd /home",
+        "adduser #{ deploy_user } --gecos \",,,,\" --disabled-password",
+        "echo #{ deploy_user }:NEW_DEPLOY_PASS | chpasswd",
+        "adduser #{ deploy_user } www-data",
+        "adduser #{ deploy_user } sudo",
+        "mkdir -p /home/#{ deploy_user }/.ssh",
+        "touch /home/#{ deploy_user }/.ssh/authorized_keys && touch /home/#{ deploy_user }/.ssh/known_hosts",
+        "chown -R #{ deploy_user }:www-data /home/#{ deploy_user }/.ssh",
+        'sed -i "s/StrictModes yes/StrictModes yes\nPasswordAuthentication no\nUseDNS no\nAllowUsers deploy postgres\n/" /etc/ssh/sshd_config'.gsub('deploy', deploy_user),
+        'sed -i "s/PermitRootLogin yes/PermitRootLogin no/" /etc/ssh/sshd_config'
+      ]
+
+      @config['default']['authentication_bag_hash']['authorized_keys'].each do |line|
+        root_commands << "echo \"#{ line }\" >> /home/#{ deploy_user }/.ssh/authorized_keys"
+      end
+
+      { 
+        run_as: 'ssh',
+        command: root_commands.join(' && ').insert(-1, ' && service ssh restart'),
+        retries: @config['cheftacular']['server_creation_tries'].to_i,
+        use_root_pass: true
+      }
+    end
+
+    def compile_deploy_execute_hash
+      sudo = "echo NEW_DEPLOY_PASS | sudo -S"
+
+      deploy_commands = [
+        "#{ sudo } apt-get update",
+        "#{ sudo } apt-get install curl #{ @config['cheftacular']['pre_install_packages'] } -y",
+        "#{ sudo } apt-get upgrade -y"
+      ]
+
+      if @config['cheftacular']['install_rvm_on_boot']
+        deploy_commands << "gpg --keyserver hkp://keys.gnupg.net --recv-keys #{ @config['cheftacular']['rvm_gpg_key'] }"
+        deploy_commands << "curl -L https://get.rvm.io | bash -s stable"
+      end
+
+      { run_as: 'ssh', command_array: deploy_commands }
+    end
+
+    def compile_rvm_execute_hash
+      rvm_source = "source /home/#{ @config['cheftacular']['deploy_user'] }/.rvm/bin/rvm &&"
+
+      final_commands = [
+        "#{ rvm_source } echo NEW_DEPLOY_PASS | rvmsudo -S rvm requirements",
+        "#{ rvm_source } rvm install #{ @config['cheftacular']['ruby_version'] }",
+        "#{ rvm_source } rvm alias create default #{ @config['cheftacular']['ruby_version'] }",
+        "#{ rvm_source } rvm gemset empty --force"
+      ]
+
+      final_commands << "#{ rvm_source } rvm install 1.9.3-p327" if @config['cheftacular']['chef_version'].to_i < 12
+
+      { run_as: 'ssh', command_array: final_commands }
+    end
+
+    def compile_install_rvm_sh_file_hashes ret_array=[]
+      sudo = "echo NEW_DEPLOY_PASS | sudo -S"
+
+      ssh_commands = [
+        "#{ sudo } mv /home/#{ @config['cheftacular']['deploy_user'] }/rvm.sh /etc/profile.d",
+        "#{ sudo } chmod 755 /etc/profile.d/rvm.sh",
+        "#{ sudo } chown root:root /etc/profile.d/rvm.sh"
+      ]
+
+      ret_array << { run_as: 'scp', upload: "#{ @config['locs']['cheftacular-lib-files'] }/rvm.sh", to: "/home/#{ @config['cheftacular']['deploy_user'] }"}
+      ret_array << { run_as: 'ssh', command: ssh_commands.join(' && ') }
+      ret_array
+    end
+
+    def execute_execution_hash_array server_hash, execution_array, output=[], create_logs=true, log_sub_dir='initial-setup'
+      execution_array.each do |execution_hash|
+        ssh_arguments  = [server_hash['address']]
+        ssh_arguments << ( execution_hash.has_key?(:use_root_pass) ? 'root' : @config['cheftacular']['deploy_user'] )
+        ssh_arguments << [{ password: server_hash['client_pass'] }] if execution_hash.has_key?(:use_root_pass)
+        begin
+          case execution_hash[:run_as]
+          when 'ssh'
+            if execution_hash.has_key?(:command)
+              Net::SSH.start(*ssh_arguments.flatten) do |ssh|
+                output << ssh.exec!(replace_placeholders_in_command(server_hash, execution_hash[:command]))
+              end
+            end
+
+            if execution_hash.has_key?(:command_array)
+              execution_hash[:command_array].each do |command|
+                puts("#{ server_name_output(server_hash) }_Preparing to execute #{ command }")
+
+                Net::SSH.start(*ssh_arguments.flatten) do |ssh|
+                  output << ssh.exec!(replace_placeholders_in_command(server_hash, command))
+                end
+              end
+            end
+          when 'scp'
+            if execution_hash.has_key?(:upload) && execution_hash.has_key?(:to)
+              puts("#{ server_name_output(server_hash) }_Preparing to upload #{ execution_hash[:upload] } to #{ execution_hash[:to] }")
+
+              Net::SCP.upload!(server_hash['address'], @config['cheftacular']['deploy_user'], execution_hash[:upload], execution_hash[:to])
+            end
+          when 'raw'
+            if execution_hash.has_key?(:command)
+              output << `#{ replace_placeholders_in_command(server_hash, execution_hash[:command]) }`
+            end
+
+            if execution_hash.has_key?(:command_array)
+              execution_hash[:command_array].each do |command|
+                puts("#{ server_name_output(server_hash) }_Preparing to execute #{ command }")
+
+                output << `#{ replace_placeholders_in_command(server_hash, command) }`
+              end
+            end
+          end
+        rescue Net::SSH::HostKeyMismatch => e
+          puts "#{ server_name_output(server_hash) }_Has a host key mismatch! Rewriting known_hosts file..."
+
+          @config['filesystem'].scrub_from_known_hosts(server_hash['address'])
+
+          sleep 15
+
+          retry
+        rescue StandardError => e
+          puts "#{ server_name_output(server_hash) }_@@@@@@@@@@@ Failing execution hash because of #{ e }!@@@@@@@@@@"
+          puts "#{ e.backtrace.join("\n") }" if @options['verbose']
+
+          if execution_hash.has_key?(:retries)
+            execution_hash[:retries] = execution_hash[:retries] -= 1
+            puts "There are #{ execution_hash[:retries] } tries left to evaluate the command."
+
+            sleep 60
+
+            raise "#{ server_name_output(server_hash) }_@@@@@@@@@@@ Unable to complete setup process!@@@@@@@@@@" if execution_hash[:retries] <= 0
+            retry
+          end
+        end
+      end
+
+      File.open("#{ @config['locs']['chef-log'] }/server-setup/#{ server_hash['node_name'] }-#{ log_sub_dir }-#{ @config['bootstrap_timestamp'] }.txt", 'a+') { |f| f.write(output.join("\n").scrub_pretty_text) }
+    end
+
+    def server_name_output server_hash
+      "#{ server_hash['node_name'] }".ljust(17,'_') + "#{ server_hash['address'] }".ljust(18,'_')
+    end
+
+    def replace_placeholders_in_command server_hash, command
+      command.gsub('NEW_DEPLOY_PASS', server_hash['deploy_password']).gsub('ADDRESS', server_hash['address']).gsub('CLIENT_PASS', server_hash['client_pass']).gsub('NODE_NAME', server_hash['node_name'])
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/bootstrappers/vyatta_bootstrap_from_queue.rb

@@ -0,0 +1,7 @@
+class Cheftacular
+  class StatelessAction
+    def vyatta_bootstrap_from_queue out=[]
+      raise "Not yet implemented!"
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/check_cheftacular_yml_keys.rb

@@ -16,6 +16,36 @@ class Cheftacular
     def check_cheftacular_yml_keys out=[], exit_on_missing=false, warn_on_missing=false
       base_message = "Your cheftacular.yml is missing the key KEY, its default value is being set to DEFAULT for this run."
 
+      #############################2.11.0################################################
+
+      unless @config['cheftacular'].has_key?('server_creation_tries')
+        @config['cheftacular']['server_creation_tries'] = 2
+      end
+
+      unless @config['cheftacular']['backup_config'].has_key?('db_primary_role')
+        puts base_message.gsub('KEY', 'backup_config:db_primary_role').gsub('DEFAULT', 'db_primary')
+
+        @config['cheftacular']['backup_config']['db_primary_role'] = 'db_primary'
+
+        warn_on_missing = true
+      end
+
+      unless @config['cheftacular'].has_key?('git')
+        puts(base_message.gsub('KEY', 'git').split(',').first + ', Please add the git high level key to your cheftacular.yml.')
+
+        @config['cheftacular']['git'] ||= {}
+
+        warn_on_missing = true
+      end
+
+      unless @config['cheftacular']['git'].has_key?('check_remote_for_branch_existence')
+        puts base_message.gsub('KEY', 'git:check_remote_for_branch_existence').gsub('DEFAULT', 'false')
+
+        @config['cheftacular']['git']['check_remote_for_branch_existence'] = false
+
+        warn_on_missing = true
+      end
+
       #############################2.10.0################################################
 
       unless @config['cheftacular'].has_key?('self_update_repository')

data/lib/cheftacular/stateless_actions/chef_bootstrap_from_queue.rb

@@ -0,0 +1,89 @@
+
+class Cheftacular
+  class StatelessActionDocumentation
+    def chef_bootstrap_from_queue
+      @config['documentation']['stateless_action'][__method__] ||= {}
+      @config['documentation']['stateless_action'][__method__]['long_description'] = [
+        "`cft chef_bootstrap_from_queue` allows you to register a node in the chef system, " + 
+        "remove any lingering data that may be associated with it and update the node's runlist if it has an entry in nodes_dir for its NODE_NAME.",
+
+        [
+          "    1. This command is part of the `cft full_bootstrap` command and cannot be called directly"
+        ]
+      ]
+
+      @config['documentation']['stateless_action'][__method__]['short_description'] = 'Bootstraps basic chef properties on the target server'
+    end
+  end
+
+  class StatelessAction
+    def chef_bootstrap_from_queue threads=[]
+      raise "This action is not meant to be called directly!" if !@options['in_scaling'] && !@options['in_single_server_creation']
+
+      #@config['stateless_action'].remove_client #just in case
+
+      execution_hash_array = compile_chef_bootstrap_commands
+
+      @config['bootstrap_timestamp'] ||= Time.now.strftime("%Y%m%d%H%M%S")
+
+      @config['server_creation_queue'].each do |server_hash|
+        puts("(#{ server_hash['node_name'] })[#{ server_hash['address'] }] Starting chef-client installation...") unless @options['quiet']
+
+        threads << Thread.new { execute_execution_hash_array(server_hash, execution_hash_array) }
+      end
+
+      threads.each { |thread| thread.join }
+
+      @options['force_yes'] = true # have the upload_nodes grab the new nodes
+
+      @config['stateless_action'].upload_nodes(true)
+    end
+
+    private
+
+    def compile_chef_bootstrap_commands final_command=[]
+      sudo = "echo NEW_DEPLOY_PASS | sudo -S"
+
+      if @config['cheftacular']['chef_version'].to_i >= 12
+        commands = [
+          "curl -L https://www.opscode.com/chef/install.sh > ~/chef-install.sh",
+          "#{ sudo } bash /home/#{ @config['cheftacular']['deploy_user'] }/chef-install.sh",
+          "rm ~/chef-install.sh"
+        ]
+
+        final_command << { run_as: 'ssh', command_array: commands }
+      end
+
+      final_command << knife_bootstrap_command
+      final_command << export_data_bag_key_file_command
+      final_command << move_data_bag_key_file_command
+
+      final_command
+    end
+
+    def knife_bootstrap_command
+      user     = @config['cheftacular']['deploy_user']
+      chef_ver = @config['cheftacular']['chef_version'].to_i >= 12 ? '12.4.0' : '11.16.4'
+
+      { run_as: 'raw', command: "knife bootstrap ADDRESS -x #{ user } -P NEW_DEPLOY_PASS -N NODE_NAME --sudo --use-sudo-password --bootstrap-version #{ chef_ver }" }
+    end
+
+    def export_data_bag_key_file_command
+      chef_loc, key_file, user, address = set_data_bag_key_variables
+
+      #"scp -oStrictHostKeyChecking=no #{ chef_loc }/#{ key_file } #{ user }@#{ address }:/home/#{ user }"
+      { run_as: 'scp', upload: "#{ chef_loc }/#{ key_file }", to: "/home/#{ user }" }
+    end
+
+    def move_data_bag_key_file_command
+      chef_loc, key_file, user, address = set_data_bag_key_variables
+
+      #"ssh -t -oStrictHostKeyChecking=no #{ user }@#{ address } \"#{ @config['helper'].sudo(address) } mv -f /home/#{ user }/#{ key_file } /etc/chef\""
+      { run_as: 'ssh', command: "echo NEW_DEPLOY_PASS | sudo -S mv -f /home/#{ user }/#{ key_file } /etc/chef" }
+    end
+
+    def set_data_bag_key_variables
+      [ @config['locs']['chef'], @config['cheftacular']['data_bag_key_file'], @config['cheftacular']['deploy_user'], 'ADDRESS' ]
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/cheftacular_yml_help.rb

@@ -4,28 +4,77 @@ class Cheftacular
     def cheftacular_yml_help
       @config['documentation']['stateless_action'][__method__] ||= {}
       @config['documentation']['stateless_action'][__method__]['long_description'] = [
-        "[NYI]`cft cheftacular_yml_help KEY` this command" +
+        "`cft cheftacular_yml_help KEY` this command" +
         "allows you to get help on the meaning of each key in your cheftacular.yml overall config.",
 
         [
-          "    1. This command can also by run with `cft yaml_help`."
+          "    1. This command can also by run with `cft yaml_help`.",
+
+          "    2. To examine nested keys, you can use colons inbetween the keys like cloud_authentication:rackspace:email"
         ]
       ]
 
-      @config['documentation']['stateless_action'][__method__]['short_description'] = '[NYI]Gives help on the keys in your cheftacular.yml'
+      @config['documentation']['stateless_action'][__method__]['short_description'] = 'Gives help on the keys in your cheftacular.yml'
 
       @config['documentation']['application'][__method__] = @config['documentation']['stateless_action'][__method__]
     end
+
+    alias_method :yml_help, :cheftacular_yml_help
+  end
+
+  class InitializationAction
+    def cheftacular_yml_help
+      
+    end
+
+    alias_method :yml_help, :cheftacular_yml_help
   end
 
   class StatelessAction
-    def cheftacular_yml_help command=''
-      raise "Not Yet Implemented"
+    def cheftacular_yml_help command='', key_nesting_array=[]
+      key_to_check = ARGV[1]
 
+      raise "This command requires a key to check the documentation, please enter one as the first argument" if key_to_check.nil?
+
+      key_to_check.split(':').each { |key| key_nesting_array << key }
+
+      doc_hash = YAML::load(ERB.new(IO.read(File.open(File.expand_path("#{ @config['locs']['doc'] }/cheftacular_yml_help.yml")))).result)
+
+      traverse_documentation_hash(doc_hash, key_nesting_array)
     end
 
-    alias_method :yaml_help, :cheftacular_yml_help
+    alias_method :yml_help, :cheftacular_yml_help
 
     private
+
+    def traverse_documentation_hash doc_hash, key_array, index=0
+      if doc_hash.has_key?(key_array[index])
+        if doc_hash[key_array[index]].class == Hash && key_array.length-1 == index
+          if doc_hash[key_array[index]]['key_description'].nil?
+            puts "Missing documentation for key #{ key_array[index] }!"
+          else
+            puts doc_hash[key_array[index]]['key_description']
+          end
+        elsif doc_hash[key_array[index]].class != Hash && key_array.length-1 == index
+          puts doc_hash[key_array[index]]
+        elsif doc_hash[key_array[index]].class == Hash && key_array.length-1 != index
+          traverse_documentation_hash(doc_hash[key_array[index]], key_array, index+1)
+        else
+          puts "You attempted to traverse the documentation with a value that was not a key to a hash (#{ key_array[index] }), the value of this key is:"
+
+          puts doc_hash[key_array[index]]
+          
+          puts "If this is not what you were searching for, please verify your config tree with `cft cheftacular_config display`"
+        end
+      elsif doc_hash.has_key?('STAR_MATCHER') && key_array.length-1 != index
+        traverse_documentation_hash(doc_hash['STAR_MATCHER'], key_array, index+1)
+      elsif doc_hash.has_key?('STAR_MATCHER') && key_array.length-1 == index
+        puts doc_hash['STAR_MATCHER']['key_description']
+      else
+        puts "The documentation hash does not have the key #{ key_array[index] }, the keys available here are:"
+
+        ap doc_hash.keys
+      end
+    end
   end
 end