cheftacular 2.10.2 → 2.11.0

data/lib/cheftacular/actions/check.rb

@@ -3,11 +3,13 @@ class Cheftacular
     def check
       @config['documentation']['action'][__method__] ||= {}
       @config['documentation']['action'][__method__]['long_description']  = [
-        "`cft check` Checks the commits for all servers for a repository (for an environment) and returns them in a simple chart. " +
+        "`cft check [all]` Checks the commits for all servers for a repository (for an environment) and returns them in a simple chart. " +
         "Also shows when these commits were deployed to the server.",
         [
           "    1. If the node has special repository based keys from TheCheftacularCookbook, this command will also display information " +
-          "about the branch and organization currently deployed to the node(s)."
+          "about the branch and organization currently deployed to the node(s).",
+
+          "    2. If the all argument is provided, all repositories will be checked for the current environment"
         ]
       ]
       @config['documentation']['action'][__method__]['short_description'] = "Checks the branches currently deployed to an env for your repo"
@@ -15,41 +17,71 @@ class Cheftacular
   end
 
   class Action
-    def check commit_hash={}, have_revisions=false, have_changed_orgs=false
+    def check commit_hash={}, have_revisions=false, have_changed_orgs=false, fetch_all_repository_data=false, headers=[], deployment_args={ in: :parallel }
       @config['filesystem'].cleanup_file_caches('current-nodes')
+
+      fetch_all_repository_data = ARGV[1] == 'all'
       
-      nodes = @config['getter'].get_true_node_objects
+      nodes = @config['getter'].get_true_node_objects(fetch_all_repository_data)
+
+      nodes = @config['parser'].exclude_nodes(nodes, [{ if: { not_env: @options['env'] } }])
+
+      repositories_to_check = @config['getter'].get_repo_names_for_repositories
+      repositories_to_check = repositories_to_check.reject { |key,val| key != @options['repository'] } unless fetch_all_repository_data 
+
+      deployment_args = { in: :groups, limit: 2, wait: 5 } if fetch_all_repository_data
 
       #this must always precede on () calls so they have the instance variables they need
       options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
 
-      on ( nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ), in: :parallel do |host|
+      on ( nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ), deployment_args do |host|
         n = get_node_from_address(nodes, host.hostname)
 
-        puts("Beginning commit check run for #{ n.name } (#{ n.public_ipaddress }) on role #{ options['role'] }") unless options['quiet']
+        puts("Beginning commit check run for #{ n.name } (#{ n.public_ipaddress })...") unless options['quiet']
 
-        commit_hash[n.name]                      = start_commit_check( n.name, n.public_ipaddress, options, locs, cheftacular)
+        repositories_to_check.each_pair do |repo, repo_hash|
+          next unless n.run_list.include?("role[#{ repo_hash['role'] }]")
 
-        if n.normal_attributes.has_key?(options['repository'])
-          commit_hash[n.name]['branch']       = n.normal_attributes[options['repository']]['repo_branch'] if n.normal_attributes[options['repository']].has_key?('repo_branch')
-          commit_hash[n.name]['organization'] = n.normal_attributes[options['repository']]['repo_group']  if n.normal_attributes[options['repository']].has_key?('repo_group')
-        end
+          options['repository'] = repo
+          commit_hash[n.name] ||= {}
+
+          commit_hash[n.name][repo] = start_commit_check( n.name, n.public_ipaddress, options, locs, cheftacular, passwords)
+
+          next if commit_hash[n.name][repo].nil?
 
-        have_revisions    = true if commit_hash[n.name].has_key?('branch')
-        have_changed_orgs = true if commit_hash[n.name].has_key?('organization')
+          if n.normal_attributes.has_key?(options['repository'])
+            commit_hash[n.name][repo]['branch']       = n.normal_attributes[options['repository']]['repo_branch'] if n.normal_attributes[options['repository']].has_key?('repo_branch')
+            commit_hash[n.name][repo]['organization'] = n.normal_attributes[options['repository']]['repo_group']  if n.normal_attributes[options['repository']].has_key?('repo_group')
+          end
+
+          have_revisions    = true if commit_hash[n.name].has_key?(repo) && commit_hash[n.name][repo].has_key?('branch')
+          have_changed_orgs = true if commit_hash[n.name].has_key?(repo) && commit_hash[n.name][repo].has_key?('organization')
+
+          sleep 5
+        end
       end
 
-      puts "\n#{ 'name'.ljust(21) }#{ 'deployed_on'.ljust(22) } #{ 'commit'.ljust(40) } #{'revision'.ljust(29) if have_revisions } #{'organization'.ljust(30) if have_changed_orgs }"
+      headers << "\n#{ 'name'.ljust(20) }"
+      headers << 'repository'.ljust(30,'_')   if repositories_to_check.length > 1
+      headers << "#{ 'deployed_on'.ljust(22) } #{ 'commit'.ljust(40) }"
+      headers << 'revision'.ljust(21)     if have_revisions
+      headers << 'organization'.ljust(30) if have_changed_orgs
+
+      puts headers.join(' ')
       nodes.each do |n|
-        unless commit_hash[n.name]['name'].blank?
-          out  = []
-          out << n.name.ljust(20, '_')
-          out << commit_hash[n.name]['time'].ljust(21)
-          out << commit_hash[n.name]['name'].ljust(39)
-          out << commit_hash[n.name]['branch'].ljust(29)       if commit_hash[n.name].has_key?('branch')
-          out << commit_hash[n.name]['organization'].ljust(30) if commit_hash[n.name].has_key?('organization') 
-
-          puts out.join(' ')
+        next if commit_hash[n.name].nil?
+        repositories_to_check.each_pair do |repo, repo_hash|
+          if commit_hash[n.name].has_key?(repo) && !commit_hash[n.name][repo].nil?
+            out  = []
+            out << n.name.ljust(20, '_')
+            out << repo.ljust(30) if repositories_to_check.length > 1
+            out << commit_hash[n.name][repo]['time'].ljust(22)
+            out << commit_hash[n.name][repo]['name'].ljust(40)
+            out << commit_hash[n.name][repo]['branch'].ljust(21, '_')  if commit_hash[n.name][repo].has_key?('branch')
+            out << commit_hash[n.name][repo]['organization'].ljust(30) if commit_hash[n.name][repo].has_key?('organization')
+
+            puts out.join(' ')
+          end
         end
       end
     end
@@ -59,7 +91,7 @@ end
 module SSHKit
   module Backend
     class Netssh
-      def start_commit_check name, ip_address, options, locs, cheftacular, out={'name'=>'', 'time'=> ''}
+      def start_commit_check name, ip_address, options, locs, cheftacular, passwords, out={'name'=>'', 'time'=> ''}
         app_loc = "#{ cheftacular['base_file_path'] }/#{ options['repository'] }/releases"
         
         if test("[ -d #{ app_loc } ]") #true if file exists
@@ -68,6 +100,8 @@ module SSHKit
 
             out['time'] = Time.parse(capture( :stat, out['name'], '--printf=%y' )).strftime('%Y-%m-%d %I:%M:%S %p')
           end
+        else
+          return nil
         end
 
         out

data/lib/cheftacular/actions/deploy.rb

@@ -26,16 +26,18 @@ class Cheftacular
   end
 
   class Action
-    def deploy
-      nodes = @config['getter'].get_true_node_objects false, true #when this is run in scaling we'll need to make sure we deploy to new nodes
+    def deploy deployment_args={ in: :groups, limit: 6, wait: 5 }
+      nodes = @config['getter'].get_true_node_objects(false) #when this is run in scaling we'll need to make sure we deploy to new nodes
 
       nodes = @config['parser'].exclude_nodes( nodes, [{ if: "role[#{ @options['negative_role'] }]" }]) if @options['negative_role']
       
       #this must always precede on () calls so they have the instance variables they need
       options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
 
+      deployment_args = { in: :groups, limit: 10, wait: 5 } if @options['env'] == 'production'
+
       #on is namespaced to SSHKit::Backend::Netssh.on 
-      on ( nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ), in: :groups, limit: 6, wait: 5 do |host|
+      on ( nodes.map { |n| @config['cheftacular']['deploy_user'] + "@" + n.public_ipaddress } ), deployment_args do |host|
         n = get_node_from_address(nodes, host.hostname)
 
         puts("Beginning chef client run for #{ n.name } (#{ n.public_ipaddress }) on role #{ options['role'] }") unless options['quiet']

data/lib/cheftacular/auditor.rb

@@ -40,10 +40,11 @@ class Cheftacular
       ret_hash
     end
 
-    def compile_audit_hash_entry_as_array audit_hash, entry_number=0, ret_array=[]
+    def compile_audit_hash_entry_as_array audit_hash, entry_number=0, ret_array=[], directory_content=''
+      directory_content = " (#{ audit_hash['directory'] })" if audit_hash.has_key?('directory')
+
       ret_array << "#{ (entry_number.to_s + '. ') unless entry_number == 0 }#{ audit_hash['command'] }"
-      ret_array << "  Hostname:  #{ audit_hash['hostname'] }"
-      ret_array << "  Directory: #{ audit_hash['directory'] }"       if audit_hash.has_key?('directory')
+      ret_array << "  Hostname:  #{ audit_hash['hostname'] }#{ directory_content }"
       ret_array << "  Arguments: #{ audit_hash['arguments'] }"       if !audit_hash['arguments'].nil? && !audit_hash['arguments'].empty?
       ret_array << "  Options:   #{ audit_hash['options'].to_hash }" unless audit_hash['options'].empty?
       

data/lib/cheftacular/cheftacular.rb

@@ -18,6 +18,7 @@ require 'rbconfig'
 require 'fog'
 require 'socket'
 require 'net/http'
+require 'net/ssh'
 require 'timeout'
 require 'slack-notifier'
 require 'cloudflare'

data/lib/cheftacular/cloud_provider.rb

@@ -6,18 +6,23 @@ class Cheftacular
     end
 
     #public address, private address
-    def parse_addresses_from_server_create_hash server_hash
-      ap server_hash
-      case @options['preferred_cloud']
-      when 'rackspace'
-        [server_hash['ipv4_address'], server_hash['addresses']['private'][0]['addr']]
-      when 'digitalocean'
-        [
-          server_hash['networks']['v4'].select { |hash| hash['type'] == 'public' }.first['ip_address'],
-          server_hash['networks']['v4'].select { |hash| hash['type'] == 'private' }.first['ip_address']
-        ]
-      else raise "CRITICAL! Encountered unsupported preferred cloud #{ @options['preferred_cloud'] }"
-      end
+    def parse_addresses_from_server_create_hash cloud_server_hash, ret_array=[]
+      ap(cloud_server_hash) if @options['verbose']
+      
+      ret_array = case @options['preferred_cloud']
+                  when 'rackspace'
+                    [cloud_server_hash['ipv4_address'], cloud_server_hash['addresses']['private'][0]['addr']]
+                  when 'digitalocean'
+                    [
+                      cloud_server_hash['networks']['v4'].select { |hash| hash['type'] == 'public' }.first['ip_address'],
+                      cloud_server_hash['networks']['v4'].select { |hash| hash['type'] == 'private' }.first['ip_address']
+                    ]
+                  else raise "CRITICAL! Encountered unsupported preferred cloud #{ @options['preferred_cloud'] }"
+                  end
+
+      raise "CRITICAL! Unable to capture addresses for the server!" if ret_array[0].nil? || ret_array[1].nil?
+
+      ret_array
     end
 
     def parse_server_root_password_from_server_create_hash server_hash, real_node_name

data/lib/cheftacular/file_system.rb

@@ -189,6 +189,26 @@ class Cheftacular
       File.read(File.expand_path("#{ @config['locs']['root'] }/Gemfile"))[/#{ gem_name }.*([\d\.]+)/][/([\d\.]+)/]
     end
 
+    def scrub_from_known_hosts target
+      puts "Clearing #{ target } from known_hosts file."
+      case CONFIG['host_os']
+      when /mswin|windows/i
+        puts "#{ __method__ } does not support this operating system at this time"
+      when /linux|arch/i
+        puts "#{ __method__ } does not support this operating system at this time"
+      when /sunos|solaris/i
+        puts "#{ __method__ } does not support this operating system at this time"
+      when /darwin/i
+        #Removes the entire line containing the string
+        `sed -i '' "s/#{ target }.*//g" ~/.ssh/known_hosts`
+
+        #remove empty lines
+        `sed -i '' "/^$/d" ~/.ssh/known_hosts`
+      else
+        puts "#{ __method__ } does not support this operating system at this time"
+      end
+    end
+
     private
     def current_file_path file_name, use_timestamp=true
       File.join( @config['locs']['app-root'], 'tmp', @config['helper'].declassify, ( use_timestamp ? "#{ Time.now.strftime("%Y%m%d") }-#{ file_name }" : file_name ))

data/lib/cheftacular/getter.rb

@@ -10,7 +10,7 @@ class Cheftacular
     end
 
     #[TODO] if ridley changes its parsing strategy
-    def get_true_node_objects get_all_nodes=false, get_new_nodes=false
+    def get_true_node_objects get_all_nodes=false
       nodes, all_nodes, names, iter_arr, file_cache_nodes, h = [],[],[],[],[],{}
 
       @config['chef_nodes'] = @config['ridley'].node.all

data/lib/cheftacular/helper.rb

@@ -126,16 +126,6 @@ class Cheftacular
       @config['dummy_sshkit'].set_log_loc_and_timestamp @config['locs']
     end
 
-    def knife_bootstrap_command
-      address  = @options['address']
-      user     = @config['cheftacular']['deploy_user']
-      password = @config['server_passwords'][@options['address']]
-      nodename = @options['node_name']
-      chef_ver = @config['cheftacular']['chef_version'].to_i >= 12 ? '12.4.0' : '11.16.4'
-
-      "knife bootstrap #{ address } -x #{ user } -P #{ password } -N #{ nodename } --sudo --use-sudo-password --bootstrap-version #{ chef_ver }"
-    end
-
     #the documentation hashes must be populated *before* this method runs for it to return anything!
     def compile_documentation_lines mode, out=[]
       doc_arr = case mode
@@ -250,24 +240,6 @@ class Cheftacular
       master_hash
     end
 
-    def install_rvm_sh_file out=[]
-      puts("Starting rvm.sh installation...") unless @options['quiet']
-
-      commands = [
-        "#{ @config['helper'].sudo(@options['address']) } mv /home/deploy/rvm.sh /etc/profile.d",
-        "#{ @config['helper'].sudo(@options['address']) } chmod 755 /etc/profile.d/rvm.sh",
-        "#{ @config['helper'].sudo(@options['address']) } chown root:root /etc/profile.d/rvm.sh"
-      ]
-
-      out << `scp -oStrictHostKeyChecking=no #{ @config['locs']['cheftacular-lib-files'] }/rvm.sh #{ @config['cheftacular']['deploy_user'] }@#{ @options['address'] }:/home/#{ @config['cheftacular']['deploy_user'] }`
-
-      commands.each do |command|
-        out << `ssh -t -oStrictHostKeyChecking=no #{ @config['cheftacular']['deploy_user'] }@#{ @options['address'] } "#{ command }"`
-      end
-
-      puts("Completed rvm.sh installation into /etc/profile.d/rvm.sh") unless @options['quiet']
-    end
-
     def send_log_bag_hash_slack_notification logs_bag_hash, method, on_failing_exit_status_message=''
       if @config['cheftacular']['slack']['webhook']
         logs_bag_hash.each_pair do |key, hash|
@@ -302,6 +274,47 @@ class Cheftacular
                                                     @config['helper'].fetch_remote_version
                                                   end
     end
+
+    def return_options_as_hash options_array, return_hash={}
+      options_array.each do |key|
+        return_hash[key] = @options[key]
+      end
+
+      return_hash
+    end
+
+    def check_if_possible_repo_state repo_state_hash, git_output=''
+      revision_to_check = repo_state_hash.has_key?('revision')            ? repo_state_hash['revision']            : nil
+      org_name_to_check = repo_state_hash.has_key?('deploy_organization') ? repo_state_hash['deploy_organization'] : @config['cheftacular']['TheCheftacularCookbook']['organization_name']
+     
+      revision_to_check = nil if revision_to_check == '<use_default>'
+
+      @config['cheftacular']['TheCheftacularCookbook']['chef_environment_to_app_repo_branch_mappings'].each_pair do |chef_env, app_env|
+        revision_to_check = app_env if @options['env'] == chef_env && revision_to_check.nil?
+      end
+
+      git_output = `git ls-remote --heads git@github.com:#{ org_name_to_check }/#{ @options['repository'] }.git`
+
+      if git_output.blank?
+        puts "! The remote organization #{ org_name_to_check } does not have the repository: #{ @options['repository'] }! Please verify your repositories and try again"
+
+        exit
+      elsif !git_output.include?(revision_to_check)
+        puts "! The remote organization #{ org_name_to_check } does not have a revision / branch #{ revision_to_check } for repository: #{ @options['repository'] } !"
+
+        sorted_heads = git_output.scan(/refs\/heads\/(.*)/).flatten.uniq.sort_by { |head| compare_strings(revision_to_check, head)}
+
+        puts "The closest matches for #{ revision_to_check } are:"
+        puts "    #{ sorted_heads.at(0) }"
+        puts "    #{ sorted_heads.at(1) }"
+        puts "    #{ sorted_heads.at(2) }"
+        puts "    #{ sorted_heads.at(3) }\n"
+
+        puts "Please verify the correct revision / branch and run this command again."
+        
+        exit
+      end
+    end
   end
 end
 

data/lib/cheftacular/initializer.rb

@@ -168,7 +168,7 @@ class Cheftacular
         end
 
         # client-list
-        opts.on('-W', '--with-priv', "On client-list this will show each server's private addresses") do
+        opts.on('-W', '--with-priv', "On cft client_list this will show each server's private addresses") do
           @options['with_private'] = true
         end 
 
@@ -243,7 +243,8 @@ class Cheftacular
     end
 
     def initialize_queues
-      @config['slack_queue'] ||= []
+      @config['slack_queue']           ||= []
+      @config['server_creation_queue'] ||= []
     end
 
     def initialize_yaml_configuration
@@ -319,6 +320,7 @@ class Cheftacular
       locs['chef-log']              = File.expand_path("#{ locs['root']}/log")             unless locs['chef-log']
       locs['app-tmp']               = File.expand_path("#{ locs['app-root']}/tmp")
       locs['examples']              = File.expand_path("../../../examples", __FILE__)
+      locs['doc']                   = File.expand_path("../../../doc", __FILE__)
       locs['cheftacular-lib']       = File.expand_path("../..", __FILE__)
       locs['cheftacular-lib-files'] = locs['cheftacular-lib'] + '/cheftacular/files'
 
@@ -472,7 +474,7 @@ class Cheftacular
     end
 
     def initialize_directories
-      ['applog', 'deploy', 'failed-deploy', 'rolelog', 'rvm', 'stashedlog'].each do |sub_log_directory|
+      ['applog', 'deploy', 'failed-deploy', 'rolelog', 'rvm', 'server-setup', 'stashedlog'].each do |sub_log_directory|
         FileUtils.mkdir_p File.join( @config['locs']['chef-log'], sub_log_directory )
       end
 

data/lib/cheftacular/parser.rb

@@ -113,6 +113,8 @@ class Cheftacular
         repo_state_hash['deploy_organization'] = nil
       end
 
+      @config['helper'].check_if_possible_repo_state(repo_state_hash) if @config['cheftacular']['git']['check_remote_for_branch_existence'] && !@config['helper'].running_on_chef_node?
+
       @config['helper'].slack_current_deploy_arguments unless @config['cheftacular']['slack']['notify_on_deployment_args'].blank?
 
       @config['ChefDataBag'].save_config_bag 
@@ -131,7 +133,7 @@ class Cheftacular
     end
 
     #parse nodes out of array based on hash, ex: [{ unless: 'role[rails]'}, {if: 'role[worker]'}, { if: { run_list: 'role[db]', role: 'pg_data' } }]
-    def exclude_nodes nodes, statement_arr, only_one_node = false, ret_arr = []
+    def exclude_nodes nodes, statement_arr, only_one_node=false, ret_arr=[]
       nodes.each do |n|
         go_next = false
 
@@ -190,17 +192,19 @@ class Cheftacular
       end
     end
 
-    def parse_to_dns dns_string
-      raise "Unable to parse DNS without @option['node_name'] for #{ dns_string }!" if dns_string.include?('NODE_NAME') && @options['node_name'].nil?
+    def parse_to_dns dns_string, node_name=''
+      raise "Unable to parse DNS without node_name for #{ dns_string }!" if dns_string.include?('NODE_NAME') && node_name.blank?
       raise "Unable to parse DNS without a tld set in the config bag for #{ @options['env'] }!" if dns_string.include?('ENV_TLD') && @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld'].nil?
 
-      dns_string.gsub('NODE_NAME', @options['node_name']).gsub('ENV_TLD', @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld'])
+      dns_string.gsub('NODE_NAME', node_name).gsub('ENV_TLD', @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld'])
     end
 
-    def parse_repository_hash_from_string string
+    def parse_repository_hash_from_string string, checked_hashes={}
       @config['getter'].get_repo_names_for_repositories.each do |repository, repository_hash|
-        return repository_hash if string.include?(repository)
+        checked_hashes[repository_hash['role']] = @config['helper'].compare_strings(string, repository)
       end
+
+      return @config['cheftacular']['repositories'][ Hash[checked_hashes.sort_by { |key, val| val }].keys.first ]
     end
 
     def parse_location_alias string

data/lib/cheftacular/queue_master.rb

@@ -14,5 +14,29 @@ class Cheftacular
 
       @config['slack_queue'] = []
     end
+
+    def sync_server_hash_into_queue server_hash, server_queue='server_creation_queue', found_result=false
+      @config[server_queue].select { |hash| hash['node_name'] == server_hash['node_name']}.each do |queue_hash|
+        @config[server_queue][ @config[server_queue].index(queue_hash) ] = queue_hash.merge(server_hash)
+
+        found_result = true
+      end
+
+      @config[server_queue] << server_hash unless found_result
+
+      server_hash
+    end
+
+    def generate_passwords_for_each_server_hash_in_queue server_queue='server_creation_queue'
+      @config[server_queue].each do |queue_hash|
+        @config[server_queue][ @config[server_queue].index(queue_hash) ] = queue_hash.merge(
+          { 'deploy_password' => @config['helper'].gen_pass(@config['cheftacular']['server_pass_length']) }
+        )
+      end
+    end
+
+    def return_hash_from_queue queue_name, hash, hash_key
+      @config[queue_name].select { |queue_hash| queue_hash[hash_key] == hash[hash_key] }.first
+    end
   end
 end

data/lib/cheftacular/stateless_actions/backups.rb

@@ -49,7 +49,7 @@ class Cheftacular
       old_role, old_env, backup_env = @options['role'], @options['env'], @config['cheftacular']['backup_config']['global_backup_environ']
 
       if backup_env != @options['env']
-        @config['initializer'].initialize_data_bags_for_environment(backup_env, false, ['addresses', 'server_passwords'])
+        @config['initializer'].initialize_data_bags_for_environment(backup_env, false, ['addresses', 'server_passwords', 'logs'])
         @config['initializer'].initialize_passwords backup_env
       end
 
@@ -57,7 +57,7 @@ class Cheftacular
       @options['env']  = backup_env
 
       puts "Deploying to backup master to force refresh of the ssh keys..." 
-      #@config['action'].deploy
+      @config['action'].deploy
 
       @options['role'] = old_role
       @options['env']  = old_env