cheftacular 2.1.2 → 2.2.0

data/lib/cheftacular/{getters.rb → getter.rb}

@@ -111,9 +111,9 @@ class Cheftacular
       ret
     end
 
-    def get_addresses_hash env='staging', ret_hash={}
+    def get_addresses_hash env='staging', ret_hash={}, mode=""
       @config[env]['addresses_bag_hash']['addresses'].each do |serv_hash|
-        ret_hash[serv_hash['public']] = { "dn" => serv_hash['dn'], "priv" => serv_hash['address'], "pub" => serv_hash['public'] }
+        ret_hash[serv_hash['public']] = serv_hash
         
       end if @config[env].has_key?('addresses_bag_hash')
 

data/lib/cheftacular/{helpers.rb → helper.rb}

@@ -10,14 +10,6 @@ class Cheftacular
       Cheftacular.to_s.underscore.dasherize
     end
 
-    #[TODO] Refactor to more accurate solution
-    def running_in_gem?
-      File.expand_path('.', __FILE__).split('/').include?('.rvm') || 
-        (File.exist?(File.expand_path("#{ Dir.getwd }/.ruby-gemset")) &&
-          File.read(File.expand_path("#{ Dir.getwd }/.ruby-gemset")) != 'hiplogiq-deploy')
-    end
-
-    #TODO add command check
     def is_command? command
       @config['action'].public_methods(false).include?(command.to_sym)
     end
@@ -30,7 +22,6 @@ class Cheftacular
       !@config['action'].public_methods(false).include?(command.to_sym) && !@config['stateless_action'].public_methods(false).include?(command.to_sym)
     end
 
-    #[TODO] Refactor to running_on_chef_node?
     def running_on_chef_node?
       Dir.entries('/etc').include?('chef')
     rescue StandardError => e
@@ -275,7 +266,7 @@ class Cheftacular
       end      
     end
 
-    def does_cheftacular_config_have? key_array, *args
+    def does_cheftacular_config_have? key_array
       cheftacular = @config['cheftacular']
       key_array   = [key_array] if key_array.is_a?(String)
       key_checks  = []
@@ -289,16 +280,16 @@ class Cheftacular
 
     def recursive_hash_check keys, hash
       if hash.has_key?(keys[0]) 
-        case hash[keys[0]].class
-        when Hash
+        case hash[keys[0]].class.to_s
+        when 'Hash'
           if !hash[keys[0]].empty?
             recursive_hash_check keys[1..keys.count-1], hash[keys[0]] 
           else
             return true
           end
-        when String
+        when 'String'
           return !hash[keys[0]].blank?
-        when Array
+        when 'Array'
           return !hash[keys[0]].empty?
         end
       else
@@ -306,6 +297,7 @@ class Cheftacular
       end
     end
 
+    #this must be in helpers because parser class is not yet loaded at the time this method is needed.
     def parse_node_name_from_client_file ret=""
       config = File.read(File.expand_path("#{ @config['locs']['chef'] }/client.rb"))
 

data/lib/cheftacular/{initializers.rb → initializer.rb}

@@ -356,8 +356,8 @@ class Cheftacular
       rescue StandardError => e
         msg = [
           "Please run this in the root of your application directory,",
-          " a ruby string to run commands against was not found in either your cheftacular.yml file or your .ruby-version file."
-        ]
+          "a ruby string to run commands against was not found in either your cheftacular.yml file or your .ruby-version file."
+        ].join(' ')
 
         @config['helper'].exception_output msg, e
       end
@@ -424,6 +424,7 @@ class Cheftacular
       @config['action_documentation']           = Cheftacular::ActionDocumentation.new(@options, @config)
       @config['stateless_action_documentation'] = Cheftacular::StatelessActionDocumentation.new(@options, @config)
       @config['dummy_sshkit']                   = SSHKit::Backend::Netssh.new(SSHKit::Host.new('127.0.0.1'))
+      @config['DNS']                            = Cheftacular::DNS.new(@options, @config)
     end
 
     def initialize_directories

data/lib/cheftacular/{parsers.rb → parser.rb}

@@ -195,5 +195,11 @@ class Cheftacular
 
       dns_string.gsub('NODE_NAME', @options['node_name']).gsub('ENV_TLD', @config[@options['env']]['config_bag_hash']['tld'])
     end
+
+    def parse_repository_hash_from_string string
+      @config['getter'].get_repo_names_for_repositories.each do |repository, repository_hash|
+        return repository_hash if string.include?(repository)
+      end
+    end
   end
 end

data/lib/cheftacular/stateless_actions/arguments.rb

@@ -46,13 +46,11 @@ class Cheftacular
 
         '1.  `-e|--except-role ROLE_NAME` will *prevent* any server with this role from being *deployed to* for the deploy command. Other commands will ignore this argument.',
 
-        '2.  `-z|--unset-revision` will unset a custom revision specified in the arg below and make the codebase utilize the default (staging branch for staging and master for production)',
+        '2.  `-z|--unset-revision` will unset a custom revision specified in the arg below and make the codebase utilize the default.',
 
         "3.  `-Z|--revision REVISION` will force the role you're deploying to to utilize the revision specified here. This can be a specific commit or a branch name.",
 
-        '    1. Note: The system does not check if the revision exists, if you pass a non-existent revision no one will be able to deploy to that role until -Z with a correction revision or -z is passed.',
-
-        '    2. *You will not be able to set a custom revision for beta environments.* The beta environments are tied to split-staging and splita/b/c/d respectively.',
+        '    1. Note: The system does not check if the revision exists, if you pass a non-existent revision no one will be able to deploy to that role until -Z with a correction revision or -z is passed.'
       ]
     end
   end

data/lib/cheftacular/stateless_actions/client_list.rb

@@ -38,7 +38,7 @@ class Cheftacular
 
         env_nodes = @config['parser'].exclude_nodes(nodes, [{ if: { not_env: env } }])
         puts "\nFound #{ env_nodes.count } #{ env } nodes:"
-        out = "  #{ 'name'.ljust(22) } #{ 'ip_address'.ljust(21) }"
+        out = "  #{ 'name'.ljust(22) } #{ 'ip_address'.ljust(20) }"
         out << "#{ 'private_address'.ljust(21) }"                   if @options['with_private']
         out << "#{ 'pass?'.ljust(5) } #{ 'domain'.ljust(41) }"      if @options['verbose']
         out << "#{ 'deploy_password'.ljust(21) }"                   if @options['verbose']
@@ -48,15 +48,15 @@ class Cheftacular
 
         auth_hash = @config[env]['server_passwords_bag_hash']
 
-        addr_data = @config['getter'].get_addresses_hash env
+        addresses_hash = @config['getter'].get_addresses_hash env
 
         env_nodes.each do |node|
           #client = @ridley.client.find(options['node_name'])
           out = "  #{ node.chef_id.ljust(22,'_') }_#{ node.public_ipaddress.ljust(20,'_') }"
 
           if @options['with_private']
-            if addr_data.has_key?(node.public_ipaddress)
-              out << addr_data[node.public_ipaddress]['priv'].ljust(20,'_')
+            if addresses_hash.has_key?(node.public_ipaddress)
+              out << addresses_hash[node.public_ipaddress]['address'].ljust(20,'_')
             else
               out << ''.ljust(20,'_')
             end
@@ -66,8 +66,8 @@ class Cheftacular
 
             out << "_" + auth_hash.has_key?("#{ node.public_ipaddress }-deploy-pass").to_s.ljust(5,'_') + "_"
 
-            if addr_data.has_key?(node.public_ipaddress)
-              out << addr_data[node.public_ipaddress]['dn'].ljust(40,'_')
+            if addresses_hash.has_key?(node.public_ipaddress)
+              out << addresses_hash[node.public_ipaddress]['dn'].ljust(40,'_')
             else
               out << ''.ljust(40,'_')
             end

data/lib/cheftacular/stateless_actions/cloud.rb

@@ -3,7 +3,7 @@ class Cheftacular
   class StatelessActionDocumentation
     def cloud
       @config['documentation']['stateless_action'] <<  [
-        "`cft cloud <FIRST_LEVEL_ARG> [<SECOND_LEVEL_ARG>[:<SECOND_LEVEL_ARG_QUERY>]*] ` this command handles talking to various cloud apis. " +
+        "`cft cloud <FIRST_LEVEL_ARG> [<SECOND_LEVEL_ARG>[:<SECOND_LEVEL_ARG_QUERY>]*] ` this command handles talking to various cloud APIs. " +
         "If no args are passed nothing will happen.",
 
         [

data/lib/cheftacular/stateless_actions/cloud_bootstrap.rb

@@ -31,7 +31,6 @@ class Cheftacular
       @options['node_name']   = ARGV[1] unless @options['node_name']
       @options['flavor_name'] = ARGV[2] unless @options['flavor_name']
       @options['descriptor']  = ARGV[3] if ARGV[3] && !@options['descriptor']
-      full_domain             = ""
 
       if `which sshpass`.empty?
         raise "sshpass not installed! Please run brew install https://raw.github.com/eugeneoden/homebrew/eca9de1/Library/Formula/sshpass.rb (or get it from your repo for linux)"
@@ -58,47 +57,22 @@ class Cheftacular
         return false
       end
 
-      if @options['with_dn']
-        domain_obj = PublicSuffix.parse @options['with_dn']
-
-        @config['stateless_action'].cloud "domain", "create_record:#{ domain_obj.domain }:#{ domain_obj.trd }:#{ @options['address'] }"
-
-        sleep 5
-
-        @config['stateless_action'].cloud "domain", "create_record:#{ domain_obj.domain }:local.#{ domain_obj.trd }:#{ @options['private_address'] }"
-
-        full_domain = @options['with_dn']
-
-        #set the wildcard domain for frontend load balancers
-        @config['stateless_action'].cloud "domain", "create_record:#{ domain_obj.domain }:*:#{ @options['address'] }" if ( @options['node_name'] =~ /mysc.*lb/ ) == 0
-      end
-      
       tld = @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld']
 
-      @config['stateless_action'].cloud "domain", "create_record:#{ tld }:#{ @options['node_name'] }:#{ @options['address'] }"
-
-      sleep 5
+      target_serv_index = @config[@options['env']]['addresses_bag_hash']['addresses'].count
 
-      @config['stateless_action'].cloud "domain", "create_record:#{ tld }:local.#{ @options['node_name'] }:#{ @options['private_address'] }"
+      compile_args = ['set_all_attributes']
 
-      full_domain = "#{ @options['node_name'] }.#{ tld }" if full_domain.blank?
+      compile_args << "set_specific_domain:#{ @options['with_dn'] }" if @options['with_dn']
 
-      @config[@options['env']]['addresses_bag_hash'] = @config[@options['env']]['addresses_bag'].reload.to_hash
+      address_hash = @config['DNS'].compile_address_hash_for_server_from_options(*compile_args)
 
-      target_serv_index = @config[@options['env']]['addresses_bag_hash']['addresses'].count
+      @config['DNS'].create_dns_record_for_domain_from_address_hash(@options['with_dn'], address_hash, "specific_domain_mode") if @options['with_dn']
 
-      @config[@options['env']]['addresses_bag_hash']['addresses'].each do |serv_hash|
-        target_serv_index = @config[@options['env']]['addresses_bag_hash']['addresses'].index(serv_hash) if serv_hash['name'] == @options['node_name']
-      end
-
-      #EX: "name": "api1", "public": "1.2.3.4", "address": "10.208.1.2", "dn":"api1.example.com", "descriptor": "lb:my-backend-codebase"
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index] ||= {}
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['name']       = @options['node_name']
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['public']     = @options['address']
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['address']    = @options['private_address']
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['dn']         = full_domain
-      @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['descriptor'] = @options['descriptor'].nil? ? @options['node_name'] : @options['descriptor']
+      @config['DNS'].create_dns_record_for_domain_from_address_hash(tld, address_hash)
 
+      @config[@options['env']]['addresses_bag_hash'] = @config[@options['env']]['addresses_bag'].reload.to_hash
+      
       @config['ChefDataBag'].save_addresses_bag
 
       @options['dont_remove_address_or_server'] = true #flag to make sure our entry isnt removed in addresses bag

data/lib/cheftacular/stateless_actions/compile_readme.rb

@@ -19,11 +19,11 @@ class Cheftacular
 
       out << '# Table of Contents for Cheftacular Commands'
 
-      out << '1. [Cheftacular Arguments and Flags](https://github.com/SocialCentivPublic/cheftacular/blob/master/lib/cheftacular/README.md#arguments-and-flags-for-cheftacular))'
+      out << '1. [Cheftacular Arguments and Flags](https://github.com/SocialCentivPublic/cheftacular/blob/master/lib/cheftacular/README.md#arguments-and-flags-for-cheftacular)'
 
       out << '2. [Application Commands](https://github.com/SocialCentivPublic/cheftacular/blob/master/lib/cheftacular/README.md#commands-that-can-be-run-in-the-application-context)'
 
-      out << '3. [DevOps Commands](https://github.com/SocialCentivPublic/cheftacular/blob/master/lib/cheftacular/README.md#commands-that-can-only-be-run-in-the-devops-context))' + "\n"
+      out << '3. [DevOps Commands](https://github.com/SocialCentivPublic/cheftacular/blob/master/lib/cheftacular/README.md#commands-that-can-only-be-run-in-the-devops-context)' + "\n"
       
       out << @config['documentation']['arguments']
 
@@ -35,6 +35,8 @@ class Cheftacular
 
       out << @config['helper'].compile_documentation_lines('stateless_action')
 
+      FileUtils.rm("#{ @config['locs']['chef-log'] }/README.md") if File.exist?("#{ @config['locs']['chef-log'] }/README.md")
+
       File.open("#{ @config['locs']['chef-log'] }/README.md", "w") { |f| f.write(out.flatten.join("\n\n")) }
     end
   end

data/lib/cheftacular/stateless_actions/get_pg_pass.rb

@@ -31,8 +31,6 @@ class Cheftacular
         database = repo_hash.has_key?('custom_database_name') ? repo_hash['custom_database_name'] : repo_hash['repo_name']
         password = @config[@options['env']]['chef_passwords_bag_hash']['pg_pass']
 
-        ap @config[@options['env']]['chef_passwords_bag_hash']
-
         if @config[@options['env']]['chef_passwords_bag_hash'].has_key?(repo_hash['repo_name']) && @config[@options['env']]['chef_passwords_bag_hash'][repo_hash['repo_name']].has_key?('pg_pass')
           password = @config[@options['env']]['chef_passwords_bag_hash'][repo_hash['repo_name']]['pg_pass'] unless @config[@options['env']]['chef_passwords_bag_hash'][repo_hash['repo_name']]['pg_pass'].empty?
         end

data/lib/cheftacular/stateless_actions/get_shorewall_allowed_connections.rb

@@ -0,0 +1,65 @@
+class Cheftacular
+  class StatelessActionDocumentation
+    def get_shorewall_allowed_connections
+      @config['documentation']['stateless_action'] <<  [
+        "`cft get_shorewall_allowed_connections` command will query a single server and return all of its ACCEPT connections " +
+        "from shorewall in it's syslog and return the results in a CSV format. Useful for tracking IP activity.",
+
+        [
+          "    1. This command will attempt to `dig` each ip address to give you the most likely culprit."
+        ]
+      ]
+    end
+  end
+
+  class StatelessAction
+    def get_shorewall_allowed_connections
+      #TODO find and load syslog files from nodes!
+      log_data << File.read(File.expand_path("#{ @locs['chef-log'] }/marketing_site_syslog_2.txt"))
+      log_data << File.read(File.expand_path("#{ @locs['chef-log'] }/marketing_site_syslog_1.txt"))
+      log_data << File.read(File.expand_path("#{ @locs['chef-log'] }/marketing_site_syslog.txt"))
+
+      exit
+
+      addresses = {}
+
+      log_data.join("\n").scan(/^.*Shorewall:net2fw:ACCEPT.*SRC=([\d]+\.[\d]+\.[\d]+\.[\d]+) DST.*DPT=80.*$/).each do |ip_address|
+        addresses[ip_address] ||= 0
+        addresses[ip_address] += 1
+      end
+
+      final_addresses = {}
+      check_count = 0
+      addresses.each_pair do |address, count|
+        domain = `dig +short -x #{ address[0] }`.chomp.split("\n").join('|')
+        domain = domain[0..(domain.length-2)]
+
+        domain = address[0] if domain.blank?
+
+        final_addresses[domain] ||= {}
+        final_addresses[domain]['addresses'] ||= []
+        final_addresses[domain]['addresses'] <<  address[0] unless final_addresses[domain]['addresses'].include?(address[0])
+        final_addresses[domain]['count']     =   count   unless final_addresses[domain].has_key?('count')
+        final_addresses[domain]['count']     +=  count   if final_addresses[domain].has_key?('count')
+
+        check_count += 1
+
+        puts "Processed #{ check_count } addresses (#{ address[0] }):#{ domain }:#{ count }"
+      end
+
+      final_addresses = final_addresses.sort_by {|key, value_hash| value_hash['count']}.to_h
+
+      final_addresses = Hash[final_addresses.to_a.reverse]
+
+      ap(final_addresses)
+
+      log_loc, timestamp = @config['helper'].set_log_loc_and_timestamp
+
+      CSV.open(File.expand_path("#{ @locs['chef-log'] }/#{ @options['node_name'] }-shorewall-parse-#{ timestamp }.csv"), "wb") do |csv| 
+        final_addresses.each_pair do |dns, info_hash|
+          csv << [dns, info_hash['addresses'].join('|'), info_hash['count']]
+        end
+      end
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/help.rb

@@ -56,7 +56,7 @@ class Cheftacular
       puts @config['helper'].compile_documentation_lines('devops').flatten.join("\n\n") if target_command == 'devops'
 
       if inference_modes.empty? && @config['helper'].is_not_command_or_stateless_command?(target_command)
-        methods = @config['action_documentation'].public_methods(false) + @config['action_documentation'].public_methods(false)
+        methods = @config['action_documentation'].public_methods(false) + @config['stateless_action_documentation'].public_methods(false)
 
         sorted_methods = methods.uniq.sort_by { |method| @config['helper'].compare_strings(target_command, method.to_s)}
 

data/lib/cheftacular/stateless_actions/initialize_data_bag_contents.rb

@@ -196,6 +196,12 @@ class Cheftacular
 
           save_on_finish = true
         end
+
+        unless hash[env].has_key?('cloudflare_activated_domains')
+          hash[env]['cloudflare_activated_domains'] = []
+
+          save_on_finish = true
+        end
       end
 
       @config['ChefDataBag'].save_config_bag(main_env) if save_on_finish

data/lib/cheftacular/stateless_actions/remove_client.rb

@@ -2,7 +2,7 @@
 class Cheftacular
   class StatelessActionDocumentation
     def remove_client
-      @config['documentation']['action'] <<  [
+      @config['documentation']['stateless_action'] <<  [
         "`cft remove_client -n NODE_NAME` removes a client (and its node data) from the chef-server. " +
         "It also removes its dns records from the cloud service (if possible). " +
         "This should not be done lightly as you will have to wipe the server and trigger another chef-client run to get it to register again"
@@ -16,9 +16,11 @@ class Cheftacular
       @options['delete_server_on_remove'] = ARGV[2] if !@options['delete_server_on_remove'] && !@options['dont_remove_address_or_server'] && ARGV[2]
       @options['delete_server_on_remove'] = 'destroy' if delete_server || @options['delete_server_on_remove']
 
+      raise "This action can only be performed if the mode is set to devops" if !@config['helper'].running_in_mode?('devops') && !@options['in_scaling']
+
       raise "The only valid argument for the 2nd argument of this command is 'destroy', please enter this or leave it blank." if ARGV[2] && ARGV[2] != 'destroy' && !@options['dont_remove_address_or_server']
 
-      raise "CannotRemoveMultipleClients, Please call this script as hip remove_client <node_name>" unless @options['node_name']
+      raise "Invalid arguments! Node name is blank. Please call this script as cft remove_client <node_name>" unless @options['node_name']
       
       nodes = @config['getter'].get_true_node_objects(false, true)
 
@@ -47,27 +49,11 @@ class Cheftacular
           if @options['delete_server_on_remove'] == 'destroy'
             @config['stateless_action'].cloud "server", "destroy:#{ @options['env'] }_#{ n.name }"
           end
-        end 
-
-        @config[@options['env']]['addresses_bag_hash'] = @config[@options['env']]['addresses_bag'].reload.to_hash
-
-        target_serv_index = nil
-
-        @config[@options['env']]['addresses_bag_hash']['addresses'].each do |serv_hash|
-          target_serv_index = @config[@options['env']]['addresses_bag_hash']['addresses'].index(serv_hash) if serv_hash['name'] == @options['node_name']
         end
 
-        if !target_serv_index.nil? && target_serv_index.is_a?(Fixnum) && !@options['dont_remove_address_or_server']
-          puts("Found entry in addresses data bag corresponding to #{ @options['node_name'] } for #{ @options['env'] }, removing...") unless @options['quiet']
-
-          domain_obj = PublicSuffix.parse @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['dn']
-
-          #delete the domain on rax if its a domain we host there for its environment
-          @config['stateless_action'].cloud "domain", "destroy_record:#{ domain_obj.tld }:#{ domain_obj.trd }" if domain_obj.tld == @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld']
+        @config[@options['env']]['addresses_bag_hash'] = @config[@options['env']]['addresses_bag'].reload.to_hash
 
-          @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index] = nil
-          @config[@options['env']]['addresses_bag_hash']['addresses'] = @config[@options['env']]['addresses_bag_hash']['addresses'].compact
-        end
+        @config['DNS'].compile_address_hash_for_server_from_options('set_hash_to_nil')
 
         @config['ChefDataBag'].save_addresses_bag
       end

data/lib/cheftacular/stateless_actions/replication_status.rb

@@ -47,7 +47,6 @@ class Cheftacular
         rep_status_hash[n.name] = start_slave_replication_report( n.name, n.public_ipaddress,  options, locs, passwords)
       end
 
-
       rep_status_hash.each_pair do |serv_name, output|
         out << "#{ serv_name }:"
 

data/lib/cheftacular/stateless_actions/slack.rb

@@ -4,6 +4,13 @@ class Cheftacular
       @config['documentation']['stateless_action'] <<  [
         "`cft slack \"MESSAGE\" [CHANNEL]` will attempt to post the message to the webhook set in your cheftacular.yml. " +
         "Slack posts to your default channel by default but if the CHANNEL argument is supplied the message will post there.",
+
+        [
+          "    1. NOTE: To prevent confusing spam from many possible sources, the username posted to slack will always be " +
+          "*Cheftacular*. This can be overloaded in the StatelessAction method \"slack\" but this is not recommended.",
+
+          "    2. Remember, if you have auditing turned on in your cheftacular.yml, you can track who sends what to slack."
+        ]
       ]
     end
   end
@@ -12,7 +19,7 @@ class Cheftacular
     def slack
       notifier = Slack::Notifier.new @config['cheftacular']['slack_webhook'], username: 'Cheftacular'
 
-      notifier.channel = '#default' unless ARGV[2]  
+      notifier.channel = ARGV[2] ? ARGV[2] : '#default'  
       notifier.ping ARGV[1]
     end
   end