cheftacular 2.2.2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29520ff1bae99c83bce312f6f5f582505776a007
-  data.tar.gz: 7f79d595110a8850083c5314671129d08709560b
+  metadata.gz: cacccbd73021b45400cb3eda03bb8cb22f1ba9fe
+  data.tar.gz: 70f82785708bd038080088e905a0ef5cf4f84598
 SHA512:
-  metadata.gz: a52cf49de433628aab008679c6df07a7f42a9ca290f7895ee2e780a21db5481be3cefc09c9887981b9b4c089c7a1e8b418fb5bfc123102cd81fa5965c77999ef
-  data.tar.gz: cb443b1cc185d8ce16f4987415b396f8eb14473f9d66615612cfea10daab85972c38b8ce43c4a59390ebbc5339be8f6e7ecc052db81d8af33e295fcdf0e39f1f
+  metadata.gz: 08b9fbaee4acc4edebff56241cdf369a30fc1684dbee05f37021c4e3d85002f267fd9c942303d6003fcbd095010d34a0a1a775d30c8e73c71f2cbca3134d8b2b
+  data.tar.gz: 2dee13991458ac134f24af9b2685e2c9e389934a5214a100d737612a23220d9d4ef5eca643a652c1be978471d685b1bcb93d90e7a9784b742bf4dce2a923583a

data/lib/cheftacular/README.md

@@ -220,8 +220,8 @@
             2. NOTE! Most flavors have spaces in them, you must use quotes at the command line to utilize them!
 
         4. `destroy:SERVER_NAME` destroys the server on the cloud. This must be an exact match of the server's actual name or the script will error.
-
-        5. `poll:SERVER_NAME` polls the cloud's server for the status of the SERVER_NAME. This command will stop polling if / when the status of the server is ACTIVE and its build progress is 100%.
+        
+        5. `poll:SERVER_NAME` polls the cloud's server for the status of the SERVER_NAME. This command will stop polling if / when the status of the server is ACTIVE and its build progress is 100%.
 
         6. `attach_volume:SERVER_NAME:VOLUME_NAME[:VOLUME_SIZE[:DEVICE_LOCATION]]` If VOLUME_NAME exists it will attach it if it is unattached otherwise it will create it
 
@@ -303,9 +303,13 @@
 
 21. `cft get_pg_pass ['clip']` command will output the current environment's pg_password to your terminal. Optionally you can pass in clip like `cft get_pg_pass clip` to have it also copy the pass to your clipboard.
 
-22. `cft get_shorewall_allowed_connections` command will query a single server and return all of its ACCEPT connections from shorewall in it's syslog and return the results in a CSV format. Useful for tracking IP activity.
+22. `cft get_shorewall_allowed_connections [PATH_TO_LOCAL_FILE] -n NODE_NAME` command will query a single server and return all of its ACCEPT connections from shorewall in it's syslog and return the results in a CSV format. Useful for tracking IP activity.
+
+    1. You must pass in a node name to query with `-n NODE_NAME`
 
-    1. This command will attempt to `dig` each ip address to give you the most likely culprit.
+    2. This command will attempt to `dig` each ip address to give you the most likely culprit.
+
+    3. If `PATH_TO_LOCAL_FILE` is not blank, the command will use that file instead of building a file on the remote server
 
 23. `cft help COMMAND|MODE` this command returns the documentation for a specific command if COMMAND matches the name of a command. Alternatively, it can be passed `action|arguments|application|current|devops|stateless_action` to fetch the commands for a specific mode.Misspellings of commands will display near hits.
 
@@ -369,12 +373,14 @@
 
 35. `cft ubuntu_bootstrap ADDRESS ROOT_PASS` This command will bring a fresh server to a state where chef-client can be run on it via `cft chef-bootstrap`. It should be noted that it is in this step where a server's randomized deploy_user sudo password is generated.
 
-36. `cft update_cloudflare_dns_from_cloud` command will force a full dns update for cloudflare. 
+36. `cft update_cloudflare_dns_from_cloud [skip_update_tld]` command will force a full dns update for cloudflare. 
 
     1. It will ensure all the subdomain entries are correct (based on the contents of the addresses data bag) and update them if they are not. It will also create the local subdomain for the entry as well if it does exist and point it to the correct private address for an environment.
 
     2. This command will also ensure any dns records on your cloud are also migrated over to cloudflare as well. This also includes the reverse in the event you would like to turn off cloudflare.
 
+    3. The argument `skip_update_tld` will stop the long process of checking and updating all the server domains _before_ cloudflare is updated. Only skip if you believe your domain info on your cloud is accurate.
+
 37. `cft update_split_branches` will perform a series of git commands that will merge all the split branches for your split_branch enabled repositories with what is currently on master and push them.
 
     1. Repository must be set with `-R REPOSITORY_NAME` for this command to work.

data/lib/cheftacular/actions/deploy.rb

@@ -33,7 +33,7 @@ class Cheftacular
         logs_bag_hash["#{ n.name }-deploy"] = { text: log_data.scrub_pretty_text, timestamp: timestamp }
       end
       
-      @config['ChefDataBag'].save_logs_bag unless @options['debug'] #the debug chef-client runs are literally too large to POST
+      #@config['ChefDataBag'].save_logs_bag unless @options['debug'] #We don't really need to store entire chef runs in the logs bag
 
       migrate(nodes) if @config['getter'].get_current_repo_config['database'] != 'none' && !@options['run_migration_already']
     end

data/lib/cheftacular/actions/migrate.rb

@@ -43,7 +43,7 @@ class Cheftacular
     end
 
     def migrate_wordpress nodes=[]
-      puts "Method #{ __migrate_wordpress__ } isb ot yet implemented"
+      puts "Method #{ __method__ } is not yet implemented"
 
       exit
     end

data/lib/cheftacular/chef/data_bag.rb

@@ -39,6 +39,8 @@ class Cheftacular
 
         item.attributes = @config[env]['logs_bag_hash'].keep_if {|key,val| key == 'id'}
 
+        sleep 5
+
         item.save
 
         @config[env]['logs_bag_hash'] = @config[env]['logs_bag'].reload.to_hash

data/lib/cheftacular/cheftacular.rb

@@ -22,6 +22,7 @@ require 'timeout'
 require 'slack-notifier'
 require 'cloudflare'
 require 'zlib'
+require 'csv'
 
 Dir["#{File.dirname(__FILE__)}/../**/*.rb"].each { |f| require f }
 
@@ -29,6 +30,8 @@ class Cheftacular
   def initialize options={'env'=>'staging'}, config={}
     @options, @config = options, config
 
+    SSHKit.config.format = :blackhole
+
     @config['start_time']  = Time.now
 
     @config['helper']      = Helper.new(@options, @config)

data/lib/cheftacular/dns.rb

@@ -180,12 +180,12 @@ class Cheftacular
       if !target_serv_index.nil? && target_serv_index.is_a?(Fixnum) && !@options['dont_remove_address_or_server'] && args.include?('set_hash_to_nil')
         puts("Found entry in addresses data bag corresponding to #{ @options['node_name'] } for #{ @options['env'] }, removing...") unless @options['quiet']
 
+        domain_obj = PublicSuffix.parse @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['dn']
+
         @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index] = nil
       
         @config[@options['env']]['addresses_bag_hash']['addresses'] = @config[@options['env']]['addresses_bag_hash']['addresses'].compact
 
-        domain_obj = PublicSuffix.parse @config[@options['env']]['addresses_bag_hash']['addresses'][target_serv_index]['dn']
-
         @config['stateless_action'].cloud "domain", "destroy_record:#{ domain_obj.tld }:#{ domain_obj.trd }" if domain_obj.tld == @config[@options['env']]['config_bag_hash'][@options['sub_env']]['tld']
       end
 

data/lib/cheftacular/initializer.rb

@@ -397,7 +397,7 @@ class Cheftacular
 
         exit
       else
-        unless File.exists?( current_version_file_path )
+        unless File.exists?( @config['helper'].current_version_file_path )
           puts "Creating file cache for #{ Time.now.strftime("%Y%m%d") } (#{ detected_version }). No new version detected."
 
           @config['helper'].write_version_file detected_version

data/lib/cheftacular/stateless_actions/chef_environment.rb

@@ -15,7 +15,7 @@ class Cheftacular
 
   class StatelessAction
     def chef_environment
-      #TODO
+      raise "Not yet Implemented"
     end
   end
 end

data/lib/cheftacular/stateless_actions/cloud.rb

@@ -7,7 +7,7 @@ class Cheftacular
         "If no args are passed nothing will happen.",
 
         [
-          "    1. `domain` 1st level argument for interacting with cloud domains",
+          "    1. `domain` first level argument for interacting with cloud domains",
 
           "        1. `list` default behavior",
 
@@ -28,7 +28,7 @@ class Cheftacular
 
           "        9. `update_record:TOP_LEVEL_DOMAIN:SUBDOMAIN_NAME:IP_ADDRESS[:RECORD_TYPE[:TTL]]` similar to `create_record`.",
 
-          "    2. `server` 1st level argument for interacting with cloud servers, " +
+          "    2. `server` first level argument for interacting with cloud servers, " +
           "if no additional args are passed the command will return a list of all servers on the preferred cloud.",
 
           "        1.  `list` default behavior",
@@ -44,7 +44,7 @@ class Cheftacular
 
           "        4. `destroy:SERVER_NAME` destroys the server on the cloud. This must be an exact match of the server's actual name or the script will error.",
 
-          "        5. `poll:SERVER_NAME` polls the cloud's server for the status of the SERVER_NAME. This command " +
+          "        5. `poll:SERVER_NAME` polls the cloud's server for the status of the SERVER_NAME. This command " +
           "will stop polling if / when the status of the server is ACTIVE and its build progress is 100%.",
 
           "        6. `attach_volume:SERVER_NAME:VOLUME_NAME[:VOLUME_SIZE[:DEVICE_LOCATION]]` " +
@@ -66,7 +66,7 @@ class Cheftacular
 
           "        9. `read_volume:SERVER_NAME:VOLUME_NAME` returns the data of VOLUME_NAME if it is attached to the server.",
 
-          "    3. `volume` 1st level argument for interacting with cloud storage volumes, if no additional args are passed the command will return a list of all cloud storage containers.",
+          "    3. `volume` first level argument for interacting with cloud storage volumes, if no additional args are passed the command will return a list of all cloud storage containers.",
             
           "        1. `list` default behavior",
 
@@ -76,7 +76,7 @@ class Cheftacular
 
           "        4. `destroy:VOLUME_NAME` destroys the volume. This operation will not work if the volume is attached to a server.",
 
-          "    4. `flavor` 1st level argument for listing the flavors available on the cloud service",
+          "    4. `flavor` first level argument for listing the flavors available on the cloud service",
 
           "        1. `list` default behavior",
 

data/lib/cheftacular/stateless_actions/cloud_bootstrap.rb

@@ -75,7 +75,7 @@ class Cheftacular
 
       @options['dont_remove_address_or_server'] = true #flag to make sure our entry isnt removed in addresses bag
 
-      full_bootstrap #bootstrap server with ruby and attach it to the chef server
+      @config['stateless_action'].full_bootstrap #bootstrap server with ruby and attach it to the chef server
     end
   end
 end

data/lib/cheftacular/stateless_actions/get_log_from_bag.rb

@@ -23,14 +23,14 @@ class Cheftacular
       nodes = @config['parser'].exclude_nodes( nodes, [{ unless: { env: @options['env'] }}])
 
       nodes.each do |node|
-        if @config[@options['env']]['logs_bag_hash'].has_key?("#{ node.name }-deploy")
-          puts("Found log data in logs bag. Outputting to #{ log_loc }/stashedlog/#{ node.name }-deploystash-#{ @config[@options['env']]['logs_bag_hash']["#{ node.name }-deploy"][:timestamp] }.txt") unless @options['quiet']
+        if @config[@options['env']]['logs_bag_hash'].has_key?("#{ node.name }-run")
+          puts("Found log data in logs bag. Outputting to #{ log_loc }/stashedlog/#{ node.name }-deploystash-#{ @config[@options['env']]['logs_bag_hash']["#{ node.name }-run"][:timestamp] }.txt") unless @options['quiet']
 
-          File.open("#{ log_loc }/stashedlog/#{ node.name }-deploystash-#{@config[@options['env']]['logs_bag_hash']["#{ node.name }-deploy"][:timestamp] }.txt", "w") do |f|
-            f.write(@config[@options['env']]['logs_bag_hash']["#{ node.name }-deploy"][:text])
+          File.open("#{ log_loc }/stashedlog/#{ node.name }-deploystash-#{@config[@options['env']]['logs_bag_hash']["#{ node.name }-run"][:timestamp] }.txt", "w") do |f|
+            f.write(@config[@options['env']]['logs_bag_hash']["#{ node.name }-run"][:text])
           end
 
-          puts(@config[@options['env']]['logs_bag_hash']["#{ node.name }-deploy"][:text]) if @options['verbose']
+          puts(@config[@options['env']]['logs_bag_hash']["#{ node.name }-run"][:text]) if @options['verbose']
         end
       end
     end

data/lib/cheftacular/stateless_actions/get_shorewall_allowed_connections.rb

@@ -2,24 +2,54 @@ class Cheftacular
   class StatelessActionDocumentation
     def get_shorewall_allowed_connections
       @config['documentation']['stateless_action'] <<  [
-        "`[NYI]cft get_shorewall_allowed_connections` command will query a single server and return all of its ACCEPT connections " +
+        "`cft get_shorewall_allowed_connections [PATH_TO_LOCAL_FILE] -n NODE_NAME` command will query a single server and return all of its ACCEPT connections " +
         "from shorewall in it's syslog and return the results in a CSV format. Useful for tracking IP activity.",
 
         [
-          "    1. This command will attempt to `dig` each ip address to give you the most likely culprit."
+          "    1. You must pass in a node name to query with `-n NODE_NAME`",
+
+          "    2. This command will attempt to `dig` each ip address to give you the most likely culprit.",
+
+          "    3. If `PATH_TO_LOCAL_FILE` is not blank, the command will use that file instead of building a file on the remote server"
         ]
       ]
     end
   end
 
   class StatelessAction
-    def get_shorewall_allowed_connections
-      raise "Not yet implemented"
-      #TODO find and load syslog files from nodes!
+    def get_shorewall_allowed_connections master_log_data=''
+
+      if ARGV[1].nil?
+        raise "Please pass a NODE_NAME with -n NODE_NAME" if @options['node_name'].nil? || @options['node_name'].empty?
+
+        nodes = @config['getter'].get_true_node_objects true
+
+        nodes = @config['parser'].exclude_nodes(nodes, [{ unless: { env: @options['env'] }}, { unless: { node: @options['node_name'] }}], true)
+
+        #this must always precede on () calls so they have the instance variables they need
+        options, locs, ridley, logs_bag_hash, pass_bag_hash, bundle_command, cheftacular, passwords = @config['helper'].set_local_instance_vars
+
+        #on is namespaced to SSHKit::Backend::Netssh.on 
+        on ( nodes.map { |n| "deploy@" + n.public_ipaddress } ) do |host|
+          n = get_node_from_address(nodes, host.hostname)
+
+          puts("Beginning shorewall log capture run for #{ n.name } (#{ n.public_ipaddress })") unless options['quiet']
+
+          master_log_data = start_shorewall_log_capture( n.name, n.public_ipaddress, options, locs, cheftacular, passwords)
+        end
+      else
+        master_log_file = ARGV[1]
+
+        raise "File not found! Did you enter the path correctly?" unless File.exist?(master_log_file)
+
+        master_log_data = File.read(File.expand_path(master_log_file))
+      end
+
+      puts("Parsing addresses from log data...") unless @options['quiet']
 
       addresses = {}
 
-      log_data.join("\n").scan(/^.*Shorewall:net2fw:ACCEPT.*SRC=([\d]+\.[\d]+\.[\d]+\.[\d]+) DST.*DPT=80.*$/).each do |ip_address|
+      master_log_data.scan(/^.*Shorewall:net2fw:ACCEPT.*SRC=([\d]+\.[\d]+\.[\d]+\.[\d]+) DST.*DPT=80.*$/).each do |ip_address|
         addresses[ip_address] ||= 0
         addresses[ip_address] += 1
       end
@@ -27,6 +57,8 @@ class Cheftacular
       final_addresses = {}
       check_count = 0
       addresses.each_pair do |address, count|
+        next if count < 100
+        
         domain = `dig +short -x #{ address[0] }`.chomp.split("\n").join('|')
         domain = domain[0..(domain.length-2)]
 
@@ -40,18 +72,18 @@ class Cheftacular
 
         check_count += 1
 
-        puts "Processed #{ check_count } addresses (#{ address[0] }):#{ domain }:#{ count }"
+        puts("Processed #{ check_count } addresses (#{ address[0] }):#{ domain }:#{ count }") unless @options['quiet']
       end
 
       final_addresses = final_addresses.sort_by {|key, value_hash| value_hash['count']}.to_h
 
       final_addresses = Hash[final_addresses.to_a.reverse]
 
-      ap(final_addresses)
+      ap(final_addresses) if @options['verbose']
 
       log_loc, timestamp = @config['helper'].set_log_loc_and_timestamp
 
-      CSV.open(File.expand_path("#{ @locs['chef-log'] }/#{ @options['node_name'] }-shorewall-parse-#{ timestamp }.csv"), "wb") do |csv| 
+      CSV.open(File.expand_path("#{ @config['locs']['chef-log'] }/shorewall-parse-#{ timestamp }.csv"), "wb") do |csv| 
         final_addresses.each_pair do |dns, info_hash|
           csv << [dns, info_hash['addresses'].join('|'), info_hash['count']]
         end
@@ -59,3 +91,39 @@ class Cheftacular
     end
   end
 end
+
+module SSHKit
+  module Backend
+    class Netssh
+      def start_shorewall_log_capture name, ip_address, options, locs, cheftacular, passwords, out=[]
+        log_loc, timestamp = set_log_loc_and_timestamp(locs)
+
+        puts("Generating master log file for shorewall for #{ name } (#{ ip_address }) at #{ log_loc }/#{ name }-shorewall-#{ timestamp }.html") unless options['quiet']
+
+        syslog_files = capture(:ls, '/var/log', :|, :grep, :syslog).split("\n")
+
+        puts("Found #{ syslog_files.count } syslog files to parse (#{ syslog_files.join(', ') }).\nPreparing to parse...") unless options['quiet']
+
+        syslog_files.each do |file|
+          puts("Parsing #{ file } into master log file...") unless options['quiet']
+
+          if file.include?('.gz')
+            sudo_execute(passwords[ip_address], :gunzip, '-c', "/var/log/#{ file }", '>>', '/tmp/syslog_master.log' )
+          else
+            sudo_execute(passwords[ip_address], :cat, "/var/log/#{ file }", '>>', '/tmp/syslog_master.log' )
+          end
+        end
+
+        puts("Writing master log...") unless options['quiet']
+
+        out << sudo_capture( passwords[ip_address], :cat, "/tmp/syslog_master.log" )
+
+        ::File.open("#{ log_loc }/#{ name }-shorewall-#{ timestamp }.html", "w") { |f| f.write(out.join("\n").scrub_pretty_text.gsub('[sudo] password for deploy: ', '')) } unless options['no_logs']
+
+        sudo_execute(passwords[ip_address], :rm, '-f', '/tmp/syslog_master.log')
+
+        out.join("\n")
+      end
+    end
+  end
+end

data/lib/cheftacular/stateless_actions/test_env.rb

@@ -30,7 +30,7 @@ class Cheftacular
 
       type = ARGV[env_index] if ARGV[env_index]
 
-      raise "Unknown split_env: #{ split_env }, can only be #{ cheftacular['run_list_environments'].values.join(', ') }" unless (split_env =~ /#{ cheftacular['run_list_environments'].values.join('|') }/) == 0
+      raise "Unknown split_env: #{ split_env }, can only be #{ @config['cheftacular']['run_list_environments'].values.join(', ') }" unless (split_env =~ /#{ @config['cheftacular']['run_list_environments'].values.join('|') }/) == 0
 
       raise "Unknown type: #{ type }, can only be 'boot' or 'destroy'" unless (type =~ /boot|destroy/) == 0
 
@@ -41,11 +41,9 @@ class Cheftacular
       @options['force_yes']  = true
       @options['in_scaling'] = true
 
-      initial_servers = @config['cheftacular']['split_env_nodes']
-
       case type
       when 'boot'
-        initial_servers.each_pair do |name, config_hash|
+        @config['cheftacular']['split_env_nodes'].each_pair do |name, config_hash|
           true_name               = name.gsub('SPLITENV', split_env) 
           @options['node_name']   = "#{ true_name }#{ 'p' if @options['env'] == 'production' }" 
           @options['flavor_name'] = config_hash.has_key?('flavor') ? config_hash['flavor'] : @config['cheftacular']['default_flavor_name']
@@ -66,7 +64,6 @@ class Cheftacular
         @options['delete_server_on_remove'] = true
 
         nodes.each do |node|
-          next if !targets.empty? && !targets.include?(node.name)
 
           @options['node_name'] = node.name
 

data/lib/cheftacular/version.rb

@@ -1,5 +1,5 @@
 class Cheftacular
   #major_version.minor_version.bugfixes
-  VERSION      = "2.2.2"
+  VERSION      = "2.3.0"
   RUBY_VERSION = "2.2.2"
 end

data/lib/sshkit/actions/start_task.rb

@@ -28,4 +28,4 @@ module SSHKit
       end
     end
   end
-end
+end

data/lib/sshkit/helpers.rb

@@ -9,6 +9,10 @@ module SSHKit
         capture :echo, pass, :|, :sudo, '-S', *args
       end
 
+      def sudo_execute pass, *args
+        execute :echo, pass, :|, :sudo, '-S', *args
+      end
+
       def sudo_test pass, file_location
         sudo_capture( pass, :test, '-e', file_location, '&&', :echo, 'true', { raise_on_non_zero_exit: false, verbosity: Logger::DEBUG }) == 'true'
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cheftacular
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.3.0
 platform: ruby
 authors:
 - Louis Alridge
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-13 00:00:00.000000000 Z
+date: 2015-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hashie