taste_tester 0.0.1 → 0.0.2

data/lib/taste_tester/config.rb

@@ -0,0 +1,92 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+require 'mixlib/config'
+require 'taste_tester/logging'
+require 'between_meals/util'
+
+module TasteTester
+  # Config file parser and config object
+  # Uses Mixlib::Config v1 syntax so it works in Chef10 omnibus...
+  # it's compatible with v2, so it should work in 11 too.
+  module Config
+    extend Mixlib::Config
+    extend TasteTester::Logging
+    extend BetweenMeals::Util
+
+    repo "#{ENV['HOME']}/ops"
+    repo_type 'git'
+    base_dir 'chef'
+    cookbook_dirs ['cookbooks']
+    role_dir 'roles'
+    databag_dir 'databags'
+    config_file '/etc/taste-tester-config.rb'
+    plugin_path '/etc/taste-tester-plugin.rb'
+    chef_zero_path '/opt/chef/embedded/bin/chef-zero'
+    verbosity Logger::WARN
+    timestamp false
+    user 'root'
+    ref_file "#{ENV['HOME']}/.chef/taste-tester-ref.json"
+    checksum_dir "#{ENV['HOME']}/.chef/checksums"
+    skip_repo_checks false
+    chef_client_command 'chef-client'
+    testing_time 3600
+    chef_port_range [5000, 5500]
+    tunnel_port 4001
+    timestamp_file '/etc/chef/test_timestamp'
+    use_ssh_tunnels true
+
+    skip_pre_upload_hook false
+    skip_post_upload_hook false
+    skip_pre_test_hook false
+    skip_post_test_hook false
+    skip_repo_checks_hook false
+
+    def self.cookbooks
+      cookbook_dirs.map do |x|
+        File.join(repo, base_dir, x)
+      end
+    end
+
+    def self.relative_cookbook_dirs
+      cookbook_dirs.map do |x|
+        File.join(base_dir, x)
+      end
+    end
+
+    def self.roles
+      File.join(repo, base_dir, role_dir)
+    end
+
+    def self.relative_role_dir
+      File.join(base_dir, role_dir)
+    end
+
+    def self.databags
+      File.join(repo, base_dir, databag_dir)
+    end
+
+    def self.relative_databag_dir
+      File.join(base_dir, databag_dir)
+    end
+
+    def self.chef_port
+      range = chef_port_range.first.to_i..chef_port_range.last.to_i
+      range.to_a.shuffle.each do |port|
+        unless port_open?(port)
+          return port
+        end
+      end
+      logger.error 'Could not find a free port in range' +
+        " [#{chef_port_range.first}, #{chef_port_range.last}]"
+      exit 1
+    end
+
+    def self.testing_end_time
+      if TasteTester::Config.testing_until
+        TasteTester::Config.testing_until.strftime('%y%m%d%H%M.%S')
+      else
+        (Time.now + TasteTester::Config.testing_time).strftime('%y%m%d%H%M.%S')
+      end
+    end
+  end
+end

data/lib/taste_tester/hooks.rb

@@ -0,0 +1,52 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+require 'taste_tester/logging'
+require 'between_meals/util'
+
+module TasteTester
+  # Hooks placeholders
+  class Hooks
+    extend TasteTester::Logging
+    extend BetweenMeals::Util
+
+    # Do stuff before we upload to chef-zero
+    def self.pre_upload(_dryrun, _repo, _last_ref, _cur_ref)
+    end
+
+    # Do stuff after we upload to chef-zero
+    def self.post_upload(_dryrun, _repo, _last_ref, _cur_ref)
+    end
+
+    # Do stuff before we put hosts in test mode
+    def self.pre_test(_dryrun, _repo, _hosts)
+    end
+
+    # This should return an array of commands to execute on
+    # remote systems.
+    def self.test_remote_cmds(_dryrun, _hostname)
+    end
+
+    # Should return a string with extra stuff to shove
+    # in the remote client.rb
+    def self.test_remote_client_rb_extra_code(_hostname)
+    end
+
+    # Do stuff after we put hosts in test mode
+    def self.post_test(_dryrun, _repo, _hosts)
+    end
+
+    # Additional checks you want to do on the repo
+    def self.repo_checks(_dryrun, _repo)
+    end
+
+    def self.get(file)
+      path = File.expand_path(file)
+      logger.warn("Loading plugin at #{path}")
+      unless File.exists?(path)
+        logger.error('Plugin file not found')
+        exit(1)
+      end
+      class_eval(File.read(path), __FILE__, __LINE__)
+    end
+  end
+end

data/lib/taste_tester/host.rb

@@ -0,0 +1,187 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+require 'fileutils'
+require 'base64'
+require 'open3'
+require 'colorize'
+
+require 'taste_tester/ssh'
+require 'taste_tester/tunnel'
+
+module TasteTester
+  # Manage state of the remote node
+  class Host
+    include TasteTester::Logging
+
+    attr_reader :name
+
+    def initialize(name, server)
+      @name = name
+      @user = ENV['USER']
+      @server = server
+      @tunnel = TasteTester::Tunnel.new(@name, @server)
+    end
+
+    def runchef
+      logger.warn("Running '#{TasteTester::Config.command}' on #{@name}")
+      cmd = "ssh #{TasteTester::Config.user}@#{@name} "
+      if TasteTester::Config.user != 'root'
+        cc = Base64.encode64(cmds).gsub(/\n/, '')
+        cmd += "\"echo '#{cc}' | base64 --decode | sudo bash -x\""
+      else
+        cmd += "\"#{cmds}\""
+      end
+      status = IO.popen(
+        cmd
+      ) do |io|
+        # rubocop:disable AssignmentInCondition
+        while line = io.gets
+          puts line.chomp!
+        end
+        # rubocop:enable AssignmentInCondition
+        io.close
+        $CHILD_STATUS.to_i
+      end
+      logger.warn("Finished #{TasteTester::Config.command}" +
+                  " on #{@name} with status #{status}")
+      if status == 0
+        msg = "#{TasteTester::Config.command} was successful" +
+          ' - please log to the host and confirm all the intended' +
+          ' changes were made'
+        logger.error msg.upcase
+      end
+    end
+
+    def test
+      logger.warn("Taste-testing on #{@name}")
+
+      # Nuke any existing tunnels that may be there
+      TasteTester::Tunnel.kill(@name)
+
+      # Then setup the tunnel
+      @tunnel.run
+      @serialized_config = Base64.encode64(config).gsub(/\n/, '')
+
+      # Then setup the testing
+      ssh = TasteTester::SSH.new(@name)
+      ssh << 'logger -t taste-tester Moving server into taste-tester' +
+        " for #{@user}"
+      ssh << "touch -t #{TasteTester::Config.testing_end_time}" +
+        " #{TasteTester::Config.timestamp_file}"
+      ssh << "echo -n '#{@serialized_config}' | base64 --decode" +
+        ' > /etc/chef/client-taste-tester.rb'
+      ssh << 'rm -vf /etc/chef/client.rb'
+      ssh << '( ln -vs /etc/chef/client-taste-tester.rb' +
+        ' /etc/chef/client.rb; true )'
+      ssh.run!
+
+      # Then run any other stuff they wanted
+      cmds = TasteTester::Hooks.test_remote_cmds(
+        TasteTester::Config.dryrun,
+        @name
+      )
+
+      if cmds && cmds.any?
+        ssh = TasteTester::SSH.new(@name)
+        cmds.each { |c| ssh << c }
+        ssh.run!
+      end
+    end
+
+    def untest
+      logger.warn("Removing #{@name} from taste-tester")
+      ssh = TasteTester::SSH.new(@name)
+      TasteTester::Tunnel.kill(@name)
+      ssh << 'rm -vf /etc/chef/client.rb'
+      ssh << 'rm -vf /etc/chef/client-taste-tester.rb'
+      ssh << 'ln -vs /etc/chef/client-prod.rb /etc/chef/client.rb'
+      ssh << 'rm -vf /etc/chef/client.pem'
+      ssh << 'ln -vs /etc/chef/client-prod.pem /etc/chef/client.pem'
+      ssh << "rm -vf #{TasteTester::Config.timestamp_file}"
+      ssh << 'logger -t taste-tester Returning server to production'
+      ssh.run!
+    end
+
+    def who_is_testing
+      ssh = TasteTester::SSH.new(@name)
+      ssh << 'grep "^# TasteTester by" /etc/chef/client.rb'
+      output = ssh.run
+      if output.first == 0
+        user = output.last.match(/# TasteTester by (.*)$/)
+        if user
+          return user[1]
+        end
+      end
+
+      # Legacy FB stuff, remove after migration. Safe for everyone else.
+      ssh = TasteTester::SSH.new(@name)
+      ssh << 'file /etc/chef/client.rb'
+      output = ssh.run
+      if output.first == 0
+        user = output.last.match(/client-(.*)-(taste-tester|test).rb/)
+        if user
+          return user[1]
+        end
+      end
+
+      return nil
+    end
+
+    def in_test?
+      ssh = TasteTester::SSH.new(@name)
+      ssh << "test -f #{TasteTester::Config.timestamp_file}"
+      if ssh.run.first == 0 && who_is_testing && who_is_testing != ENV['USER']
+        true
+      else
+        false
+      end
+    end
+
+    def keeptesting
+      logger.warn("Renewing taste-tester on #{@name} until" +
+        " #{TasteTester::Config.testing_end_time}")
+      TasteTester::Tunnel.kill(@name)
+      @tunnel = TasteTester::Tunnel.new(@name, @server)
+      @tunnel.run
+    end
+
+    private
+
+    def config
+      if TasteTester::Config.use_ssh_tunnels
+        url = "http://localhost:#{@tunnel.port}"
+      else
+        url = "http://#{@server.host}:#{TasteTester::State.port}"
+      end
+      ttconfig = <<-eos
+# TasteTester by #{@user}
+# Prevent people from screwing up their permissions
+if Process.euid != 0
+  puts 'Please run chef as root!'
+  Process.exit!
+end
+
+log_level                :info
+log_location             STDOUT
+chef_server_url          '#{url}'
+Ohai::Config[:plugin_path] << '/etc/chef/ohai_plugins'
+
+      eos
+
+      extra = TasteTester::Hooks.test_remote_client_rb_extra_code(@name)
+      if extra
+        ttconfig += <<-eos
+# Begin user-hook specified code
+        #{extra}
+# End user-hook secified code
+
+        eos
+      end
+
+      ttconfig += <<-eos
+puts 'INFO: Running on #{@name} in taste-tester by #{@user}'
+      eos
+      return ttconfig
+    end
+  end
+end

data/lib/taste_tester/logging.rb

@@ -0,0 +1,55 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+# rubocop:disable ClassVars, UnusedMethodArgument, UnusedBlockArgument
+require 'logger'
+
+module TasteTester
+  # Logging wrapper
+  module Logging
+    @@use_log_formatter = false
+    @@level = Logger::WARN
+    @@formatter_proc = nil
+
+    def logger
+      logger = Logging.logger
+      logger.formatter = formatter
+      logger.level = @@level
+      logger
+    end
+
+    def self.logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def self.formatterproc=(p)
+      @@formatter_proc = p
+    end
+
+    def self.use_log_formatter=(use_log_formatter)
+      @@use_log_formatter = use_log_formatter
+    end
+
+    def self.verbosity=(level)
+      @@level = level
+    end
+
+    def formatter
+      return @@formatter_proc if @@formatter_proc
+      if @@use_log_formatter
+        proc do |severity, datetime, progname, msg|
+          if severity == 'ERROR'
+            msg = msg.red
+          end
+          "[#{datetime.strftime('%Y-%m-%dT%H:%M:%S%:z')}] #{severity}: #{msg}\n"
+        end
+      else
+        proc do |severity, datetime, progname, msg|
+          msg.to_s.prepend("#{severity}: ") unless severity == 'WARN'
+          if severity == 'ERROR'
+            msg = msg.to_s.red
+          end
+          "#{msg}\n"
+        end
+      end
+    end
+  end
+end

data/lib/taste_tester/server.rb

@@ -0,0 +1,122 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+require 'fileutils'
+require 'socket'
+require 'timeout'
+
+require 'between_meals/util'
+require 'taste_tester/config'
+require 'taste_tester/state'
+
+module TasteTester
+  # Stateless chef-zero server management
+  class Server
+    include TasteTester::Config
+    include TasteTester::Logging
+    extend ::BetweenMeals::Util
+
+    attr_accessor :user, :host
+
+    def initialize
+      @state = TasteTester::State.new
+      @ref_file = TasteTester::Config.ref_file
+      ref_dir = File.dirname(File.expand_path(@ref_file))
+      @zero_path = TasteTester::Config.chef_zero_path
+      unless File.directory?(ref_dir)
+        begin
+          FileUtils.mkpath(ref_dir)
+        rescue => e
+          logger.warn("Chef temp dir #{ref_dir} missing and can't be created")
+          logger.warn(e)
+        end
+      end
+
+      @user = ENV['USER']
+
+      # If we are using SSH tunneling listen on localhost, otherwise listen
+      # on all addresses - both v4 and v6. Note that on localhost, ::1 is
+      # v6-only, so we default to 127.0.0.1 instead.
+      @addr = TasteTester::Config.use_ssh_tunnels ? '127.0.0.1' : '::'
+      @host = 'localhost'
+    end
+
+    def start
+      return if TasteTester::Server.running?
+      @state.wipe
+      logger.warn('Starting taste-tester server')
+      write_config
+      start_chef_zero
+    end
+
+    def stop
+      logger.warn('Stopping taste-tester server')
+      stop_chef_zero
+    end
+
+    def restart
+      logger.warn('Restarting taste-tester server')
+      if TasteTester::Server.running?
+        stop_chef_zero
+      end
+      write_config
+      start_chef_zero
+    end
+
+    def port
+      @state.port
+    end
+
+    def port=(port)
+      @state.port = port
+    end
+
+    def latest_uploaded_ref
+      @state.ref
+    end
+
+    def latest_uploaded_ref=(ref)
+      @state.ref = ref
+    end
+
+    def self.running?
+      if TasteTester::State.port
+        return port_open?(TasteTester::State.port)
+      end
+      false
+    end
+
+    private
+
+    def write_config
+      knife = BetweenMeals::Knife.new(
+        :logger => logger,
+        :user => @user,
+        :host => @host,
+        :port => port,
+        :role_dir => TasteTester::Config.roles,
+        :cookbook_dirs => TasteTester::Config.cookbooks,
+        :checksum_dir => TasteTester::Config.checksum_dir,
+      )
+      knife.write_user_config
+    end
+
+    def start_chef_zero
+      @state.wipe
+      @state.port = TasteTester::Config.chef_port
+      logger.info("Starting chef-zero of port #{@state.port}")
+      Mixlib::ShellOut.new(
+        "/opt/chef/embedded/bin/chef-zero --host #{@addr}" +
+        " --port #{@state.port} -d"
+      ).run_command.error!
+    end
+
+    def stop_chef_zero
+      @state.wipe
+      logger.info('Killing your chef-zero instances')
+      s = Mixlib::ShellOut.new("pkill -9 -u #{ENV['USER']} -f bin/chef-zero")
+      s.run_command
+      # You have to give it a moment to stop or the stat fails
+      sleep(1)
+    end
+  end
+end

data/lib/taste_tester/ssh.rb

@@ -0,0 +1,60 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+module TasteTester
+  # Thin ssh wrapper
+  class SSH
+    include TasteTester::Logging
+    include BetweenMeals::Util
+
+    attr_reader :output
+
+    def initialize(host, timeout = 5, tunnel = false)
+      @host = host
+      @timeout = timeout
+      @tunnel = tunnel
+      @cmds = []
+    end
+
+    def add(string)
+      @cmds << string
+    end
+
+    alias_method :<<, :add
+
+    def run
+      @status, @output = exec(cmd, logger)
+    end
+
+    def run!
+      @status, @output = exec!(cmd, logger)
+    rescue => e
+      # rubocop:disable LineLength
+      error = <<-MSG
+SSH returned error while connecting to #{TasteTester::Config.user}@#{@host}
+The host might be broken or your SSH access is not working properly
+Try doing 'ssh -v #{TasteTester::Config.user}@#{@host}' and come back once that works
+MSG
+      # rubocop:enable LineLength
+      error.lines.each { |x| logger.error x.strip }
+      logger.error(e.message)
+    end
+
+    private
+
+    def cmd
+      @cmds.each do |cmd|
+        logger.info("Will run: '#{cmd}' on #{@host}")
+      end
+      cmds = @cmds.join(' && ')
+      cmd = "ssh -T -o BatchMode=yes -o ConnectTimeout=#{@timeout} "
+      cmd += "#{TasteTester::Config.user}@#{@host} "
+      if TasteTester::Config.user != 'root'
+        cc = Base64.encode64(cmds).gsub(/\n/, '')
+        cmd += "\"echo '#{cc}' | base64 --decode | sudo bash -x\""
+      else
+        cmd += "\'#{cmds}\'"
+      end
+      cmd
+    end
+  end
+end

data/lib/taste_tester/state.rb

@@ -0,0 +1,87 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+require 'fileutils'
+require 'socket'
+require 'timeout'
+
+require 'between_meals/util'
+require 'taste_tester/config'
+
+module TasteTester
+  # State of taste-tester processes
+  class State
+    include TasteTester::Config
+    include TasteTester::Logging
+    include ::BetweenMeals::Util
+
+    def initialize
+      ref_dir = File.dirname(File.expand_path(
+        TasteTester::Config.ref_file
+      ))
+      unless File.directory?(ref_dir)
+        begin
+          FileUtils.mkpath(ref_dir)
+        rescue => e
+          logger.error("Chef temp dir #{ref_dir} missing and can't be created")
+          logger.error(e)
+          exit(1)
+        end
+      end
+    end
+
+    def port
+      TasteTester::State.read(:port)
+    end
+
+    def port=(port)
+      write(:port, port)
+    end
+
+    def ref
+      TasteTester::State.read(:ref)
+    end
+
+    def ref=(ref)
+      write(:ref, ref)
+    end
+
+    def self.port
+      TasteTester::State.read(:port)
+    end
+
+    def wipe
+      if TasteTester::Config.ref_file &&
+          File.exists?(TasteTester::Config.ref_file)
+        File.delete(TasteTester::Config.ref_file)
+      end
+    end
+
+    private
+
+    def write(key, value)
+      begin
+        state = JSON.parse(File.read(TasteTester::Config.ref_file))
+      rescue
+        state = {}
+      end
+      state[key.to_s] = value
+      ff = File.open(
+        TasteTester::Config.ref_file,
+        'w'
+      )
+      ff.write(state.to_json)
+      ff.close
+    rescue => e
+      logger.error('Unable to write the reffile')
+      logger.debug(e)
+      exit 0
+    end
+
+    def self.read(key)
+      JSON.parse(File.read(TasteTester::Config.ref_file))[key.to_s]
+    rescue => e
+      logger.debug(e)
+      nil
+    end
+  end
+end

data/lib/taste_tester/tunnel.rb

@@ -0,0 +1,53 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+
+module TasteTester
+  # Thin ssh tunnel wrapper
+  class Tunnel
+    include TasteTester::Logging
+    include BetweenMeals::Util
+
+    attr_reader :port
+
+    def initialize(host, server, timeout = 5)
+      @host = host
+      @server = server
+      @timeout = timeout
+      if TasteTester::Config.testing_until
+        @delta_secs = TasteTester::Config.testing_until.strftime('%s').to_i -
+          Time.now.strftime('%s').to_i
+      else
+        @delta_secs = TasteTester::Config.testing_time
+      end
+    end
+
+    def run
+      @port = TasteTester::Config.tunnel_port
+      logger.info("Setting up tunnel on port #{@port}")
+      @status, @output = exec!(cmd, logger)
+    rescue
+      logger.error 'Failed bringing up ssh tunnel'
+      exit(1)
+    end
+
+    def cmd
+      cmds = "echo \\\$\\\$ > #{TasteTester::Config.timestamp_file} &&" +
+      " touch -t #{TasteTester::Config.testing_end_time}" +
+      " #{TasteTester::Config.timestamp_file} && sleep #{@delta_secs}"
+      cmd = "ssh -T -o BatchMode=yes -o ConnectTimeout=#{@timeout} " +
+        "-o ExitOnForwardFailure=yes -f -R #{@port}:localhost:" +
+        "#{@server.port} root@#{@host} \"#{cmds}\""
+      cmd
+    end
+
+    def self.kill(name)
+      ssh = TasteTester::SSH.new(name)
+      # Since commands are &&'d together, and we're using &&, we need to
+      # surround this in paryns, and make sure as a whole it evaluates
+      # to true so it doesn't mess up other things... even though this is
+      # the only thing we're currently executing in this SSH.
+      ssh << "( [ -s #{TasteTester::Config.timestamp_file} ]" +
+        " && kill -- -\$(cat #{TasteTester::Config.timestamp_file}); true )"
+      ssh.run!
+    end
+  end
+end