tdd_deploy 0.0.1

data/lib/tdd_deploy/environ.rb

@@ -0,0 +1,272 @@
+module TddDeploy
+
+    # == module TddDeploy::Environ
+    #
+    # provides management for host provisioning and deployment variables - aka 'the environment'.
+    # The 'enviornment' - as used here - is not the Ruby ENV hash
+    #
+    # three types of variables are supported
+    #  :int - which are integers
+    #  :string - which are strings
+    #  :list - which are input as comma separated strings [the actual separator is /[\s,]+/ ]
+    #    which are used internally as arrays of strings. So, to set a :list variable you
+    #    write 'foo = bar,baz,boop', and when you use it you get back ['bar', 'baz', 'boop']
+    #    (this inanity was done so we could save the environment as a simple ascii file)
+    #
+    # Environment variables:
+    #
+    # === Integer variables
+    # * 'ssh_timeout' - timeout in seconds used to terminate remote commands fired off by ssh
+    # * 'site_base_port' - Base port for site servers. For example, if a pack of mongrels or thin
+    # servers provide the rails end of the web site, they listen on 'site_base_port', +1, etd
+    # * 'site_num_servers' - number of mongrels or thins to spin up
+    #
+    # === String variables
+    # * 'host_admin' - user name used on remote hosts. Should not be root, but should be in /etc/sudoers
+    # * 'local_admin' - user name of on local hosts which can ssh into remote hosts via public key authentication
+    # * 'local_admin_email' - email of local admin who should receive monitoring emails
+    # * 'site' - name of site This should satisfy /[a-z][a-z0-9_]*.
+    # * 'site_user' - name of site user. TddDeploy assumes that each site will have a unique user on the remote host.
+    # this makes it easy to tell nginx and monit where to find configuration files for each site [both
+    # know how to included globbed paths]
+    #
+    # === List Variables
+    # * 'hosts' - list of all hosts - defaults to balance_hosts + db_hosts + web_hosts
+    # * 'balance_hosts' - load balancing servers [may be empty, in which case 'hosts' is used]
+    # * 'db_hosts' - hosts which run the database server [may be empty, in which case 'hosts' is used]
+    # * 'web_hosts' - hosts which run the web server [may be empty, in which case 'hosts' is used]
+    
+  module Environ
+    
+    # == DataCache
+    #
+    # DataCache is a hack to provide a single shared data store for all classes which
+    # include TddDeploy::Environ
+    class DataCache
+      class << self
+        attr_accessor :env_hash, :env_types, :env_defaults
+      end
+    end
+    
+    ENV_FNAME = 'site_host_setup.env'
+
+    # set up all the standard accessors
+    # lazy initialize DataCache.env_hash
+    def env_hash
+      read_env || reset_env unless defined?(DataCache.env_hash)
+      DataCache.env_hash
+    end
+    
+    def env_hash=(hash)
+      raise ArgumentError.new("env_hash=(): arg must be a hash") unless hash.is_a? Hash
+      if !(tmp = hash.keys - DataCache.env_types.keys).empty?
+        raise ArgumentError.new("env_hash=(): Illegal Keys in value: #{tmp.join(',')}")
+      elsif !(tmp = DataCache.env_types.keys - hash.keys).empty?
+        raise ArgumentError.new("env_hash=(): Missing Keys in value: #{tmp.join(',')}")
+      else
+        DataCache.env_hash = hash
+      end
+    end
+    
+    DataCache.env_types = {
+      'ssh_timeout' => :int,
+      'site_base_port' => :int,
+      'site_num_servers' => :int,
+
+      'host_admin' => :string,
+      'local_admin' => :string,
+      'local_admin_email' => :string,
+
+      'site' => :string,
+      'site_user' => :string,
+
+      # 'hosts' => :list,
+      'balance_hosts' => :list,
+      'db_hosts' => :list,
+      'web_hosts' => :list,
+    }
+    
+    DataCache.env_defaults ||= {
+      'ssh_timeout' => 5,
+      'site_base_port' => 8000,
+      'site_num_servers' => 3,
+
+      'host_admin' => "host_admin",
+      'local_admin' => "local_admin",
+      'local_admin_email' => "local_admin@bogus.tld",
+
+      'site' => "site",
+      'site_user' => "site_user",
+
+      # 'hosts' => "bar,foo",
+      'balance_hosts' => '',
+      'db_hosts' => 'bar,foo',
+      'web_hosts' => 'bar,foo',
+    }
+    
+    def env_types
+      DataCache.env_types
+    end
+
+    def env_defaults
+      DataCache.env_defaults
+    end
+
+    # set_env(value_hash {}) - convenience method which sets values of the environment
+    # hash using a hash rather than one-at-a-time
+    def set_env(value_hash = {})
+      DataCache.env_hash ||= {}
+      value_hash.each do |k, v|
+        k = k.to_s
+        case self.env_types[k]
+        when :int then DataCache.env_hash[k] = v.to_i
+        when :string then DataCache.env_hash[k] = v.to_s
+        when :list then DataCache.env_hash[k] = self.str_to_list(v)
+        else
+          if k == 'hosts'
+            if DataCache.env_hash['web_hosts'] == DataCache.env_hash['db_hosts']
+              DataCache.env_hash['web_hosts'] =
+                DataCache.env_hash['db_hosts'] = self.str_to_list(v)
+            else
+              raise RuntimeError.new("#{self}#reset_env(): Cannot assign value to 'hosts' if web_hosts &/or db_hosts already set.\n web_hosts: #{DataCache.env_hash['web_hosts']}\n db_hosts: #{DataCache.env_hash['db_hosts']}")
+              # raise RuntimeError.new("Cannot change hosts key if web_hosts != db_hosts")
+            end
+          else
+            raise ArgumentError.new("#{self}#reset_env(): Illegal environment key: #{k}")
+          end
+        end
+      end
+    end
+    
+    def clear_env
+      DataCache.env_hash = {}
+    end
+    
+    # reset_env resets env_hash to env_defaults
+    def reset_env
+      clear_env
+      set_env self.env_defaults
+    end
+
+    # reads the environment from TddDeploy::Environ::ENV_FNAME (site_host_setup.env) if the file exists
+    # someplace between the current directory and the root of the filesystem
+    def read_env
+      dir_path = Dir.pwd
+      DataCache.env_hash ||= {}
+      loop do
+        path = File.join dir_path, TddDeploy::Environ::ENV_FNAME
+        if File.exists? TddDeploy::Environ::ENV_FNAME
+          line_no = 0
+          if f = File.new(path, 'r')
+            begin
+              f.each do |line|
+                line_no += 1
+                if line =~ /^\s*(\w+)\s*=\s*(.*?)\s*$/
+                  key = $1.downcase
+                  if self.env_types.keys.include? key
+                    self.send "#{key}=".to_sym, $2
+                    # self.env_hash[key] = self.env_types[key] == :list ? self.str_to_list($2) : $2.to_s
+                  else
+                    raise ArugmentError.new("TddDeploy::Environ#read_env: Error in #{TddDeploy::Error::ENV_FNAME}: #{line_no}: Illegal Key: #{key}")
+                  end
+                else
+                  raise ArugmentError.new("TddDeploy::Environ#read_env: Error in #{TddDeploy::Error::ENV_FNAME}: #{line_no}: Unmatched Line: #{line}}")
+                end
+              end
+            ensure
+              f.close
+            end
+            return self.env_hash
+          else
+            raise RuntimeError.new("Unable to open #{path} for reading")
+          end
+        elsif dir_path.length <= 1
+          # reached root level, so initialize to defaults and exit
+          return nil
+        else
+          # move to parent directory
+          dir_path = File.expand_path('..', dir_path)
+        end
+      end
+      nil
+    end
+
+    def str_to_list str
+      case
+      when str.is_a?(String) then str.split(/[\s,]+/).uniq.sort
+      when str.is_a?(Array) then str.uniq.sort
+      else
+        raise ArgumentError.new("str_to_list: #{str}")
+      end
+    end
+    
+    def list_to_str key
+      tmp = self.env_hash[key]
+      tmp.is_a?(Array) ? tmp.join(',') : tmp.to_s
+    end
+
+    # saves the current environment in the current working directory in the file
+    # 'site_host_setup.env' [aka TddDeploy::Environ::ENV_FNAME]
+    def save_env
+      f = File.new(TddDeploy::Environ::ENV_FNAME, "w")
+      self.env_types.keys.each do |k|
+        v = self.env_hash[k] || ''
+        case self.env_types[k]
+        when :int then f.write "#{k}=#{v}\n"
+        when :string then f.write "#{k}=#{v}\n"
+        when :list then
+          f.write "#{k}=#{self.list_to_str(k)}\n" unless k == 'hosts'
+        else
+          raise RuntimeError("unknown key: #{k}")
+        end
+      end
+      f.close
+    end
+
+    # accessors for all defined env variables
+    def hosts
+      (self.web_hosts.to_a + self.db_hosts.to_a + self.balance_hosts.to_a).uniq.sort
+    end
+    
+    def hosts=(list)
+      if (self.web_hosts.nil? && self.db_hosts.nil?) || self.web_hosts == self.db_hosts
+        self.web_hosts =
+          self.db_hosts = self.str_to_list(list)
+      else
+        raise RuntimeError.new("Cannot assign value to 'hosts' if web_hosts &/or db_hosts already set.\n web_hosts: #{self.web_hosts}\n db_hosts: #{self.db_hosts}")
+      end
+    end
+
+    # create accessors for all keys in env_types
+    tmp = ''
+    DataCache.env_types.each do |k, t|
+      tmp +=<<-EOF
+      def #{k}
+        self.env_hash['#{k}']
+      end
+      EOF
+      case DataCache.env_types[k]
+      when :int
+        tmp +=<<-EOF
+        def #{k}=(v)
+          self.env_hash['#{k}'] = v.to_i
+        end
+        EOF
+      when :string
+        tmp +=<<-EOF
+        def #{k}=(v)
+          self.env_hash['#{k}'] = v.to_s
+        end
+        EOF
+      when :list
+        tmp +=<<-EOF
+        def #{k}=(v)
+         self.env_hash['#{k}'] = self.str_to_list(v)
+        end
+        EOF
+      end
+    end
+
+    class_eval tmp
+  end
+end

data/lib/tdd_deploy/host_tests/host_connection.rb

@@ -0,0 +1,29 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  class HostConnection < TddDeploy::Base
+    # ping - pings all hosts
+    def ping
+      require 'net/ping'
+      result = true
+      self.hosts.each do |host|
+        result &= assert Net::Ping::External.new(host).ping?, "Host #{host} should respond to ping"
+       end
+       result
+    end
+
+    # ssh_login - attempts to log in as *host_admin* on all hosts from current user
+    def ssh_login
+      deploy_test_on_all_hosts "/home/#{self.host_admin}\n", "unable to connect via ssh" do
+        'pwd'
+      end
+    end
+    
+    # ssh_login_as_root - attempts to log in as *root* on all hosts from current user
+    def ssh_login_as_root
+      deploy_test_on_all_hosts_as 'root', '/root', "unable to connect as root via ssh" do
+        'pwd'
+      end
+    end
+  end
+end

data/lib/tdd_deploy/host_tests/remote_ip_tables.rb

@@ -0,0 +1,34 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  # = TddDeploy::RemoteIpTables
+  #
+  # checks to see if iptables is working by attempting to connect to each host on a collection
+  # of 'interesting' ports. the ports probed are: 20, 23, 25, 53, 5432, 2812
+  #
+  class RemoteIpTables < TddDeploy::Base
+    # tcp_some_blocked_ports - checks TCP ports
+    def tcp_some_blocked_ports
+      self.hosts.each do |host|
+        # Linode seems to refuse to block 21 - FTP control
+        #  [20, 21, 23, 25, 53, 5432, 2812].each do |port|
+        [20, 23, 25, 53, 5432, 2812].each do |port|
+          tcp_socket = TCPSocket.new(host, port) rescue 'failed'
+          assert_equal 'failed', tcp_socket, "Host: #{host}: Should not be able to connect via tcp to port #{port}"
+        end
+      end
+    end
+
+    # udp_some_blocked_ports - checks UDP ports
+    def udp_some_blocked_ports
+      self.hosts.each do |host|
+        # Linode seems to refuse to block 21 - FTP control
+        #  [20, 21, 23, 25, 53, 5432, 2812].each do |port|
+        [20, 23, 25, 53, 5432, 2812].each do |port|
+          udp_socket = UDPSocket.new(host, port) rescue 'failed'
+          assert_equal 'failed', udp_socket, "Host: #{host}: Should not be able to connect via udp to port #{port}"
+        end
+      end
+    end
+  end
+end

data/lib/tdd_deploy/host_tests/remote_monit.rb

@@ -0,0 +1,17 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  class RemoteMonit < TddDeploy::Base
+    def test_monit_installed
+      deploy_test_on_all_hosts '/usr/bin/monit', "monit is not installed" do
+       'ls /usr/bin/monit'
+      end
+    end
+
+    def test_monit_running
+      deploy_test_on_all_hosts /\smonit\s/, "monit is not running" do
+       'ps -p `cat /var/run/monit.pid`'
+      end
+    end
+  end
+end

data/lib/tdd_deploy/host_tests/remote_nginx.rb

@@ -0,0 +1,17 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  class RemoteNginx < TddDeploy::Base
+    def test_nginx_installed
+      deploy_test_on_all_hosts '/usr/sbin/nginx', "nginx is not installed" do
+        'ls /usr/sbin/nginx'
+      end
+    end
+
+    def test_nginx_running
+      deploy_test_on_all_hosts /\snginx\s/, "nginx is not running" do
+        'ps -p `cat /var/run/nginx.pid`'
+      end
+    end
+  end
+end

data/lib/tdd_deploy/host_tests/remote_postfix.rb

@@ -0,0 +1,23 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  class RemotePostfix < TddDeploy::Base
+    def test_postfix_installed
+      deploy_test_on_all_hosts '/usr/sbin/postfix', "postfix is not installed" do
+        'ls /usr/sbin/postfix'
+      end
+    end
+
+    def test_postfix_running
+      deploy_test_on_all_hosts_as 'root', /\smaster\s/, "postfix is not running" do
+        'ps -p `cat /var/spool/postfix/pid/master.pid`'
+      end
+    end
+
+    # def test_postfix_accepts_mail
+    #   deploy_test_on_all_hosts "mail works\n", 'postfix accepts mail' do |host, login, userid|
+    #     "echo \"test mail\" | mail -s 'test mail' -r #{userid}@#{host} #{self.local_admin_email} && echo 'mail works'"
+    #   end
+    # end
+  end
+end

data/lib/tdd_deploy/host_tests/remote_postgresql.rb

@@ -0,0 +1,17 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  class RemotePostgresql < TddDeploy::Base
+    def test_postgresql_installed
+      deploy_test_on_all_hosts "/usr/bin/postgres\n", 'postgres not installed' do |host, login, admin|
+        "ls /usr/bin/postgres"
+      end
+    end
+
+    def test_postgresql_running
+      deploy_test_on_all_hosts /postgres\s*\|\s*postgres/, "postgresql server not running" do
+        "psql --command='\\l' postgres postgres"
+      end
+    end
+  end
+end

data/lib/tdd_deploy/railengine.rb

@@ -0,0 +1,10 @@
+module TddDeploy
+  class Engine < Rails::Engine
+    # engine_name "tdd_deploy"
+    rake_tasks do
+      load "tdd_deploy/../tasks/tdd_deploy.rake"
+    end
+    initializer 'active_support.add_tdd_deploy' do
+    end
+  end
+end

data/lib/tdd_deploy/run_methods.rb

@@ -0,0 +1,126 @@
+module TddDeploy
+  module RunMethods
+    require 'net/ssh'
+    
+    # runs the output of the block on all hosts defined in self.hosts as user self.host_admin.
+    # Returns a hash of two element arrays containing output [stdout, stderr] returned from the command.
+    # Hash keys are host names as strings.
+    def run_on_all_hosts(&block)
+      run_on_all_hosts_as self.host_admin, &block
+    end
+
+    # Runs the output of the block on all hosts defined in self.hosts as user 'userid'.
+    # Returns a hash of two element arrays containing output [stdout, stderr] returned from the command.
+    # Hash keys are host names as strings.
+    def run_on_all_hosts_as(userid, &block)
+      results = {}
+      self.hosts.each do |host|
+        results[host] = run_in_ssh_session_as(userid, host, &block)
+      end
+      results
+    end
+
+    # Runs the command secified in &block on 'host' as user 'self.host_admin'.
+    # Returns an array [stdout, stderr] returned from the command.
+    def run_in_ssh_session(host, &block)
+      run_in_ssh_session_as(self.host_admin, host, &block)
+    end
+
+    # Runs the command secified in &block on 'host' as user 'userid'.
+    # Returns an array [stdout, stderr] returned from the command.
+    def run_in_ssh_session_as(userid, host, &block)
+      login = "#{userid}@#{host}"
+      match = Regexp.new(match) if match.is_a? String
+      raise ArgumentError, 'match expression cannot be empty' if match =~ ''
+
+      rsp = nil
+      err_rsp = nil
+      cmd = block.call(host, login, userid)
+
+      begin
+        ssh_session = Net::SSH.start(host, userid, :timeout => self.ssh_timeout)
+        raise "Unable to establish connecton to #{host} as #{userid}" if ssh_session.nil?
+
+        ssh_session.open_channel do |channel|
+          channel.exec(cmd) do |ch, success|
+            ch.on_data do |chn, data|
+              rsp ||= ''
+              rsp += data.to_s
+            end
+
+            ch.on_extended_data do |chn, data|
+              err_rsp ||= ''
+              err_rsp += data.to_s
+            end
+
+          end
+        end
+
+        # must do this or the channel only runs once
+        ssh_session.loop(5)
+
+        ssh_session.close
+      rescue Exception => e
+        err_rsp = "error talking to #{host} as #{userid}: #{e.message}"
+        ssh_session.shutdown! unless ssh_session.nil?
+      end
+      [rsp, err_rsp, cmd]
+    end
+  
+    # run locally runs a comman locally and returns the output of stdout, stderr, and the command
+    # run in a 3 element array
+    #
+    # to send input to the subprocess, include the optional 'stdin_text' parameter. Don't
+    # forget to add newlines, if you need them.
+    def run_locally(stdin_text = nil, &block)
+      raise ArgumentError.new('block required') unless block_given?
+
+      cmd = block.call
+      # cmd = "echo '#{stdin_text}' | #{cmd}" if stdin_text
+
+      raise "Unable to run_locally - fork method not useable" unless Process.respond_to? :fork
+
+      # preload stdin if there is input to avoid a race condition
+      if stdin_text
+        stdin_pipe, child_stdin = IO.pipe if stdin_text
+        count = child_stdin.write(stdin_text.to_s)
+        child_stdin.close
+      end
+
+      child_stdout, stdout_pipe = IO.pipe
+      child_stderr, stderr_pipe = IO.pipe
+      unless (pid = Process.fork)
+        if stdin_text
+          STDIN.reopen(stdin_pipe)
+        else
+          STDIN.close
+        end
+        STDOUT.reopen(stdout_pipe)
+        STDERR.reopen(stderr_pipe)
+        begin
+          Process.exec ENV, cmd
+        rescue SystemCallError => e
+          STDERR.write("Unable to execute command '#{cmd}'\n  #{e}")
+        ensure
+          exit
+        end
+      end
+
+      Process.wait
+
+      stdout = ''
+      stderr = ''
+      loop do
+        reads, writes, obands = IO.select([child_stdout, child_stderr], [], [], 1)
+        break if reads.nil?
+        stdout += child_stdout.read_nonblock(1024) if reads.include? child_stdout
+        stderr += child_stderr.read_nonblock(1024) if reads.include? child_stderr
+      end
+      stdout = nil if stdout.empty?
+      stderr = nil if stderr.empty?
+
+      [stdout, stderr, cmd]
+    end
+
+  end
+end

data/lib/tdd_deploy/server.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+$:.unshift File.expand_path('../lib', __FILE__)
+
+require 'tdd_deploy'
+
+module TddDeploy
+  class Server < TddDeploy::Base
+    LIB_DIR = File.expand_path('../..', __FILE__)
+    HOST_TESTS_DIR = File.join(Dir.pwd, 'lib', 'tdd_deploy', 'host_tests')
+    SITE_TESTS_DIR = File.join(Dir.pwd, 'lib', 'tdd_deploy', 'site_tests')
+    LOCAL_TESTS_DIR = File.join(Dir.pwd, 'lib', 'tdd_deploy', 'local_tests')
+
+    attr_accessor :test_classes
+  
+    def initialize *args
+      @already_defined = TddDeploy.constants
+      super
+    end
+
+    def load_all_tests
+      if TddDeploy::Base.children == [self.class]
+        [TddDeploy::Server::HOST_TESTS_DIR, TddDeploy::Server::SITE_TESTS_DIR,
+            TddDeploy::Server::LOCAL_TESTS_DIR].each do |dir|
+          if File.exists?(dir)
+            puts "gathering tests from #{dir}"
+            Dir.new(dir).each do |fname|
+              require File.join(dir, fname) unless fname[0] == '.'
+            end
+          else
+            puts "skipping #{dir} - no such directory"
+          end
+        end
+      end
+      self.test_classes = TddDeploy::Base.children - [self.class]
+    end
+    
+    def run_all_tests
+      load_all_tests
+
+      ret = true
+      self.test_classes.each do |klass|
+        obj = klass.new
+        # puts "#{klass}.instance_methods: #{klass.instance_methods(false)}"
+        klass.instance_methods(false).each do |func|
+          ret &= obj.send func.to_sym
+        end
+      end
+      ret
+    end
+    
+    def call(env)
+      run_all_tests
+      body = ["<h1>TDD Test Results:</h1>", self.test_results]
+      return [200, {'Content-Length' => body.join('').length.to_s, 'Content-Type' => 'text/html'}, body]
+    end
+  end
+end
+

data/lib/tdd_deploy/site_tests/site_layout.rb

@@ -0,0 +1,46 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  # == TddDeploy::SiteLayout
+  #
+  #  tests for the existence of several directories on all hosts as *site_user* in
+  #  the *site_user* home directory.
+  #
+  #  The sub directories tested for are:
+  #
+  #     *site* - a directory named for the name of the site.
+  #     *site*/releases - a standard directory used by Capistrano
+  #     *site*/nginx.conf - an nginx configuratino fragment which tells nginx to
+  #      proxy the site's *thin* servers
+  #     *site*/monitrc - a monit configuration fragment which tells monit how to monitor
+  #      the site's *thin* servers.
+  class SiteLayout < TddDeploy::Base
+    def test_site_subdir
+      deploy_test_on_all_hosts_as self.site_user, "#{self.site}/", \
+          "directory /home/#{self.site_user}/#{self.site} should exist" do
+        'ls -F'
+      end
+    end
+
+    def test_releases_subdir
+      deploy_test_on_all_hosts_as self.site_user, "releases", \
+          "directory /home/#{self.site_user}/#{self.site}/releases should exist" do
+        "ls -F #{self.site}"
+      end
+    end
+
+    def test_monitrc
+      deploy_test_on_all_hosts_as self.site_user, 'monitrc', \
+          "file /home/#{self.site_user}/monitrc should exist" do
+            'ls'
+      end
+    end
+
+    def test_nginx_conf
+      deploy_test_on_all_hosts_as self.site_user, 'nginx.conf', \
+          "file /home/#{self.site_user}/nginx.conf should exist" do
+            'ls'
+      end
+    end
+  end
+end

data/lib/tdd_deploy/site_tests/site_user.rb

@@ -0,0 +1,14 @@
+require 'tdd_deploy/base'
+
+module TddDeploy
+  # = TddDeploy::SiteUser
+  #
+  # tests all hosts to make sure that the local user can log on as *site_user*
+  class SiteUser < TddDeploy::Base
+    def test_login_as_site_user
+      deploy_test_on_all_hosts_as self.site_user, "/home/#{self.site_user}", "unable to log into host as #{self.site_user}" do
+        'pwd'
+      end
+    end
+  end
+end

data/lib/tdd_deploy/version.rb

@@ -0,0 +1,3 @@
+module TddDeploy
+  VERSION = '0.0.1'
+end