VladTheEnterprising 0.1.4

data/files/xen_master/xend-config.sxp

@@ -0,0 +1,139 @@
+# -*- sh -*-
+
+#
+# Xend configuration file.
+#
+
+# This example configuration is appropriate for an installation that 
+# utilizes a bridged network configuration. Access to xend via http
+# is disabled.  
+
+# Commented out entries show the default for that entry, unless otherwise
+# specified.
+
+#(logfile /var/log/xen/xend.log)
+#(loglevel DEBUG)
+
+#(xend-http-server no)
+#(xend-unix-server no)
+#(xend-tcp-xmlrpc-server no)
+#(xend-unix-xmlrpc-server yes)
+#(xend-relocation-server no)
+# The relocation server should be kept desactivated unless using a trusted
+# network, the domain virtual memory will be exchanged in raw form without
+# encryption of the communication. See also xend-relocation-hosts-allow option
+
+#(xend-unix-path /var/lib/xend/xend-socket)
+
+# Port xend should use for the HTTP interface, if xend-http-server is set.
+#(xend-port            8000)
+
+# Port xend should use for the relocation interface, if xend-relocation-server
+# is set.
+#(xend-relocation-port 8002)
+
+# Address xend should listen on for HTTP connections, if xend-http-server is
+# set.
+# Specifying 'localhost' prevents remote connections.
+# Specifying the empty string '' (the default) allows all connections.
+#(xend-address '')
+#(xend-address localhost)
+
+# Address xend should listen on for relocation-socket connections, if
+# xend-relocation-server is set.
+# Meaning and default as for xend-address above.
+#(xend-relocation-address '')
+
+# The hosts allowed to talk to the relocation port.  If this is empty (the
+# default), then all connections are allowed (assuming that the connection
+# arrives on a port and interface on which we are listening; see
+# xend-relocation-port and xend-relocation-address above).  Otherwise, this
+# should be a space-separated sequence of regular expressions.  Any host with
+# a fully-qualified domain name or an IP address that matches one of these
+# regular expressions will be accepted.
+#
+# For example:
+#  (xend-relocation-hosts-allow '^localhost$ ^.*\.example\.org$')
+#
+#(xend-relocation-hosts-allow '')
+(xend-relocation-hosts-allow '^localhost$ ^localhost\\.localdomain$')
+
+# The limit (in kilobytes) on the size of the console buffer
+#(console-limit 1024)
+
+##
+# To bridge network traffic, like this:
+#
+# dom0: fake eth0 -> vif0.0 -+
+#                            |
+#                          bridge -> real eth0 -> the network
+#                            |
+# domU: fake eth0 -> vifN.0 -+
+#
+# use
+#
+(network-script network-bridge)
+#
+# Your default ethernet device is used as the outgoing interface, by default. 
+# To use a different one (e.g. eth1) use
+#
+# (network-script 'network-bridge netdev=eth1')
+#
+# The bridge is named xenbr0, by default.  To rename the bridge, use
+#
+# (network-script 'network-bridge bridge=<name>')
+#
+# It is possible to use the network-bridge script in more complicated
+# scenarios, such as having two outgoing interfaces, with two bridges, and
+# two fake interfaces per guest domain.  To do things like this, write
+# yourself a wrapper script, and call network-bridge from it, as appropriate.
+#
+#(network-script network-dummy)
+
+# The script used to control virtual interfaces.  This can be overridden on a
+# per-vif basis when creating a domain or a configuring a new vif.  The
+# vif-bridge script is designed for use with the network-bridge script, or
+# similar configurations.
+#
+# If you have overridden the bridge name using
+# (network-script 'network-bridge bridge=<name>') then you may wish to do the
+# same here.  The bridge name can also be set when creating a domain or
+# configuring a new vif, but a value specified here would act as a default.
+#
+# If you are using only one bridge, the vif-bridge script will discover that,
+# so there is no need to specify it explicitly.
+#
+(vif-script vif-bridge)
+
+
+## Use the following if network traffic is routed, as an alternative to the
+# settings for bridged networking given above.
+#(network-script network-route)
+#(vif-script     vif-route)
+
+
+## Use the following if network traffic is routed with NAT, as an alternative
+# to the settings for bridged networking given above.
+#(network-script network-nat)
+#(vif-script     vif-nat)
+
+
+# Dom0 will balloon out when needed to free memory for domU.
+# dom0-min-mem is the lowest memory level (in MB) dom0 will get down to.
+# If dom0-min-mem=0, dom0 will never balloon out.
+(dom0-min-mem 256)
+
+# In SMP system, dom0 will use dom0-cpus # of CPUS
+# If dom0-cpus = 0, dom0 will take all cpus available
+(dom0-cpus 0)
+
+# Whether to enable core-dumps when domains crash.
+#(enable-dump no)
+
+# The tool used for initiating virtual TPM migration
+#(external-migration-tool '')
+
+# The interface for VNC servers to listen on. Defaults
+# to 127.0.0.1  To restore old 'listen everywhere' behaviour
+# set this to 0.0.0.0
+#(vnc-listen '127.0.0.1')

data/lib/vlad/dba/mysql.rb

@@ -0,0 +1,445 @@
+require 'date'
+require 'time'
+require 'highline/import'
+require 'monitor'
+require 'resolv'
+require 'vlad/translator'
+require 'vlad/environmentalist'
+require 'digest/md5'
+
+
+namespace :vlad do
+  namespace :mysql do
+    desc "Bootstraps mysql slave candiates, installs mysql, syncs them to the master, and starts them."
+    task :bootstrap_new_slaves => [
+                                   :assert_production_target_subset,
+                                   :assert_dumpfile,
+                                   :assert_new_slaves,
+                                   :assert_single_master,
+                                   :ask_root_password,
+                                   :ask_replication_password,
+                                   :install_profile,
+                                   :revoke_replication,
+                                   :grant_replication,
+                                   :pre_bootstrap,
+                                   :install_mysql,
+                                   :configure_slaves,
+                                   :dump_master,
+                                   :sync_slaves,
+                                   :start_new_slaves,
+                                   :check_slaves,]
+
+
+    task :ask_root_password  do
+      mysql_root_password
+    end
+
+    task :ask_replication_password  do
+      mysql_replication_password
+    end
+
+
+    desc "Builds slaves configs from template, installs on slave candidates, restarts mysql on each"
+    task :configure_slaves => [
+                               :assert_production_target_subset,
+                               :assert_dumpfile,
+                               :assert_new_slaves,
+                               :assert_single_master,
+                               :build_configs,
+                               :copy_configs,
+                               :nfs_copy_configs,
+                               :link_config,
+                               :configure_restart_slaves ]
+
+    task :assert_dumpfile do
+      if ENV["DUMPFILE"]
+        unless File.exist?(ENV["DUMPFILE"])
+          raise "DUMPFILE '#{ENV["DUMPFILE"]} doesn't exit!"
+        end
+      end
+    end
+
+    task :assert_production_target_subset do
+      if target_environment == "production"
+        unless defined? target_subset
+          raise "No target subset given, please specify a database group subset to use i.e. TARGET='staging:vendor_content'"
+        end
+      end
+    end
+
+    task :assert_new_slaves do
+      if ENV['BOOTSTRAP']
+        host = ENV['BOOTSTRAP']
+        unless Rake::RemoteTask::hosts_for(:db_slave).include?(host)
+          raise "Error: '#{host}' not a :db_slave in target subset '#{target_environment}:#{target_subset}'"
+        end
+        role :new_slave, ENV['BOOTSTRAP']
+      end
+      unless Rake::RemoteTask.hosts_for(:new_slave).size > 0
+        raise "No new slaves to bootstrap"
+      end
+    end
+
+    task :assert_single_master do
+      if  Rake::RemoteTask.hosts_for(:db_master).size != 1
+        raise "Error! This task demands one and only one host have the :db_master role assigned"
+      end
+    end
+
+    remote_task :install_profile, :roles => [:db_master, :db_slave, :config_holder, :new_slave, :slave_reset_candidate ] do
+      if File.exist?("files/mysql/bash_profile")
+        `cat files/mysql/bash_profile | ssh #{target_host} "cat > .bash_profile"`
+        run "if [ -e .bashrc ]; then rm .bashrc; fi; ln -s .bash_profile .bashrc"
+        run "if [ -e .profile ]; then rm .profile; fi; ln -s .bash_profile .profile"
+      end
+    end
+
+    remote_task :grant_dump, :roles => :db_master do
+      slaves = Rake::RemoteTask.hosts_for(:new_slave)
+      slaves.each do |slave|
+        name = slave.gsub(/\..*$/,"").gsub(/-/, "")
+        mysql_query "grant all privileges  on *.* to '#{name}_dump'@'#{slave}' identified by '#{mysql_replication_password}'", "", :u => "root", :p => mysql_root_password
+        ip =  Resolv.getaddress(slave)
+        if ip
+          mysql_query "grant all privileges  on *.* to '#{name}_dump'@'#{ip}' identified by '#{mysql_replication_password}'", "", :u => "root", :p => mysql_root_password
+        end
+      end
+    end
+
+    remote_task :grant_replication, :roles => :db_master do
+      slaves = Rake::RemoteTask.hosts_for(:new_slave)
+      slaves.each do |slave|
+        name = slave.gsub(/\..*$/,"").gsub(/-/, "")
+        mysql_query "grant replication slave, super, reload  on *.* to '#{name}_repl'@'#{slave}' identified by '#{mysql_replication_password}'", "", :u => "root", :p => mysql_root_password
+        ip =  Resolv.getaddress(slave)
+        if ip
+          mysql_query "grant replication slave, super, reload  on *.* to '#{name}_repl'@'#{ip}' identified by '#{mysql_replication_password}'", "", :u => "root", :p => mysql_root_password
+        end
+      end
+    end
+
+    remote_task :revoke_replication, :roles => :db_master do
+      slaves = Rake::RemoteTask.hosts_for(:new_slave)
+      slaves.each do |slave|
+        name = slave.gsub(/\..*$/,"").gsub(/-/, "")
+        mysql_query "REVOKE ALL PRIVILEGES, GRANT OPTION FROM #{name}_repl", "", :u => "root", :p => mysql_root_password, :force => true
+        mysql_query "drop user #{name}_repl", "", :u => "root", :p => mysql_root_password, :force => true
+      end
+    end
+
+
+
+    remote_task :revoke_dump, :roles => :db_master do
+      slaves = Rake::RemoteTask.hosts_for(:new_slave)
+      slaves.each do |slave|
+        name = slave.gsub(/\..*$/,"").gsub(/-/, "")
+        mysql_query "REVOKE ALL PRIVILEGES, GRANT OPTION FROM #{name}_dump", "", :u => "root", :p => mysql_root_password, :force => true
+        mysql_query "drop user #{name}_dump", "", :u => "root", :p => mysql_root_password, :force => true
+      end
+    end
+
+
+    remote_task :set_mysql_root_password, :roles => [:new_slave] do
+      run "sudo su - mysql -c  'cd #{mysql_path}; ./bin/mysqladmin -u root password \"#{mysql_root_password}\"' "
+      run "sudo su - mysql -c  'cd #{mysql_path}; ./bin/mysqladmin -u root -h #{target_host} password \"#{mysql_root_password}\"' "
+    end
+
+
+    remote_task :copy_configs, :roles => :new_slave do
+      mysql_config_copy
+    end
+
+    remote_task :nfs_copy_configs, :roles => :config_holder do
+      mysql_config_nfs_copy
+    end
+
+
+    remote_task :link_config, :roles => [:new_slave] do
+      mysql_config_link
+    end
+
+    remote_task :configure_restart_slaves, :roles => [:new_slave] do
+      mysql_config_restart
+    end
+
+    remote_task :build_configs,  :roles => :config_holder do
+      puts "Building slave configs"
+      slaves = Rake::RemoteTask.hosts_for(:new_slave)
+      slaves.each do |hostname|
+        name = hostname.gsub(/\..*$/,"").gsub(/-/, "")
+        destfile =  "files/mysql/configs/#{name}.cnf"
+        template_file = "files/mysql/templates/#{name}.cnf.tmpl"
+        unless File.exist? template_file
+          template_file = "files/mysql/templates/db_slave.cnf.tmpl"
+        end
+        template =  File.read(template_file)
+        server_id = Resolv.getaddress(hostname) + "-" + hostname
+        server_id = server_id.sum.to_s + Resolv.getaddress(hostname).gsub(/\./, "").match(/\d\d\d\d/).to_s
+        template.gsub!(/__SERVER_ID__/,server_id)
+        template.gsub!(/__HOSTNAME__/, name)
+        template.gsub!(/__DATADIR__/, mysql_datadir)
+        rstanza = ""
+        databases.each do |dbname|
+          rstanza << "replicate-do-db             = #{dbname}\n"
+        end
+        template.gsub!(/__REPLICATE_DO_DB__/, rstanza)
+        File.open(destfile, "w") do |file|
+          file.write(template)
+        end
+        puts "Generated: #{destfile}"
+      end
+    end
+
+    remote_task :pre_bootstrap, :roles => [:new_slave] do
+      mysql_slave_pre_bootstrap
+    end
+
+    remote_task :install_mysql, :roles => [:new_slave]  do
+      mysql_install
+    end
+
+    task :check_reset_all_slaves do
+      mysql_data_environments each do |group|
+        Kernel.fork { Kernel.exec "rake #{target_environment}:#{group} vlad:mysql:check_reset_slaves"}
+        Process.wait
+      end
+    end
+    desc "Checks that slaves are in sync, will try to start them if so."
+    task :check_reset_slaves =>  [:ask_root_password, :check_reset_each_slave ]
+    remote_task :check_reset_each_slave, :roles => :db_slave do
+      raise "Must specifiy database group type  in target" unless defined? target_subset
+      begin
+        assert_slave_running
+      rescue
+        puts "#{target_host}: slave DOWN. Restarting."
+        begin
+          start_slave
+          sleep 15
+          assert_slave_running
+        rescue
+          puts  "#{target_host}: restart FAILED."
+        end
+      end
+    end
+
+    remote_task :start_new_slaves, :roles => :new_slave do
+      begin
+        mysql_query "flush privileges; slave start;", "", :u => "root"
+      rescue
+        mysql_query "flush privileges; slave start;", "", :u => "root", :p => mysql_root_password
+      end
+    end
+
+
+    def start_slave
+      mysql_query "flush privileges; slave start;", "", :u => "root"
+    end
+
+    remote_task :dump_master, :roles => :db_master do
+      unless ENV["DUMPFILE"]
+        datfile = mysql_dumpfile.sub(/\.gz$/,"")
+        run "mkdir -p `dirname #{datfile}`; exit 0"
+        run "#{mysqldump} --single-transaction --master-data  -uroot -p'#{mysql_root_password}' -A > #{datfile}"
+        `mkdir -p \`dirname #{mysql_dumpfile}\``
+        run "nice gzip #{datfile}"
+        cmd="ssh #{target_host} cat #{mysql_dumpfile}  > #{mysql_dumpfile}"
+        puts cmd
+        `#{cmd}`
+        run "if [ -e #{mysql_dumpfile} ]; then rm #{mysql_dumpfile}; else exit 1; fi"
+      end
+    end
+
+    remote_task :sync_slaves, :roles => :new_slave  do
+      mysql_dumpfile = ENV["DUMPFILE"] if ENV["DUMPFILE"]
+      run "#{mysql_path_bin}/mysql -vvf -u root -e \"slave stop\;\";  exit 0"
+      run "#{mysql_path_bin}/mysql -vvf -u root -e \"change master to master_host='#{mysql_master_host}', master_user='#{shortname}_repl', master_password='#{mysql_replication_password}';\""
+      run "mkdir -p `dirname #{mysql_dumpfile}`"
+      scp mysql_dumpfile, mysql_dumpfile 
+      run "cat #{mysql_dumpfile} | gunzip - | #{mysql_path_bin}/mysql -uroot"
+      run "rm  #{mysql_dumpfile}"
+    end
+
+    def assert_slave_running
+      cmd = "ssh #{target_host} \"#{mysql} -vvvf -uroot -p'#{mysql_root_password}' -e 'show slave status\\G' 2>/dev/null\""
+      output=`#{cmd}`
+      raise "DOWN" unless output.match(/Slave_IO_Running: Yes/)
+      raise "DOWN" unless output.match(/Slave_SQL_Running: Yes/)
+      puts "#{target_host}: slave OK"
+    end
+
+    def assert_agent_running
+      cmd = ssh #{target_host} "ps -ef | grep mysql | grep agent | grep -v grep"
+      puts cmd
+      output=`#{cmd}`
+      raise "DOWN" unless output.match(/mysql-service-agent/)
+      puts "#{target_host}: agent OK"
+    end
+
+    desc "shows sync status of all slaves"
+    remote_task :check_slaves, :roles => :db_slave  do
+      begin
+        assert_slave_running
+      rescue Exception => e
+        puts "#{target_host}: slave DOWN"
+      end
+    end
+    task :check_slaves  => [:ask_root_password]
+
+    desc "Checks mysql agent status on all slaves"
+    remote_task :check_restart_agents, :roles => [:db_slave, :db_master] do
+      cmd1 = "#{mysql_agent_init} start"
+      cmd2 =  "#{mysql_agent_init_retry} start"
+      begin
+        assert_agent_running
+      rescue
+        begin
+          run cmd1
+        rescue
+          run cmd2
+        end
+      end
+    end
+
+    desc "checks mysql agent status on all servers"
+    remote_task :check_agents, :roles => [:db_slave, :db_master] do
+      begin
+        assert_agent_running
+      rescue Exception => e
+        puts "#{target_host}: agent DOWN"
+      end
+    end
+
+    desc "stops replication on all slaves"
+    remote_task :stop_slaves, :roles => :db_slave do
+      mysql_query "slave stop;", "", :u => "root",  :p => mysql_root_password
+    end
+
+    desc "starts replication on all slaves"
+    remote_task :start_slaves, :roles => :db_slave do
+      mysql_query "flush privileges; slave start;", "", :u => "root",  :p => mysql_root_password
+    end
+    desc "shows available databases"
+    remote_task :show_all_databases, :roles => :db_master do
+      puts "#{target_host}"
+      mysql_query "SHOW DATABASES;", "", :u => "root", :p => mysql_root_password
+    end
+    task :show_all_databases => [:ask_root_password]
+
+    remote_task :install_dot_my_cnf, :roles => :db_slave do
+      cnf = "[client]\n"
+      cnf << "host     = localhost\n"
+      cnf << "user     = root\n"
+      cnf << "password = #{mysql_root_password}\n"
+      File.open("/tmp/my.cnf.#{target_host}", "w") do |file|
+        file.write(cnf)
+      end
+      scp "/tmp/my.cnf.#{target_host}", ".my.cnf"
+    end
+
+    remote_task :remove_dot_my_cnf, :roles => :db_slave do
+      `rm /tmp/my.cnf.#{target_host}; exit 0`
+      run "rm -f .my.cnf; exit 0"
+    end
+  end
+end
+
+dialect( :name => :default_mysql,
+         :assert => lambda{  true },
+         :set => {
+           :mysql_master_host =>  lambda {  Rake::RemoteTask.hosts_for(:db_master).first },
+           :mysql_master_ip =>   lambda { Resolv.getaddress(Rake::RemoteTask.fetch(mysql_master_host)) },
+           :mysql_stop_cmd => lambda { "sudo #{mysql_init} stop" },
+           :mysql_start_cmd => lambda { "sudo #{mysql_init} start" },
+           :mysql_safety_stop_cmd => lambda { "sudo #{mysql_init} stop; exit 0" },
+           :mysql_dumpfile => lambda {  "dumps/#{Rake::RemoteTask::hosts_for(:db_master).first}-#{version}.dat.gz" },
+           :mysqldump => "mysqldump",
+           :mysql => "mysql",
+           :shortname => lambda { target_host.gsub(/\..*$/,"").gsub(/-/,"") },
+
+         })
+
+dialect( :name => :debian_flavor_mysql,
+         :assert => lambda {  run "apt-get -h >/dev/null 2>&1"  },
+         :set => {
+           :mysql_datadir => "/var/lib/mysql",
+           :mysql_config_base => "/etc/mysql/",
+           :mysql_path_bin => "/usr/bin",
+           :mysql_err => "/var/log/mysql.err",
+           :my_cnf => lambda { "/etc/mysql/conf.d/#{shortname}.cnf" },
+           :my_src_cnf => lambda { "files/mysql/configs/#{shortname}.cnf" },
+           :my_tmp_cnf => lambda { "/tmp/my.cnf-#{version}"},
+           :my_dest_cnf => lambda { my_cnf },
+           :mysql_config_nfs_copy => lambda { true },
+           :mysql_config_copy => lambda {
+             if File.exist?(my_src_cnf)
+               rsync my_src_cnf, my_tmp_cnf
+               run "sudo cp #{my_tmp_cnf} #{my_dest_cnf}"
+             end
+           },
+           :mysql_config_link => lambda { true
+
+           },
+           :mysql_config_restart => lambda {
+             run mysql_safety_stop_cmd
+             mysql_config_link
+             run mysql_start_cmd
+           },
+           :mysql_slave_pre_bootstrap  =>  lambda {
+             run mysql_safety_stop_cmd
+             #run "sudo tar cvzf #{mysql_datadir}/#{version}-ibdata-backup.tar.gz #{mysql_datadir}/ib* 2>/dev/null; exit 0;"
+           },
+           :mysql_install => lambda {
+             run "sudo apt-get --force-yes -y install mysql-server mysql-server-5.0 mysql-common"
+             mysql_start_cmd
+           },
+           :mysql_init => "/etc/init.d/mysql",
+         })
+
+
+def mysql_query *args, &b
+  Thread.current[:task].mysql_query(*args, &b)
+end
+
+def mysql_root_password
+  unless defined? current_mysql_root_password
+    pw = ask("Enter mysql root password: "){ |q| q.echo = "x"}
+    set(:current_mysql_root_password, pw)
+  end
+  current_mysql_root_password
+end
+
+
+def mysql_replication_password
+  unless defined? current_mysql_replication_password
+    pw = ask("Enter replication password: "){ |q| q.echo = "x"}
+    set(:current_mysql_replication_password, pw)
+  end
+  current_mysql_replication_password
+end
+
+class Rake::RemoteTask
+  def mysql_query( sql, database="", options = {})
+    if options[:force]
+      force_enabled = true
+      options[:force] = nil
+    end
+    if options[:p] == ""
+      options[:p] = nil
+    end
+    args = options.keys.collect do |key|
+      case key
+      when :p :
+          if options[key] && options[key] != ""
+            "-#{key}'#{options[key]}'"
+          end
+      when :force :
+          ""
+      else
+        "-#{key}'#{options[key]}'"
+      end
+    end.join " "
+    command = "echo \"#{sql}\" | #{mysql} #{args} #{database}"
+    command << "; exit 0" if force_enabled
+    run command
+  end
+end