VladTheEnterprising 0.1.7 → 0.1.8

data/README.txt

@@ -1,6 +1,9 @@
 VladTheEnterprising
     by Michael L. Welles
+    michael@bangstate.com
     http://rubyforge.org/projects/vladenvironment/ 
+    Contributions by:
+       Alexander Rubin - arubin@mysql.com
 
 == DESCRIPTION:
 
@@ -90,6 +93,52 @@ Currently consists of the following:
   necessary stuff on those machines while still working as expected
   on the ubuntu hosts in the subenvironment.
 
+  Migrations and Versioning
+  =========================
+ 
+  As of 1.8, schema migrations and versioning are supported.  By 
+  default the app will look under db/migrations/[schema name]/[version
+  no]-[text]/[upgrade.sql|rollback.sql]
+
+  So, for example. Lets say I have a the folowing files:
+
+  db/migrations/UserContent/1-version_column/rollback.sql
+  db/migrations/UserContent/1-version_column/upgrade.sql
+  db/migrations/UserContent/2-crnr_sp_get_asset_comments/rollback.sql
+  db/migrations/UserContent/2-crnr_sp_get_asset_comments/upgrade.sql
+  db/migrations/UserContent/3-crnr_sp_update_asset_taxonomy_properties/rollback.sql
+  db/migrations/UserContent/3-crnr_sp_update_asset_taxonomy_properties/upgrade.sql
+
+  And I run the following command:
+
+  "rake vlad:mysql:migrate TARGET=testing:user_content"
+
+  VladtheDBA will will perform the following steps on :db_master
+  in my testing:user_content subenvironment:
+
+   - For each database defined there, if database 
+     doesn't have a "schema_info" table, vlad will create one.
+  
+   - It will look for a directory for that schema under db/migrations.
+
+   - If it finds one (in this case, UserContent), it will perform the
+     upgrade.sql for each migration migration subdirectory in
+     numerical order, until the version in the schema info table
+     is up to date with the migrations.
+
+  Running a rollback is as follows: 
+
+  "rake vlad:mysql:migrate TARGET=testing:user_content
+  DATABASE=UserContent VERSION=1"
+
+  Will cause vlad to decremant backwards through the migrations
+  running the DDL in rollback.sql, until it gets back to version 1.
+
+  The idea was a blatant ripoff of activerecord migrations in rails,
+  but just in a framework neutral way, and with the only assumption
+  being that there's ssh and the mysql client available on the
+  db_master.
+
 * The Xen Master
   
   Beginnings of a series of tasks for autmating the creation and
@@ -120,7 +169,7 @@ Currently consists of the following:
 
 (The MIT License)
 
-Copyright (c) 2007 FIX
+Copyright (c) 2007
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -4,6 +4,8 @@ require 'rubygems'
 require 'hoe'
 require 'vlad'
 require 'highline/import'
+require './lib/nytd/dbslayer'
+
 require './lib/vlad/enterprising.rb'
 require './lib/nytd/builder'
 Hoe.new('VladTheEnterprising', VLAD_ENTERPRISING_VERSION) do |p|

data/lib/vlad/dba.rb

@@ -1,10 +1,7 @@
 require 'vlad/dba/mysql'
-namespace :vlad do
-  namespace :dba do
-    task :assert_single_master do
-      if  Rake::RemoteTask.hosts_for(:db_master).size != 1
-        raise "Error! This task demands one and only one host have the :db_master role assigned"
-      end
-    end
-  end
+unless defined? mysql_user
+  set(:mysql_user, 'root')
+  #if defined? mysql_root_password
+  #set(:mysql_password, mysql_root_password)
+  #end
 end

data/lib/vlad/dba/mysql.rb

@@ -4,10 +4,23 @@ require 'highline/import'
 require 'monitor'
 require 'resolv'
 require 'digest/md5'
-
+require 'tempfile'
+unless defined? mysql_user
+  set(:mysql_user, 'root')
+  #if defined? mysql_root_password
+  #set(:mysql_password, mysql_root_password)
+  #end
+end
 
 namespace :vlad do
   namespace :mysql do
+    desc "Checks mysql user open files ulimit on db servers"
+    remote_task :check_ulimit_open_files, :roles => [:db_master, :db_slave] do
+      n = run("#{sudo} su - mysql -c 'ulimit -n' | tail -1 ")
+      puts "#{target_host} mysql user open file ulimit: #{n}"
+    end
+
+
     desc "Bootstraps mysql slave candiates, installs mysql, syncs them to the master, and starts them."
     task :bootstrap_new_slaves => [
                                    :assert_production_target_subset,
@@ -172,8 +185,8 @@ namespace :vlad do
          "db_slave_#{os}.cnf.tmpl",
          "#{name}.cnf.tmpl",
         ].each do |t|
-          tt = "files/mysql/templates/#{t}"          
-          template_file = tt if File.exist?(tt)        
+          tt = "files/mysql/templates/#{t}"
+          template_file = tt if File.exist?(tt)
         end
         template =  File.read(template_file)
         server_id = Resolv.getaddress(hostname) + "-" + hostname
@@ -238,6 +251,25 @@ namespace :vlad do
       mysql_query "flush privileges; slave start;", "", :u => "root"
     end
 
+    desc "dumps, compresses, and copys database(s) from db_master, will copy all databases unless one is specified in a DATABASE= arg"
+    remote_task :dump, :roles => :db_master do
+      database = ENV["DATABASE"] || "all_tables"
+      db_arg = ENV["DATABASE"] || "-A"
+      dumpfile = ENV["DUMPFILE"] || mysql_dumpfile
+      datfile = dumpfile.sub(/\.gz$/,"").sub(/-/, "-#{database}-")
+      dumpfile = datfile + ".gz"
+      run "mkdir -p `dirname #{datfile}`; exit 0"
+      run "#{mysqldump} --single-transaction -uroot -p'#{mysql_root_password}' #{db_arg} > #{datfile}"
+      `mkdir -p \`dirname #{datfile}\``
+      run "nice gzip #{datfile}"
+      cmd="ssh #{target_host} cat #{dumpfile}  > #{dumpfile}"
+      `#{cmd}`
+      run "if [ -e #{dumpfile} ]; then rm #{dumpfile}; else exit 1; fi"
+      puts "Done: #{dumpfile}"
+    end
+    task :dump => [:ask_root_password]
+
+    desc "Creates a master-data dump from the target environment db_master, copies to dumps/"
     remote_task :dump_master, :roles => :db_master do
       unless ENV["DUMPFILE"]
         datfile = mysql_dumpfile.sub(/\.gz$/,"")
@@ -246,20 +278,23 @@ namespace :vlad do
         `mkdir -p \`dirname #{mysql_dumpfile}\``
         run "nice gzip #{datfile}"
         cmd="ssh #{target_host} cat #{mysql_dumpfile}  > #{mysql_dumpfile}"
-        puts cmd
         `#{cmd}`
         run "if [ -e #{mysql_dumpfile} ]; then rm #{mysql_dumpfile}; else exit 1; fi"
       end
     end
+    task :dump_master => [:ask_root_password]
 
     remote_task :slave_sync, :roles => :new_slave  do
-      mysql_dumpfile = ENV["DUMPFILE"] if ENV["DUMPFILE"]
+      if ENV["DUMPFILE"]
+        set( :mysql_dumpfile, ENV["DUMPFILE"] )
+      end
       run "#{mysql_path_bin}/mysql -vvf -u root -e \"slave stop\;\";  exit 0"
       run "#{mysql_path_bin}/mysql -vvf -u root -e \"change master to master_host='#{mysql_master_host}', master_user='#{shortname}_repl', master_password='#{mysql_replication_password}';\""
       run "mkdir -p `dirname #{mysql_dumpfile}`"
       #puts "#{target_host}"
       scp mysql_dumpfile, mysql_dumpfile
       run "cat #{mysql_dumpfile} | gunzip - | #{mysql_path_bin}/mysql -uroot"
+
       run "rm  #{mysql_dumpfile}"
     end
 
@@ -273,7 +308,6 @@ namespace :vlad do
 
     def assert_agent_running
       cmd = ssh #{target_host} "ps -ef | grep mysql | grep agent | grep -v grep"
-      puts cmd
       output=`#{cmd}`
       raise "DOWN" unless output.match(/mysql-service-agent/)
       puts "#{target_host}: agent OK"
@@ -289,6 +323,7 @@ namespace :vlad do
     end
     task :check_slaves  => [:ask_root_password]
 
+
     desc "Checks mysql agent status on all slaves"
     remote_task :check_restart_agents, :roles => [:db_slave, :db_master] do
       cmd1 = "#{mysql_agent_init} start"
@@ -344,9 +379,130 @@ namespace :vlad do
       `rm /tmp/my.cnf.#{target_host}; exit 0`
       run "rm -f .my.cnf; exit 0"
     end
+
+    desc "Executes query specified in QUERY='' argument on database specified in the DATABASE='' argument on the :db_master for the target environment and prints the results"
+    remote_task :query, :roles => :db_master do
+      raise "Please specify what QUERY to run against what database, i.e. QUERY='select foo from bar' DATABASE='Movies' TARGET='staging'" unless ENV['QUERY']
+      puts mysql_query(ENV['QUERY'],ENV['DATABASE'], :u => 'root', :p => mysql_root_password )
+    end
+
+    desc "Executes query specified in QUERY='' argument on database specified in the DATABASE='' argument on all the slaves for the target environment and prints the results"
+    remote_task :query_slaves, :roles => :db_slave do
+      raise "Please specify what QUERY to run against what database, i.e. QUERY='select foo from bar' DATABASE='Movies' TARGET='staging'" unless ENV['QUERY']
+      puts "#{target_host}: " + mysql_query(ENV['QUERY'],ENV['DATABASE'], :u => 'root', :p => mysql_root_password ).gsub("\n", "*****")
+    end
+
+    remote_task :schema_info, :roles => :db_master do
+      databases.select{ |db| db == "mysql" ? false : true }.each do |database|
+        begin
+        output = mysql_query("select version from schema_info limit 1",
+                             database,
+                             :u => 'root',
+                             :p => mysql_root_password)
+
+          current_version = output.split(/\n/)[1].to_i
+        rescue
+          current_version = "unknown"
+        end
+        puts "#{target_host} schema info"
+        puts "#{database} version #{current_version}"
+      end
+    end
+    task :schema_info => [:ask_root_password]
+
+    desc "Runs db migrations for current TARGET can be controlled with better granuarity with DATABASE and VERSION arguments"
+    remote_task :migrate, :roles => :db_master do
+      assert_database_if_version_given()
+      migration_dir ||= "db/migrations"
+      current_version = {}
+      targets = ENV['DATABASE']
+      targets ||= databases.select{ |db| db == "mysql" ? false : true }
+      targets.each do |database|
+        database_dir = File.join(migration_dir, database)
+        begin
+        output = mysql_query("select version from schema_info limit 1",
+                             database,
+                             :u => 'root',
+                             :p => mysql_root_password)
+          current_version = output.split(/\n/)[1].to_i
+        rescue
+          current_version = 0
+        end
+        migrations = { }
+        dirs =  Dir.new(database_dir).entries.select{|d| d.match(/^\d+/)}
+        dirs.each { |dir| migrations[dir.sub(/[^\d]*$/, "")] = dir }
+        target_version = ENV['VERSION'].to_i if ENV['VERSION']
+        target_version ||= migrations.keys.sort{|a,b| a.to_i <=> b.to_i }.last.to_i rescue 0
+        case
+        when current_version < target_version
+          have_migrations = lambda {  current_version < target_version }
+          iterator =  lambda { |v| v + 1}
+          action = "upgrade"
+          next_file = lambda {|v| File.join(database_dir, migrations[(v + 1 ).to_s], "#{action}.sql") rescue nil}
+          puts "Upgrading schema #{database} from version #{current_version} to version #{target_version} on #{target_host}"
+        when current_version > target_version
+          have_migrations = lambda { current_version > target_version }
+          iterator =  lambda { |v| v - 1}
+          action = "rollback"
+          next_file = lambda { |v| File.join(database_dir, migrations[v.to_s], "#{action}.sql") }
+          puts "Downgrading #{database} from version #{current_version} to version #{target_version} on #{target_host}"
+        else
+          have_migrations = lambda{ false }
+          puts "#{database} already at version #{target_version} on #{target_host}"
+        end
+
+        while have_migrations.call() do
+          next_version = iterator.call(current_version)
+          migration_file = next_file.call(current_version)
+          if migration_file && File.exist?(migration_file)
+          sql = "start transaction;\n"
+            sql << "CREATE TABLE IF NOT EXISTS schema_info ( version INT default 0 );\n"
+            sql << File.read(migration_file)
+            sql << "\n"
+            sql << "update schema_info set version = #{next_version};\n"
+            sql << "commit;"
+            src = "/tmp/#{database}-version-#{File.basename(File.dirname(migration_file))}-#{File.basename(migration_file)}"
+            File.open(src, "w"){ |f| f.write(sql) }
+            scp src, src
+            if Rake.application.options.trace
+              puts "============================================\n"
+              puts sql
+              puts "============================================\n"
+            end
+            mysql_query("source #{src};", database, :u => 'root', :p => mysql_root_password)
+            puts "Executed: #{migration_file} on #{target_host}"
+          end
+          current_version = next_version
+        end
+        puts "Migration complete."
+      end
+    end
+    task :migrate => [:ask_root_password, :assert_single_master]
+
+    desc "restores a dump to current db_master from file set in mysql_dumpfile or in a DUMPFILE argument"
+    remote_task :restore, :roles => :db_master do
+      dump = ENV["DUMPFILE"] || dump
+      raise "Please specify a valid dumpfile to restore via :mysql_dumpfile in the config or use the DUMPFILE argument!" unless File.exist?(dump)
+      scp dump, dump
+      p = ""
+      unless mysql_root_password.nil? or mysql_root_password == ""
+        p = "-p'#{mysql_root_password}' "
+      end
+      if ENV['DATABASE']
+        mysql_query "DROP DATABASE IF EXISTS #{ENV['DATABASE']}", "", :u => 'root', :p => mysql_root_password
+        mysql_query "CREATE DATABASE IF NOT EXISTS #{ENV['DATABASE']}", "", :u => 'root', :p => mysql_root_password
+      end
+      run "cat #{dump} | gunzip - | #{mysql_path_bin}/mysql -uroot #{p} #{ENV['DATABASE']}"
+    end
+    task :restore  => [:ask_root_password, :assert_single_master]
   end
 end
 
+def assert_database_if_version_given
+  if ENV['VERSION']
+    raise "You must specify a database if you specify a target version!  i.e. DATABASE=Movies VERSION=93" unless ENV['DATABASE']
+  end
+end
 dialect( :name => :default_mysql,
          :assert => lambda{  true },
          :set => {

data/lib/vlad/enterprising.rb

@@ -4,7 +4,7 @@ require 'vlad/environmentalist'
 require 'vlad/xen_master'
 require 'vlad/dba'
 require 'vlad/webmaster/apache'
-VLAD_ENTERPRISING_VERSION = '0.1.7' unless defined? VLAD_ENTERPRISING_VERSION
+VLAD_ENTERPRISING_VERSION = '0.1.8' unless defined? VLAD_ENTERPRISING_VERSION
 
 
 # rsync the given files to <tt>target_host</tt>.

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.0
+rubygems_version: 0.9.4
 specification_version: 1
 name: VladTheEnterprising
 version: !ruby/object:Gem::Version 
-  version: 0.1.7
-date: 2007-10-17 00:00:00 -04:00
+  version: 0.1.8
+date: 2007-11-02 00:00:00 -04:00
 summary: Adds 'enterprisey' features to Vlad The Deployer
 require_paths: 
 - lib