sml-rubber 0.9.13 → 1.5.5

data/lib/rubber/recipes/rubber/static_ips.rb

@@ -80,6 +80,89 @@ namespace :rubber do
     destroy_static_ip(ip)
   end
 
+  desc 'Move a static IP address from DONOR machine to RECEIVER machine.'
+  task :move_static_ip do
+    donor_alias = get_env 'DONOR', 'Instance alias to get the IP from (e.g., web01)', true
+    receiver_alias = get_env 'RECEIVER', 'Instance alias to assign the IP to (e.g., web02)', true
+
+    # Sanity checks
+    donor = rubber_instances[donor_alias]
+    fatal "Instance does not exist: #{donor_alias}" unless donor
+
+    static_ip = donor.static_ip
+    fatal 'No static IP address to move exists' unless static_ip && static_ip != ''
+
+    receiver = rubber_instances[receiver_alias]
+    fatal "Instance does not exist: #{receiver_alias}" unless receiver
+
+    # Temporary removal of the instances.
+    old_donor = rubber_instances.remove(donor_alias)
+    old_receiver = rubber_instances.remove(receiver_alias)
+
+    rubber_instances.save
+
+    # Getting rid of alias->IP mappings and SSH's known_hosts records.
+    load_roles
+    setup_aliases
+    cleanup_known_hosts(old_donor)
+    cleanup_known_hosts(old_receiver)
+
+    # Detachment of EIPA.
+    success = cloud.detach_static_ip(static_ip)
+    fatal "Failed to detach static IP address #{static_ip}" unless success
+    rubber_instances.artifacts['static_ips'].delete(old_donor.name)
+
+    rubber_instances.save
+
+    # Attachment of EIPA.
+    success = cloud.attach_static_ip(static_ip, old_receiver.instance_id)
+    fatal "Failed to associate static IP address #{static_ip}" unless success
+
+    print "Waiting for #{receiver_alias} to get the address"
+    while true do
+      print '.'
+      sleep 3
+      instance = cloud.describe_instances(old_receiver.instance_id).first
+      break if instance[:external_ip] == static_ip
+    end
+
+    # Partial cleanup of static IP records.
+    rubber_instances.artifacts['static_ips'][old_receiver.name] = static_ip
+
+    rubber_instances.save
+
+    # First half of the sync.
+    new_receiver = Rubber::Configuration::InstanceItem.new(old_receiver.name,
+      old_receiver.domain, old_receiver.roles, old_receiver.instance_id,
+      old_receiver.security_groups)
+    new_receiver.static_ip = static_ip
+    rubber_instances.add(new_receiver)
+
+    rubber_instances.save
+
+    refresh_instance(receiver_alias)
+
+    print "Waiting for #{donor_alias} to get a new address"
+    while true do
+      print '.'
+      sleep 3
+      instance = cloud.describe_instances(old_donor.instance_id).first
+      break if instance[:external_ip] && instance[:external_ip] != ''
+    end
+
+    # Second half of the sync.
+    new_donor = Rubber::Configuration::InstanceItem.new(old_donor.name,
+      old_donor.domain, old_donor.roles, old_donor.instance_id,
+      old_donor.security_groups)
+    rubber_instances.add(new_donor)
+
+    rubber_instances.save
+
+    refresh_instance(donor_alias)
+
+    logger.info "Run 'cap rubber:describe_static_ips' to check the allocated ones"
+  end
+
   def allocate_static_ip()
     ip = cloud.create_static_ip()
     fatal "Failed to allocate static ip" if ip.nil?

data/lib/rubber/recipes/rubber/utils.rb

@@ -16,23 +16,26 @@ namespace :rubber do
     default_roles = rubber_env.staging_roles || "*"
     roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
 
-    # some bootstraps update code (bootstrap_db) but if you don't have that role, need to do it here
-    # Since release directory variable gets reused by cap, we have to just do the symlink here - doing
-    # a update again will fail
-    set :rubber_code_was_updated, false
-    after "deploy:update_code" do
-      set :rubber_code_was_updated, true
-    end
-
     if rubber_instances[instance_alias]
       logger.info "Instance already exists, skipping to bootstrap"
     else
       rubber.create
     end
+
+    # stop everything before so monit doesn't start stuff during bootstrapping
+    # if its already installed due to a bundled instance
+    deploy.stop rescue nil
+
     rubber.bootstrap
-    # stop everything in case we have a bundled instance with monit, etc starting at boot
+    
+    # stop everything after in case package upgrades during bootstrap start up
+    # services - we should be able to safely do a deploy:start below
     deploy.stop rescue nil
-    if ! rubber_code_was_updated
+
+    # some bootstraps update code (bootstrap_db) but if you don't have that role, need to do it here
+    # Since release directory variable gets reused by cap, we have to just do the symlink here - doing
+    # a update again will fail
+    if ! fetch(:rubber_code_was_updated, false)
       deploy.update_code
     end
     deploy.symlink
@@ -105,8 +108,8 @@ namespace :rubber do
     slices = []
     servers.each do |rolename, svrs|
       next if svrs.size == 0
-      # figure out size of each slice by deviding server count by # of groups
-      slice_size = svrs.size / (options.delete(:groups) || 2)
+      # figure out size of each slice by dividing server count by # of groups
+      slice_size = (Float(svrs.size) / (options.delete(:groups) || 2)).round
       slice_size = 1 if slice_size == 0
       slice_idx = 0
       svrs.each_slice(slice_size) do |srv_slice|
@@ -147,22 +150,30 @@ namespace :rubber do
     return local_alias
   end
 
-  def prepare_script(name, contents)
+  def prepare_script(name, contents, stop_on_error_cmd=rubber_env.stop_on_error_cmd)
     script = "/tmp/#{name}"
     # this lets us abort a script if a command in the middle of it errors out
-    contents = "#{rubber_env.stop_on_error_cmd}\n#{contents}" if rubber_env.stop_on_error_cmd
+    contents = "#{stop_on_error_cmd}\n#{contents}" if stop_on_error_cmd
     put(contents, script)
     return script
   end
 
-  def run_script(name, contents)
+  def run_script(name, contents, opts = {})
+    args = opts.delete(:script_args)
     script = prepare_script(name, contents)
-    run "sh #{script}"
+    run "bash #{script} #{args}", opts
   end
 
-  def sudo_script(name, contents)
+  def sudo_script(name, contents, opts = {})
+    args = opts.delete(:script_args)
     script = prepare_script(name, contents)
-    sudo "sh #{script}"
+    run "#{sudo} bash -l #{script} #{args}", opts
+  end
+
+  def top.rsudo(command, opts = {}, &block)
+    user = opts.delete(:as)
+    args = "-H -u #{user}" if user
+    run "#{sudo opts} #{args} bash -l -c '#{command}'", opts, &block
   end
 
   def get_env(name, desc, required=false, default=nil)
@@ -200,4 +211,16 @@ namespace :rubber do
     return opts
   end
 
+  # some bootstraps update code (bootstrap_db), so keep track so we don't do it multiple times  
+  after "deploy:update_code" do
+    set :rubber_code_was_updated, true
+  end
+  
+  def update_code_for_bootstrap
+    unless (fetch(:rubber_code_was_updated, false))
+      deploy.setup
+      deploy.update_code
+    end
+  end
+  
 end

data/lib/rubber/recipes/rubber/volumes.rb

@@ -104,7 +104,7 @@ namespace :rubber do
       if vol_spec['mount'] && vol_spec['filesystem']
         # then format/mount/etc if we don't have an entry in hosts file
         task :_setup_volume, :hosts => ic.external_ip do
-          rubber.run_script 'setup_volume', <<-ENDSCRIPT
+          rubber.sudo_script 'setup_volume', <<-ENDSCRIPT
             if ! grep -q '#{vol_spec['mount']}' /etc/fstab; then
               if mount | grep -q '#{vol_spec['mount']}'; then
                 umount '#{vol_spec['mount']}'
@@ -135,7 +135,7 @@ namespace :rubber do
     if ! ic.partitions.include?(part_id)
       # then format/mount/etc if we don't have an entry in hosts file
       task :_setup_partition, :hosts => ic.external_ip do
-        rubber.run_script 'setup_partition', <<-ENDSCRIPT
+        rubber.sudo_script 'setup_partition', <<-ENDSCRIPT
           if ! fdisk -l 2>&1 | grep -q '#{partition_spec['partition_device']}'; then
             if grep -q '#{partition_spec['disk_device']}\\b' /etc/fstab; then
               umount #{partition_spec['disk_device']}
@@ -175,7 +175,7 @@ namespace :rubber do
       end
       # then format/mount/etc if we don't have an entry in hosts file
       task :_zero_partitions, :hosts => ic.external_ip do
-        rubber.run_script 'zero_partitions', <<-ENDSCRIPT
+        rubber.sudo_script 'zero_partitions', <<-ENDSCRIPT
           # zero out parition for performance (see amazon DevGuide)
           echo "Zeroing out raid partitions to improve performance, this way take a while"
           #{zero_script}
@@ -200,7 +200,7 @@ namespace :rubber do
     end
 
     task :_setup_raid_volume, :hosts => ic.external_ip do
-      rubber.run_script 'setup_raid_volume', <<-ENDSCRIPT
+      rubber.sudo_script 'setup_raid_volume', <<-ENDSCRIPT
         if ! grep -q '#{raid_spec['device']}' /etc/fstab; then
           if mount | grep -q '#{raid_spec['mount']}'; then
             umount '#{raid_spec['mount']}'

data/lib/rubber/tasks/rubber.rb

@@ -98,11 +98,70 @@ namespace :rubber do
     end
   end
 
+
+  desc <<-DESC
+    Backup database to given backup directory
+    The following arguments affect behavior:
+    BACKUP_DIR (required):  Directory where backups will be stored
+    BACKUP_NAME (required): What to name the backup
+    BACKUP_CMD (required):  Command used to backup
+    BACKUP_AGE (3):         Delete rotated logs older than this many days in the past
+  DESC
+  task :backup do
+    dir = get_env('BACKUP_DIR', true)
+    name = get_env('BACKUP_NAME', true)
+    cmd = get_env('BACKUP_CMD', true)
+    age = (get_env('BACKUP_AGE') || 3).to_i
+
+    time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
+    FileUtils.mkdir_p(dir)
+
+    backup_cmd = cmd.gsub(/%([^%]+)%/, '#{\1}')
+    backup_cmd = eval('%Q{' + backup_cmd + '}')
+
+    puts "Backing up with command:"
+    sh backup_cmd
+    puts "Backup created"
+
+    s3_prefix = "#{name}/"
+    backup_bucket = cloud_provider.backup_bucket
+    if backup_bucket
+      init_s3
+      unless AWS::S3::Bucket.list.find { |b| b.name == backup_bucket }
+        AWS::S3::Bucket.create(backup_bucket)
+      end
+      newest = Dir.entries(dir).grep(/^[^.]/).sort_by {|f| File.mtime(File.join(dir,f))}.last
+      dest = "#{s3_prefix}#{newest}"
+      puts "Saving backup to S3: #{backup_bucket}:#{dest}"
+      AWS::S3::S3Object.store(dest, open(File.join(dir, newest)), backup_bucket)
+    end
+
+    tdate = Date.today - age
+    threshold = Time.local(tdate.year, tdate.month, tdate.day)
+    puts "Cleaning backups older than #{age} days"
+    Dir["#{dir}/*"].each do |file|
+      if File.mtime(file) < threshold
+        puts "Deleting #{file}"
+        FileUtils.rm_f(file)
+      end
+    end
+
+    if backup_bucket
+      puts "Cleaning S3 backups older than #{age} days from: #{backup_bucket}:#{s3_prefix}"
+      AWS::S3::Bucket.objects(backup_bucket, :prefix => s3_prefix).clone.each do |obj|
+        if Time.parse(obj.about["last-modified"]) < threshold
+          puts "Deleting #{obj.key}"
+          obj.delete
+        end
+      end
+    end
+  end
+
   desc <<-DESC
     Backup database to given backup directory
     The following arguments affect behavior:
     BACKUP_DIR (required): Directory where db backups will be stored
-    BACKUP_AGE (7):        Delete rotated logs older than this many days in the past
+    BACKUP_AGE (3):        Delete rotated logs older than this many days in the past
     DBUSER (required)      User to connect to the db as
     DBPASS (optional):     Pass to connect to the db with
     DBHOST (required):     Host where the db is
@@ -120,15 +179,15 @@ namespace :rubber do
     pass = nil if pass.strip.size == 0
     host = get_env('DBHOST', true)
     name = get_env('DBNAME', true)
-    
+
     raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless rubber_env.db_backup_cmd
     db_backup_cmd = rubber_env.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
     db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
-    
+
     puts "Backing up database with command:"
     sh db_backup_cmd
     puts "Created backup: #{backup_file}"
-    
+
     s3_prefix = "db/"
     backup_bucket = cloud_provider.backup_bucket
     if backup_bucket
@@ -150,7 +209,7 @@ namespace :rubber do
         FileUtils.rm_f(file)
       end
     end
-    
+
     if backup_bucket
       puts "Cleaning S3 backups older than #{age} days from: #{backup_bucket}:#{s3_prefix}"
       AWS::S3::Bucket.objects(backup_bucket, :prefix => s3_prefix).clone.each do |obj|
@@ -211,7 +270,6 @@ namespace :rubber do
 
   end
 
-
   def get_env(name, required=false)
     value = ENV[name]
     raise("#{name} is required, pass using environment") if required && ! value

data/lib/rubber/util.rb

@@ -30,8 +30,43 @@ module Rubber
     end
 
     def self.is_rails?
-      File.exist?(File.join(RUBBER_ROOT, 'config', 'environment.rb'))
+      File.exist?(File.join(RUBBER_ROOT, 'config', 'boot.rb'))
     end
 
+    def self.is_rails2?
+      defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::MAJOR == 2
+    end
+
+    def self.is_rails3?
+      defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::MAJOR == 3
+    end
+
+    def self.is_bundler?
+      File.exist?(File.join(RUBBER_ROOT, 'Gemfile'))
+    end
+
+    def self.rubber_as_plugin?
+      File.exist?(File.join(RUBBER_ROOT, 'vendor/plugins/rubber'))
+    end
+
+    def self.prompt(name, desc, required=false, default=nil)
+      value = ENV.delete(name)
+      msg = "#{desc}"
+      msg << " [#{default}]" if default
+      msg << ": "
+      unless value
+        print msg
+        value = gets
+      end
+      value = value.size == 0 ? default : value
+      self.fatal "#{name} is required, pass using environment or enter at prompt" if required && ! value
+      return value
+    end
+
+    def self.fatal(msg, code=1)
+      puts msg
+      exit code
+    end
+    
   end
 end

data/rails/init.rb

@@ -0,0 +1,6 @@
+# this file just makes it easy to refer to rubber config
+# variables in your apps's config'
+#
+
+require "rubber"
+Rubber::initialize(Rails.root, Rails.env)

data/test/environment_test.rb

@@ -1,6 +1,9 @@
+require 'rubygems'
+gem 'test-unit'
+
 require 'test/unit'
 require 'tempfile'
-require 'test_helper'
+require File.expand_path(File.join(__FILE__, '..', 'test_helper'))
 
 class EnvironmentTest < Test::Unit::TestCase
   include Rubber::Configuration
@@ -39,6 +42,14 @@ class EnvironmentTest < Test::Unit::TestCase
     assert_equal 'host1val2', e.var2, 'env not retrieving right val for method missing'
   end
 
+  def test_host_override
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
+    e = env.bind('norole', 'host2')
+    assert_equal 'host2val3', e['var3'], 'env not retrieving right val'
+    assert_equal %w[host2val4a host2val4b], e['var4'], 'env not retrieving right val'
+    assert_equal [{'var51a' => 'val51a', 'var52a' => 'val52a'}, {'var53' => 'val53'}, {'var54' => 'val54'}], e['var5'], 'env not retrieving right val'
+  end
+
   def test_combine
     env = Rubber::Configuration::Environment
     assert_equal "new", env.combine("old", "new"), "Last should win for scalar combine"
@@ -115,4 +126,19 @@ class EnvironmentTest < Test::Unit::TestCase
     assert_equal 'val5', e.var2.var9, 'env not retrieving right val'
   end
 
+  def test_instances_in_expansion
+    instance = InstanceItem.new('host1', 'domain.com', [RoleItem.new('role1')], '')
+    instance.external_ip = "1.2.3.4"
+    instances = Instance.new(Tempfile.new('testforinstanceexpansion').path)
+    instances.add(instance)
+
+    File.expects(:exist?).returns(true)
+    YAML.expects(:load_file).returns({'var1' =>'"#{rubber_instances.for_role("role1").first.external_ip}"'})
+    Rubber::Configuration.expects(:rubber_instances).returns(instances)
+    env = Rubber::Configuration::Environment.new(nil)
+    e = env.bind()
+
+    assert_equal "\"1.2.3.4\"", e['var1']    
+  end
+
 end

data/test/generator_test.rb

@@ -1,6 +1,9 @@
+require 'rubygems'
+gem 'test-unit'
+
 require 'test/unit'
 require 'tempfile'
-require 'test_helper'
+require File.expand_path(File.join(__FILE__, '..', 'test_helper'))
 
 class GeneratorTest < Test::Unit::TestCase
   include Rubber::Configuration
@@ -90,6 +93,14 @@ class GeneratorTest < Test::Unit::TestCase
     assert ! File.exists?(post_file), "post should not have been executed as dest file unchanged"
     assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
 
+    FileUtils.rm_f(post_file)
+    gen = Generator.new(nil, nil, nil)
+    gen.force = true
+    gen.transform(src)
+    assert File.exists?(out_file.path), "transform did not generate an output file"
+    assert File.exists?(post_file), "forced transform did not execute post"
+    assert_equal "hello", File.read(out_file.path).strip, "transformed contents are incorrect"
+
     FileUtils.rm_f(post_file)
     gen = Generator.new(nil, nil, nil)
     gen.no_post = true
@@ -175,7 +186,7 @@ class GeneratorTest < Test::Unit::TestCase
       hello
     SRC
 
-    assert_raises RuntimeError do
+    assert_raises do
       Generator.new(nil, nil, nil).transform(src)
     end
     
@@ -221,7 +232,7 @@ class GeneratorTest < Test::Unit::TestCase
     FileUtils.rm_rf(out_dir)
     assert ! File.exists?(out_dir)
 
-    g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", ['role1'], ['host1'], :out_dir => out_dir)
+    g = Generator.new("#{File.dirname(__FILE__)}/fixtures/basic", ['role1'], 'host1', :out_dir => out_dir)
     g.run()
     assert File.directory?(out_dir), "scoped transform did not create dir"
     assert_equal ['bar.conf', 'foo.conf'], list_dir(out_dir), "scoped transform did not create correct files"
@@ -232,12 +243,12 @@ class GeneratorTest < Test::Unit::TestCase
     FileUtils.rm_rf(out_dir)
   end
 
-  def test_ordering
+  def test_ordering_of_additive
     out_dir = "#{Dir::tmpdir}/test_rubber_ordering"
     FileUtils.rm_rf(out_dir)
     assert ! File.exists?(out_dir)
     
-    g = Generator.new("#{File.dirname(__FILE__)}/fixtures/generator_order", ['role2', 'role1'], ['host1'], :out_dir => out_dir)
+    g = Generator.new("#{File.dirname(__FILE__)}/fixtures/generator_order", ['role2', 'role1'], 'host1', :out_dir => out_dir)
     g.run()
     assert File.directory?(out_dir), "transform did not create dir"
     assert_equal ['out.conf'], list_dir(out_dir), "transform did not create correct file"
@@ -320,4 +331,40 @@ def test_file_pattern
     Generator.new(nil, nil, nil).transform(src)
     assert ! File.exists?(out_file.path), "transform didn't skip generation of an output file"
   end
+
+  def test_backup
+    out_file = Tempfile.new('testbak')
+    assert ! File.exists?("#{out_file.path}.bak")
+    File.open(out_file.path, 'w') {|f| f.write("howdy")}
+    
+    src = <<-SRC
+      <%
+        @path = '#{out_file.path}'
+      %>
+     hello <%= Time.now.to_f %>
+    SRC
+
+    Generator.new(nil, nil, nil).transform(src)
+
+    assert File.exists?("#{out_file.path}.bak"), "transform didn't generate backup"
+    assert_match /howdy/, File.read("#{out_file.path}.bak"), "transform backup has wrong contents"
+  end
+
+  def test_no_backup
+    out_file = Tempfile.new('testnobak')
+    assert ! File.exists?("#{out_file.path}.bak")
+    File.open(out_file.path, 'w') {|f| f.write("howdy")}
+    src = <<-SRC
+      <%
+        @path = '#{out_file.path}'
+        @backup = false
+      %>
+     hello <%= Time.now.to_f %>
+    SRC
+
+    Generator.new(nil, nil, nil).transform(src)
+
+    assert ! File.exists?("#{out_file.path}.bak"), "transform shouldn't generate backup"
+  end
+
 end