nirvdrum-rubber 1.1.7

data/lib/rubber/tasks/rubber.rb

@@ -0,0 +1,279 @@
+require 'fileutils'
+require 'date'
+require 'time'
+require 'aws/s3'
+require 'rubber'
+
+namespace :rubber do
+
+  def rubber_env()
+    Rubber::Configuration.rubber_env
+  end
+
+  def rubber_instances()
+    Rubber::Configuration.rubber_instances
+  end
+
+  def cloud_provider
+    rubber_env.cloud_providers[rubber_env.cloud_provider]
+  end
+
+  def init_s3()
+    AWS::S3::Base.establish_connection!(:access_key_id => cloud_provider.access_key, :secret_access_key => cloud_provider.secret_access_key)
+  end
+
+  desc "Generate system config files by transforming the files in the config tree"
+  task :config do
+    cfg = Rubber::Configuration.get_configuration(RUBBER_ENV)
+    instance_alias = cfg.environment.current_host
+    instance = cfg.instance[instance_alias]
+    if instance
+      roles = instance.role_names
+      env = cfg.environment.bind(roles, instance_alias)
+      gen = Rubber::Configuration::Generator.new("#{RUBBER_ROOT}/config/rubber", roles, instance_alias)
+    elsif RUBBER_ENV == 'development'
+      roles = cfg.environment.known_roles
+      role_items = roles.collect do |r|
+        Rubber::Configuration::RoleItem.new(r, r == "db" ? {'primary' => true} : {})
+      end
+      env = cfg.environment.bind(roles, instance_alias)
+      domain = env.domain
+      instance = Rubber::Configuration::InstanceItem.new(instance_alias, domain, role_items, 'dummyid', ['dummygroup'])
+      instance.external_host = instance.full_name
+      instance.external_ip = "127.0.0.1"
+      instance.internal_host = instance.full_name
+      instance.internal_ip = "127.0.0.1"
+      cfg.instance.add(instance)
+      gen = Rubber::Configuration::Generator.new("#{RUBBER_ROOT}/config/rubber", roles, instance_alias)
+      gen.fake_root ="#{RUBBER_ROOT}/tmp/rubber"
+    else
+      puts "Instance not found for host: #{instance_alias}"
+      exit 1
+    end
+
+    if ENV['NO_POST']
+      gen.no_post = true
+    end
+    if ENV['FILE']
+      gen.file_pattern = ENV['FILE']
+    end
+    if ENV['FORCE']
+      gen.force = true
+    end
+    gen.stop_on_error_cmd = env.stop_on_error_cmd
+    gen.run
+
+  end
+
+  desc <<-DESC
+    Rotate rails app server logfiles.  Should be run right after midnight.
+    The following arguments affect behavior:
+    LOG_DIR (required): Directory where log files are located
+    LOG_FILE (*.log):   File pattern to match to find logs to rotate
+    LOG_AGE (7):        Delete rotated logs older than this many days in the past
+  DESC
+  task :rotate_logs do
+    log_src_dir = get_env('LOG_DIR', true)
+    log_file_glob = get_env('LOG_FILES') || "*.log"
+    log_file_age = (get_env('LOG_AGE') || 7).to_i
+
+    rotated_date = (Date.today - 1).strftime('%Y%m%d')
+    puts "Rotating logfiles located at: #{log_src_dir}/#{log_file_glob}"
+    Dir["#{log_src_dir}/#{log_file_glob}"].each do |logfile|
+      rotated_file = "#{logfile}.#{rotated_date}"
+      if File.exist?(rotated_file)
+        rotated_file += "_#{Time.now.to_i}"
+      end
+      FileUtils.cp(logfile, rotated_file)
+      File.truncate logfile, 0
+    end
+
+    tdate = Date.today - log_file_age
+    threshold = Time.local(tdate.year, tdate.month, tdate.day)
+    puts "Cleaning rotated log files older than #{log_file_age} days"
+    Dir["#{log_src_dir}/#{log_file_glob}.[0-9]*"].each do |logfile|
+      if File.mtime(logfile) < threshold
+        FileUtils.rm_f(logfile)
+      end
+    end
+  end
+
+
+  desc <<-DESC
+    Backup database to given backup directory
+    The following arguments affect behavior:
+    BACKUP_DIR (required):  Directory where backups will be stored
+    BACKUP_NAME (required): What to name the backup
+    BACKUP_CMD (required):  Command used to backup
+    BACKUP_AGE (3):         Delete rotated logs older than this many days in the past
+  DESC
+  task :backup do
+    dir = get_env('BACKUP_DIR', true)
+    name = get_env('BACKUP_NAME', true)
+    cmd = get_env('BACKUP_CMD', true)
+    age = (get_env('BACKUP_AGE') || 3).to_i
+
+    time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
+    FileUtils.mkdir_p(dir)
+
+    backup_cmd = cmd.gsub(/%([^%]+)%/, '#{\1}')
+    backup_cmd = eval('%Q{' + backup_cmd + '}')
+
+    puts "Backing up with command:"
+    sh backup_cmd
+    puts "Backup created"
+
+    s3_prefix = "#{name}/"
+    backup_bucket = cloud_provider.backup_bucket
+    if backup_bucket
+      init_s3
+      unless AWS::S3::Bucket.list.find { |b| b.name == backup_bucket }
+        AWS::S3::Bucket.create(backup_bucket)
+      end
+      newest = Dir.entries(dir).grep(/^[^.]/).sort_by {|f| File.mtime(File.join(dir,f))}.last
+      dest = "#{s3_prefix}#{newest}"
+      puts "Saving backup to S3: #{backup_bucket}:#{dest}"
+      AWS::S3::S3Object.store(dest, open(File.join(dir, newest)), backup_bucket)
+    end
+
+    tdate = Date.today - age
+    threshold = Time.local(tdate.year, tdate.month, tdate.day)
+    puts "Cleaning backups older than #{age} days"
+    Dir["#{dir}/*"].each do |file|
+      if File.mtime(file) < threshold
+        puts "Deleting #{file}"
+        FileUtils.rm_f(file)
+      end
+    end
+
+    if backup_bucket
+      puts "Cleaning S3 backups older than #{age} days from: #{backup_bucket}:#{s3_prefix}"
+      AWS::S3::Bucket.objects(backup_bucket, :prefix => s3_prefix).clone.each do |obj|
+        if Time.parse(obj.about["last-modified"]) < threshold
+          puts "Deleting #{obj.key}"
+          obj.delete
+        end
+      end
+    end
+  end
+
+  desc <<-DESC
+    Backup database to given backup directory
+    The following arguments affect behavior:
+    BACKUP_DIR (required): Directory where db backups will be stored
+    BACKUP_AGE (3):        Delete rotated logs older than this many days in the past
+    DBUSER (required)      User to connect to the db as
+    DBPASS (optional):     Pass to connect to the db with
+    DBHOST (required):     Host where the db is
+    DBNAME (required):     Database name to backup
+  DESC
+  task :backup_db do
+    dir = get_env('BACKUP_DIR', true)
+    age = (get_env('BACKUP_AGE') || 3).to_i
+    time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
+    backup_file = "#{dir}/#{RUBBER_ENV}_dump_#{time_stamp}.sql.gz"
+    FileUtils.mkdir_p(File.dirname(backup_file))
+
+    user = get_env('DBUSER', true)
+    pass = get_env('DBPASS')
+    pass = nil if pass.strip.size == 0
+    host = get_env('DBHOST', true)
+    name = get_env('DBNAME', true)
+
+    raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless rubber_env.db_backup_cmd
+    db_backup_cmd = rubber_env.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
+    db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
+
+    puts "Backing up database with command:"
+    sh db_backup_cmd
+    puts "Created backup: #{backup_file}"
+
+    s3_prefix = "db/"
+    backup_bucket = cloud_provider.backup_bucket
+    if backup_bucket
+      init_s3
+      unless AWS::S3::Bucket.list.find { |b| b.name == backup_bucket }
+        AWS::S3::Bucket.create(backup_bucket)
+      end
+      dest = "#{s3_prefix}#{File.basename(backup_file)}"
+      puts "Saving db backup to S3: #{backup_bucket}:#{dest}"
+      AWS::S3::S3Object.store(dest, open(backup_file), backup_bucket)
+    end
+
+    tdate = Date.today - age
+    threshold = Time.local(tdate.year, tdate.month, tdate.day)
+    puts "Cleaning backups older than #{age} days"
+    Dir["#{dir}/*"].each do |file|
+      if File.mtime(file) < threshold
+        puts "Deleting #{file}"
+        FileUtils.rm_f(file)
+      end
+    end
+
+    if backup_bucket
+      puts "Cleaning S3 backups older than #{age} days from: #{backup_bucket}:#{s3_prefix}"
+      AWS::S3::Bucket.objects(backup_bucket, :prefix => s3_prefix).clone.each do |obj|
+        if Time.parse(obj.about["last-modified"]) < threshold
+          puts "Deleting #{obj.key}"
+          obj.delete
+        end
+      end
+    end
+  end
+
+  desc <<-DESC
+    Restores a database backup from s3.
+    This tries to find the last backup made or the s3 object identified by the
+    key FILENAME
+    The following arguments affect behavior:
+    FILENAME (optional):   key of S3 object to use
+    DBUSER (required)      User to connect to the db as
+    DBPASS (optional):     Pass to connect to the db with
+    DBHOST (required):     Host where the db is
+    DBNAME (required):     Database name to backup
+  DESC
+  task :restore_db_s3 do
+    file = get_env('FILENAME')
+    user = get_env('DBUSER', true)
+    pass = get_env('DBPASS')
+    pass = nil if pass && pass.strip.size == 0
+    host = get_env('DBHOST', true)
+    name = get_env('DBNAME', true)
+    
+    raise "No db_restore_cmd defined in rubber.yml" unless rubber_env.db_restore_cmd
+    db_restore_cmd = rubber_env.db_restore_cmd.gsub(/%([^%]+)%/, '#{\1}')
+    db_restore_cmd = eval('%Q{' + db_restore_cmd + '}')
+
+    # try to fetch a matching file from s3 (if backup_bucket given)
+    backup_bucket = cloud_provider.backup_bucket
+    raise "No backup_bucket defined in rubber.yml" unless backup_bucket
+    if (init_s3 &&
+        AWS::S3::Bucket.list.find { |b| b.name == backup_bucket })
+      s3objects = AWS::S3::Bucket.find(backup_bucket,
+                 :prefix => 'db/') 
+      if file
+        puts "trying to fetch #{file} from s3"
+        data = s3objects.detect { |o| file == o.key }
+      else
+        puts "trying to fetch last modified s3 backup"
+        data = s3objects.max {|a,b| a.about["last-modified"] <=> b.about["last-modified"] }
+      end
+    end
+    raise "could not access backup file via s3" unless data
+
+    puts "piping restore data to command [#{db_restore_cmd}]"
+    IO.popen(db_restore_cmd, mode='w') do |p|
+      data.value do |segment|
+        p.write segment
+      end
+    end
+
+  end
+
+  def get_env(name, required=false)
+    value = ENV[name]
+    raise("#{name} is required, pass using environment") if required && ! value
+    return value
+  end
+
+end

data/lib/rubber/util.rb

@@ -0,0 +1,37 @@
+
+
+module Rubber
+  module Util
+
+    def self.symbolize_keys(map)
+      map.inject({}) do |options, (key, value)|
+        options[key.to_sym || key] = value
+        options
+      end
+    end
+    
+    def self.stringify(val)
+      case val
+      when String
+        val
+      when Hash
+        val.inject({}) {|h, a| h[stringify(a[0])] = stringify(a[1]); h}
+      when Enumerable
+        val.collect {|v| stringify(v)}
+      else
+        val.to_s
+      end
+      
+    end
+
+    # Opens the file for writing by root
+    def self.sudo_open(path, perms, &block)
+      open("|sudo tee #{path} > /dev/null", perms, &block)
+    end
+
+    def self.is_rails?
+      File.exist?(File.join(RUBBER_ROOT, 'config', 'environment.rb'))
+    end
+
+  end
+end

data/rails/init.rb

@@ -0,0 +1,9 @@
+# this file just makes it easy to refer to rubber config
+# variables in your apps's config'
+#
+
+require "rubber"
+Rubber::initialize(RAILS_ROOT, RAILS_ENV)
+
+::RUBBER_CONFIG = Rubber::Configuration.rubber_env
+::RUBBER_INSTANCES = Rubber::Configuration.rubber_instances

data/test/environment_test.rb

@@ -0,0 +1,133 @@
+require 'test/unit'
+require 'tempfile'
+require 'test_helper'
+
+class EnvironmentTest < Test::Unit::TestCase
+  include Rubber::Configuration
+
+  def test_known_roles
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
+    assert_equal ['role1', 'role2'], env.known_roles, "list of know roles not correct"
+  end
+
+  def test_env
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
+    e = env.bind()
+    assert_equal 'val1', e['var1'], 'env not retrieving right val'
+    assert_equal 'val2', e['var2'], 'env not retrieving right val'
+    assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
+    assert_equal 'val2', e.var2, 'env not retrieving right val for method missing'
+    
+    assert_equal 'val3', e.var3, 'env not retrieving right val for config in supplemental file'
+
+    e = env.bind('role1', 'nohost')
+    assert_equal 'val1', e['var1'], 'env not retrieving right val'
+    assert_equal 'role1val2', e['var2'], 'env not retrieving right val'
+    assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
+    assert_equal 'role1val2', e.var2, 'env not retrieving right val for method missing'
+
+    e = env.bind('role1', 'host1')
+    assert_equal 'val1', e['var1'], 'env not retrieving right val'
+    assert_equal 'host1val2', e['var2'], 'env not retrieving right val'
+    assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
+    assert_equal 'host1val2', e.var2, 'env not retrieving right val for method missing'
+
+    e = env.bind('norole', 'host1')
+    assert_equal 'val1', e['var1'], 'env not retrieving right val'
+    assert_equal 'host1val2', e['var2'], 'env not retrieving right val'
+    assert_equal 'val1', e.var1, 'env not retrieving right val for method missing'
+    assert_equal 'host1val2', e.var2, 'env not retrieving right val for method missing'
+  end
+
+  def test_combine
+    env = Rubber::Configuration::Environment
+    assert_equal "new", env.combine("old", "new"), "Last should win for scalar combine"
+    assert_equal 5, env.combine(1, 5), "Last should win for scalar combine"
+    assert_equal [1, 2, 3, 4], env.combine([1, 2, 3], [3, 4]), "Arrays should be unioned when combined"
+    assert_equal({1 => "1", 2 => "2", 3 => "3", 4 => "4"}, env.combine({1 => "1", 2 => "2"}, {3 => "3", 4 => "4"}), "Maps should be unioned when combined")
+    assert_equal({1 => "2"}, env.combine({1 => "1"}, {1 => "2"}), "Last should win for scalars in maps when combined")
+    assert_equal({1 => {1 => "1", 2 => "2"}}, env.combine({1 => {1 => "1"}}, {1 => {2 => "2"}}), "Maps should be unioned recursively when combined")
+  end
+
+  def test_expansion
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/expansion")
+    e = env.bind()
+    assert_equal 'val1', e['var1']
+    assert_equal 'val2', e['var2']
+    assert_equal 'val1', e['var3']
+    assert_equal '4 is val2', e['var4']
+    assert_equal 'val1', e['var5']
+    assert_equal %w[lv1 lv2 val1], e['list1']
+    expected = {'mk1' => 'mv1', 'mk2' => 'mv2', 'mk3' => 'val2'}
+    e.map1.each do |k, v|
+      assert_equal expected[k], v
+    end
+      
+    e = env.bind('role1', 'nohost')
+    assert_equal 'role1val1', e['var1']
+    assert_equal 'role1val1', e['var3']
+    assert_equal %w[lv1 lv2 role1val1 role1lv1 role1val2], e['list1']
+
+    e = env.bind('role1', 'host1')
+    assert_equal 'host1val1', e['var1']
+    assert_equal 'host1val1', e['var3']
+    assert_equal %w[lv1 lv2 host1val1 role1lv1 host1val2 host1lv1], e['list1'] # lists are additive
+    
+    e = env.bind('norole', 'host1')
+    assert_equal 'host1val1', e['var1']
+    assert_equal 'host1val1', e['var3']
+    assert_equal %w[lv1 lv2 host1val1 host1lv1 host1val2], e['list1']
+  end
+
+  def test_bool_expansion
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/expansion")
+    e = env.bind()
+    assert_equal true, e['truevar']
+    assert_equal false, e['falsevar']
+    assert_equal true, e['truevar_exp']
+    assert 'true' != e['truevar_exp']
+    assert_equal false, e['falsevar_exp']
+    assert 'false' != e['falsevar_exp']
+    assert_equal 'true thing', e['faketruevar_exp']      
+  end
+
+  def test_secret_env
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/basic")
+    e = env.bind()
+    assert_nil e['rubber_secret'], 'env should not have secret set'
+    
+    fixture_dir = File.expand_path("#{File.dirname(__FILE__)}/fixtures/secret")
+    env = Rubber::Configuration::Environment.new(fixture_dir)
+    e = env.bind()
+    assert_equal "#{fixture_dir}/secret.yml", e['rubber_secret'], 'env should have secret set'
+    assert_equal "secret_val", e['secret_key'], 'env should have gotten setting from secret file'    
+  end
+
+  def test_nested_ref
+    env = Rubber::Configuration::Environment.new("#{File.dirname(__FILE__)}/fixtures/nested")
+    e = env.bind()
+    assert_equal 'val1', e.var1, 'env not retrieving right val'
+    assert_equal 'val3', e.var2.var3, 'env not retrieving right val'
+    assert_equal({'var5' => 'val5'}, e.var2.var4, 'env not retrieving right val')
+    assert_equal ['val6a', 'val6b'], e.var2.var6, 'env not retrieving right val'
+    assert_equal 'val1', e.var2.var7, 'env not retrieving right val'
+    assert_equal 'val3', e.var2.var8, 'env not retrieving right val'
+    assert_equal 'val5', e.var2.var9, 'env not retrieving right val'
+  end
+
+  def test_instances_in_expansion
+    instance = InstanceItem.new('host1', 'domain.com', [RoleItem.new('role1')], '')
+    instance.external_ip = "1.2.3.4"
+    instances = Instance.new(Tempfile.new('testforinstanceexpansion').path)
+    instances.add(instance)
+
+    File.expects(:exist?).returns(true)
+    YAML.expects(:load_file).returns({'var1' =>'"#{rubber_instances.for_role("role1").first.external_ip}"'})
+    Rubber::Configuration.expects(:rubber_instances).returns(instances)
+    env = Rubber::Configuration::Environment.new(nil)
+    e = env.bind()
+
+    assert_equal "\"1.2.3.4\"", e['var1']    
+  end
+
+end