rubber 1.15.0 → 2.0.0.pre1

data/lib/rubber/commands/config.rb

@@ -0,0 +1,70 @@
+
+module Rubber
+  module Commands
+
+    class Config < Clamp::Command
+
+      def self.subcommand_name
+        "config"
+      end
+
+      def self.subcommand_description
+        "Transform templates in the config/rubber tree"
+      end
+      
+      def self.description
+        "Generate system files by transforming the files in the config/rubber tree"
+      end
+      
+      option ["--host", "-h"], "HOST", "Override the instance's host for generation"
+      option ["--roles", "-r"], "ROLES", "Override the instance's roles for generation" do |str|
+        str.split(",")
+      end
+      option ["--file", "-p"], "FILE", "Only generate files matching the given pattern"
+      option ["--no_post", "-n"], :flag, "Skip running post commands"
+      option ["--force", "-f"], :flag, "Overwrite files that already exist"
+      
+      def execute
+        cfg = Rubber::Configuration.get_configuration(Rubber.env)
+        instance_alias = cfg.environment.current_host
+        instance = cfg.instance[instance_alias]
+        if instance
+          role_names = instance.role_names
+          env = cfg.environment.bind(role_names, instance_alias)
+          gen = Rubber::Configuration::Generator.new("#{Rubber.root}/config/rubber", role_names, instance_alias)
+        elsif ['development', 'test'].include?(Rubber.env)
+          instance_alias = host || instance_alias
+          role_names = roles || cfg.environment.known_roles
+          role_items = role_names.collect do |r|
+            Rubber::Configuration::RoleItem.new(r, r == "db" ? {'primary' => true} : {})
+          end
+          env = cfg.environment.bind(role_names, instance_alias)
+          domain = env.domain
+          instance = Rubber::Configuration::InstanceItem.new(instance_alias, domain, role_items,
+                                                             'dummyid', 'm1.small', 'ami-7000f019', ['dummygroup'])
+          instance.external_host = instance.full_name
+          instance.external_ip = "127.0.0.1"
+          instance.internal_host = instance.full_name
+          instance.internal_ip = "127.0.0.1"
+          cfg.instance.add(instance)
+          gen = Rubber::Configuration::Generator.new("#{Rubber.root}/config/rubber", role_names, instance_alias)
+          gen.fake_root ="#{Rubber.root}/tmp/rubber"
+        else
+          puts "Instance not found for host: #{instance_alias}"
+          exit 1
+        end
+        
+        if file
+          gen.file_pattern = file
+        end
+        gen.no_post = no_post?
+        gen.force = force?
+        gen.stop_on_error_cmd = env.stop_on_error_cmd
+        gen.run
+
+      end
+
+    end
+
+  end
+end

data/lib/rubber/commands/cron.rb

@@ -0,0 +1,127 @@
+require 'open4'
+require 'fileutils'
+
+module Rubber
+  module Commands
+
+    class Cron < Clamp::Command
+
+      def self.subcommand_name
+        "cron"
+      end
+
+      def self.subcommand_description
+        "A cron-safe way for running commands"
+      end
+      
+      def self.description
+        Rubber::Util.clean_indent( <<-EOS
+          Runs the given command, sending all stdout/stderr to a logfile, but echoing
+          the entire file if the command exits with an error.  Exits with the same
+          error code the command exited with
+        EOS
+        )
+      end
+      
+      # options for all tasks
+      option ["-e", "--echoerr"],
+             :flag,
+             "Log _and_ echo stderr",
+             :default => false
+      option ["-o", "--echoout"],
+             :flag,
+             "Log _and_ echo stdout",
+             :default => false
+      option ["-r", "--rootdir"],
+             "ROOTDIR",
+             "Root dir to cd into before running\n (default: <Rubber.root>)"
+      option ["-l", "--logfile"],
+             "LOGFILE",
+             "Logs output to the given file\n (default: <rootdir>/log/cron-sh-<time>.log)"
+      option ["-u", "--user"],
+             "USER",
+             "User to run the command as"
+      option ["--task"],
+             :flag,
+             "Run the arguments with rubber"
+      option ["--ruby"],
+             :flag,
+             "Run the arguments with ruby"
+      option ["--runner"],
+             :flag,
+             "Run the arguments with rails runner"
+      option ["--rake"],
+             :flag,
+             "Run the arguments with rake"
+      parameter "COMMAND ...", "the command to run"
+
+      def execute
+        self.rootdir ||= Rubber.root
+        self.logfile ||= "#{Rubber.root}/log/cron-sh-#{Time.now.tv_sec}.log"
+        
+        cmd = command_list
+        log = logfile
+        
+        if task?
+          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
+          log = "#{rootdir}/log/cron-task-#{ident}-#{Time.now.tv_sec}.log"
+          cmd = ["rubber"] + cmd
+        elsif ruby?
+          ruby_code = cmd.join(' ')
+          ident = ruby_code.gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
+          log = "#{rootdir}/log/cron-ruby-#{ident}-#{Time.now.tv_sec}.log"
+          cmd = ["ruby", "-e", ruby_code]
+        elsif runner?
+          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
+          log = "#{rootdir}/log/cron-runner-#{ident}-#{Time.now.tv_sec}.log"
+          cmd = ["rails", "runner"] + cmd
+        elsif rake?
+          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
+          log = "#{rootdir}/log/cron-rake-#{ident}-#{Time.now.tv_sec}.log"
+          cmd = ["rake"] + cmd
+        end
+        
+        if user
+          if user =~ /^[0-9]+$/
+            uid = user.to_i
+          else
+            uid = Etc.getpwnam(user).uid
+          end
+          Process::UID.change_privilege(uid) if uid != Process.euid
+        end
+        
+        # make sure dir containing logfile exists
+        FileUtils.mkdir_p(File.dirname(log))
+        
+        # set current directory to rootdir
+        Dir.chdir(rootdir)
+  
+        status = Open4::popen4(*cmd) do |pid, stdin, stdout, stderr|
+          File.open(log, "w") do | fh |
+            threads = []
+            threads <<  Thread.new(stdout) do |stdout|
+               stdout.each { |line| $stdout.puts line if echoout?; fh.print line; fh.flush }
+            end
+            threads <<  Thread.new(stderr) do |stderr|
+               stderr.each { |line| $stderr.puts line if echoerr?; fh.print line; fh.flush }
+            end
+            threads.each { |t| t.join }
+          end
+        end
+        
+        result = status.exitstatus
+        if result != 0
+          puts ""
+          puts "*** Process exited with non-zero error code, full output follows"
+          puts "*** Command was: #{cmd.join(' ')}"
+          puts ""
+          puts IO.read(log)
+        end
+        
+        exit(result)
+      end
+    
+    end
+    
+  end
+end

data/lib/rubber/commands/util.rb

@@ -0,0 +1,308 @@
+
+module Rubber
+  module Commands
+
+    class RotateLogs < Clamp::Command
+      
+      def self.subcommand_name
+        "util:rotate_logs"
+      end
+
+      def self.subcommand_description
+        "Rotate the matching log files"
+      end
+      
+      option ["-d", "--directory"],
+             "DIRECTORY",
+             "The directory containing files to be rotated\nRequired"
+      option ["-p", "--pattern"],
+             "PATTERN",
+             "The glob pattern for matching files\n",
+             :default => "*.log"
+      option ["-a", "--age"],
+             "AGE",
+             "The number of days to keep rotated files\n",
+             :default => 7,
+             &Proc.new {|a| Integer(a)}
+
+      def execute
+        signal_usage_error "DIRECTORY is required" unless directory
+        
+        log_src_dir = directory
+        log_file_glob = pattern
+        log_file_age = age
+
+        rotated_date = (Date.today - 1).strftime('%Y%m%d')
+        puts "Rotating logfiles located at: #{log_src_dir}/#{log_file_glob}"
+        Dir["#{log_src_dir}/#{log_file_glob}"].each do |logfile|
+          rotated_file = "#{logfile}.#{rotated_date}"
+          if File.exist?(rotated_file)
+            rotated_file += "_#{Time.now.to_i}"
+          end
+          FileUtils.cp(logfile, rotated_file)
+          File.truncate logfile, 0
+        end
+
+        tdate = Date.today - log_file_age
+        threshold = Time.local(tdate.year, tdate.month, tdate.day)
+        puts "Cleaning rotated log files older than #{log_file_age} days"
+        Dir["#{log_src_dir}/#{log_file_glob}.[0-9]*"].each do |logfile|
+          if File.mtime(logfile) < threshold
+            FileUtils.rm_f(logfile)
+          end
+        end
+      end
+
+    end
+      
+    class Backup < Clamp::Command
+      
+      def self.subcommand_name
+        "util:backup"
+      end
+
+      def self.subcommand_description
+        "Performs a cyclical backup"
+      end
+      
+      def self.description
+        "Performs a cyclical backup by storing the results of COMMAND to the backup\ndirectory (and the cloud)"
+      end
+      
+      option ["-n", "--name"],
+             "NAME",
+             "What to name the backup\nRequired"
+      option ["-d", "--directory"],
+             "DIRECTORY",
+             "The directory to stage backups into\nRequired"
+      option ["-c", "--command"],
+             "COMMAND",
+             "The command used to extract the data to be\nbacked up\nRequired"
+      option ["-a", "--age"],
+             "AGE",
+             "The number of days to keep rotated files\n",
+             :default => 7,
+             &Proc.new {|a| Integer(a)}
+
+      def execute
+        signal_usage_error "NAME, DIRECTORY and COMMAND are required" unless name && directory && command
+        
+        # extra variables for command interpolation
+        time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
+        dir = directory
+        
+        # differentiate by env
+        cloud_prefix = "#{name}/"
+        self.name = "#{Rubber.env}_#{self.name}"
+
+        FileUtils.mkdir_p(directory)
+      
+        backup_cmd = command.gsub(/%([^%]+)%/, '#{\1}')
+        backup_cmd = eval('%Q{' + backup_cmd + '}')
+      
+        puts "Backing up with command: '#{backup_cmd}'"
+        system backup_cmd || fail("Command failed: '#{backup_cmd.inspect}'")
+        puts "Backup created"
+      
+        backup_bucket = Rubber.cloud.env.backup_bucket
+        if backup_bucket
+          newest = Dir.entries(directory).grep(/^[^.]/).sort_by {|f| File.mtime(File.join(directory,f))}.last
+          dest = "#{cloud_prefix}#{newest}"
+          puts "Saving backup to cloud: #{backup_bucket}:#{dest}"
+          Rubber.cloud.storage(backup_bucket).store(dest, open(File.join(directory, newest)))
+        end
+      
+        tdate = Date.today - age
+        threshold = Time.local(tdate.year, tdate.month, tdate.day)
+        puts "Cleaning backups older than #{age} days"
+        Dir["#{directory}/*"].each do |file|
+          if File.mtime(file) < threshold
+            puts "Deleting #{file}"
+            FileUtils.rm_f(file)
+          end
+        end
+      
+        if backup_bucket
+          puts "Cleaning cloud backups older than #{age} days from: #{backup_bucket}:#{cloud_prefix}"
+          Rubber.cloud.storage(backup_bucket).walk_tree(cloud_prefix) do |f|
+            if f.last_modified < threshold
+              puts "Deleting #{f.key}"
+              f.destroy
+            end
+          end
+        end
+      end
+      
+    end
+  
+    class BackupDb < Clamp::Command
+      
+      def self.subcommand_name
+        "util:backup_db"
+      end
+
+      def self.subcommand_description
+        "Performs a cyclical database backup"
+      end
+      
+      def self.description
+        Rubber::Util.clean_indent( <<-EOS
+          Performs a cyclical backup of the database by storing the results of COMMAND
+          to the backup directory (and the cloud)
+        EOS
+        )
+      end
+      
+      option ["-d", "--directory"],
+             "DIRECTORY",
+             "The directory to stage backups into\nRequired"
+      option ["-u", "--dbuser"],
+             "DBUSER",
+             "The database user to connect with\nRequired"
+      option ["-p", "--dbpass"],
+             "DBUSER",
+             "The database password to connect with"
+      option ["-h", "--dbhost"],
+             "DBHOST",
+             "The database host to connect to\nRequired"
+      option ["-n", "--dbname"],
+             "DBNAME",
+             "The database name to backup\nRequired"
+      option ["-a", "--age"],
+             "AGE",
+             "The number of days to keep rotated files\n",
+             :default => 7,
+             &Proc.new {|a| Integer(a)}
+
+      def execute
+        signal_usage_error "DIRECTORY, DBUSER, DBHOST, DBNAME are required" unless directory && dbuser && dbhost && dbname
+        
+        
+        time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
+        backup_file = "#{directory}/#{Rubber.env}_dump_#{time_stamp}.sql.gz"
+        FileUtils.mkdir_p(File.dirname(backup_file))
+        
+        # extra variables for command interpolation
+        dir = directory
+        user = dbuser
+        pass = dbpass
+        pass = nil if pass && pass.strip.size == 0
+        host = dbhost
+        name = dbname
+      
+        raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless Rubber.config.db_backup_cmd
+        db_backup_cmd = Rubber.config.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
+        db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
+      
+        puts "Backing up database with command: '#{db_backup_cmd}'"
+        system db_backup_cmd || fail("Command failed: '#{db_backup_cmd.inspect}'")
+        puts "Created backup: #{backup_file}"
+      
+        cloud_prefix = "db/"
+        backup_bucket = Rubber.cloud.env.backup_bucket
+        if backup_bucket
+          dest = "#{cloud_prefix}#{File.basename(backup_file)}"
+          puts "Saving db backup to cloud: #{backup_bucket}:#{dest}"
+          Rubber.cloud.storage(backup_bucket).store(dest, open(backup_file))
+        end
+      
+        tdate = Date.today - age
+        threshold = Time.local(tdate.year, tdate.month, tdate.day)
+        puts "Cleaning backups older than #{age} days"
+        Dir["#{directory}/*"].each do |file|
+          if file =~ /#{Rubber.env}_dump_/ && File.mtime(file) < threshold
+            puts "Deleting #{file}"
+            FileUtils.rm_f(file)
+          end
+        end
+      
+        if backup_bucket
+          puts "Cleaning cloud backups older than #{age} days from: #{backup_bucket}:#{cloud_prefix}"
+          Rubber.cloud.storage(backup_bucket).walk_tree(cloud_prefix) do |f|
+            if f.key =~ /#{Rubber.env}_dump_/ && f.last_modified < threshold
+              puts "Deleting #{f.key}"
+              f.destroy
+            end
+          end
+        end
+        
+      end
+
+    end
+  
+    class RestoreDb < Clamp::Command
+      
+      def self.subcommand_name
+        "util:restore_db"
+      end
+
+      def self.subcommand_description
+        "Performs a restore of the database"
+      end
+      
+      option ["-f", "--filename"],
+             "FILENAME",
+             "The key of cloud object to use\nMost recent if not supplied"
+      option ["-u", "--dbuser"],
+             "DBUSER",
+             "The database user to connect with\nRequired"
+      option ["-p", "--dbpass"],
+             "DBPASS",
+             "The database password to connect with"
+      option ["-h", "--dbhost"],
+             "DBHOST",
+             "The database host to connect to\nRequired"
+      option ["-n", "--dbname"],
+             "DBNAME",
+             "The database name to backup\nRequired"
+
+      def execute
+        signal_usage_error "DBUSER, DBHOST are required" unless dbuser && dbhost
+
+        # extra variables for command interpolation
+        file = filename
+        user = dbuser
+        pass = dbpass
+        pass = nil if pass && pass.strip.size == 0
+        host = dbhost
+        name = dbname
+      
+        raise "No db_restore_cmd defined in rubber.yml" unless Rubber.config.db_restore_cmd
+        db_restore_cmd = Rubber.config.db_restore_cmd.gsub(/%([^%]+)%/, '#{\1}')
+        db_restore_cmd = eval('%Q{' + db_restore_cmd + '}')
+      
+        # try to fetch a matching file from the cloud (if backup_bucket given)
+        backup_bucket = Rubber.cloud.env.backup_bucket
+        raise "No backup_bucket defined in rubber.yml" unless backup_bucket
+        
+        key = nil
+        cloud_prefix = "db/"
+        if filename
+          key = "#{cloud_prefix}#{filename}"
+        else
+          puts "trying to fetch last modified cloud backup"
+          max = nil
+          Rubber.cloud.storage(backup_bucket).walk_tree(cloud_prefix) do |f|
+            if f.key =~ /#{Rubber.env}_dump_/
+              max = f if max.nil? || f.last_modified > max.last_modified
+            end
+          end
+          key = max.key if max
+        end
+        
+        raise "could not access backup file from cloud" unless key
+      
+        puts "piping restore data from #{backup_bucket}:#{key} to command [#{db_restore_cmd}]"
+        
+        IO.popen(db_restore_cmd, 'wb') do |p|
+          Rubber.cloud.storage(backup_bucket).fetch(key) do |chunk|
+            p.write chunk
+          end
+        end
+      
+      end
+      
+    end
+      
+  end
+end