sml-rubber 0.9.1

data/lib/rubber/generator.rb

@@ -0,0 +1,197 @@
+require 'erb'
+require 'find'
+require 'fileutils'
+
+module Rubber
+  module Configuration
+
+    # Handles selection and transformation of a set of config files
+    # based on the host/role they belong to
+    class Generator
+      attr_accessor :file_pattern
+      attr_accessor :no_post
+      attr_accessor :force
+      attr_accessor :fake_root
+      attr_accessor :stop_on_error_cmd
+
+      def initialize(config_dir, roles, host, options={})
+        @config_dir = config_dir
+        @roles = roles.to_a.reverse #First roles take precedence
+        @host = host || 'no_host'
+        @options=options
+      end
+
+      def run
+        config_dirs = []
+        config_dirs << "#{@config_dir}/common/**/**"
+        @roles.sort.each {|role| config_dirs <<  "#{@config_dir}/role/#{role}/**/**" }
+        config_dirs << "#{@config_dir}/host/#{@host}/**/**"
+
+        pat = Regexp.new(file_pattern) if file_pattern
+        
+        config_dirs.each do |dir|
+          Dir[dir].sort.each do |f|
+            next if f =~ /\/(CVS|\.svn)\//
+            if File.file?(f) && (! pat || pat.match(f))
+              LOGGER.info{"Transforming #{f}"}
+              begin
+                transform(IO.read(f), @options)
+              rescue Exception => e
+                lines = e.backtrace.grep(/^\(erb\):([0-9]+)/) {|b| Regexp.last_match(1) }
+                LOGGER.error{"Transformation failed for #{f}#{':' + lines.first if lines.first}"}
+                LOGGER.error e.message
+                exit 1
+              end
+            end
+          end
+        end
+      end
+
+      # Transforms the ERB template given in srcfile and writes the result to
+      # dest_file (if not nil) before returning it
+      def transform(src_data, options={})
+        config = ConfigDescriptor.new
+
+        # for development/test, if we have a fake root, echo any
+        # calls to system
+        if fake_root
+          class << config
+            def system(*args)
+              puts ("Not running system command during a fake_root transformation: #{args.inspect}")
+            end
+            def open(*args)
+              if args.first && args.first =~ /^|/
+                puts ("Not running open/pipe command during a fake_root transformation: #{args.inspect}")
+              else
+                super
+              end
+            end
+            alias ` system
+            alias exec system
+            alias fork system
+          end
+        end
+
+        config.options = options
+        template = ERB.new(src_data, nil, "-")
+        result = template.result(config.get_binding())
+        
+        return if config.skip
+
+        config_path = config.path
+
+        # for development/test, if we have a fake root, then send config
+        # output there, and also put write_cmd output there
+        if fake_root
+          config_path = "write_cmd_" + config.write_cmd.gsub(/[^a-z0-9_-]/i, '') if config.write_cmd
+          config_path = "#{fake_root}/#{config_path}" if config_path
+        end
+
+        if ! config_path && ! (config.read_cmd && config.write_cmd)
+          raise "Transformation requires either a output filename or command"
+        end
+
+        reader = config_path || "|#{config.read_cmd}"
+        orig = IO.read(reader) rescue ""
+
+        # When additive is set we need to only replace between our delimiters
+        if config.additive
+          additive = ["# start rubber #{@host}", "# end rubber #{@host}"] unless additive.is_a? Array
+          pat = /#{config.additive[0]}.*#{config.additive[1]}/m
+          new = "#{config.additive[0]}#{result}#{config.additive[1]}"
+          if orig =~ pat
+            result = orig.gsub(pat, new)
+          else
+            result = orig + new + "\n"
+          end
+        end
+
+        # Only do something if the transformed result is different than what
+        # is currently in the destination file
+        if orig != result || force
+          # create dirs as needed
+          FileUtils.mkdir_p(File.dirname(config_path)) if config_path
+
+          # Write a backup of original
+          open("#{config_path}.bak", 'w') { |f| f.write(orig) } if config_path
+
+          # Write out transformed file
+          writer = config_path || "|#{config.write_cmd}"
+          open(writer, 'w') do |pipe|
+            pipe.write(result)
+          end
+          if config.write_cmd && ! fake_root && $?.exitstatus != 0
+            raise "Config command failed execution:  #{config.write_cmd}"
+          end
+
+          # Set file permissions and owner if needed
+          FileUtils.chmod(config.perms, config_path) if config.perms && config_path
+          FileUtils.chown(config.owner, config.group, config_path) if config_path && (config.owner || config.group)
+
+          # Run post transform command if needed
+          if config.post
+            if orig == result
+              LOGGER.info("Nothing to do, not running post command")
+            elsif no_post
+              LOGGER.info("Not running post command as no post specified")
+            elsif fake_root
+              LOGGER.info("Not running post command as a fake root was given: #{config.post}")
+            else
+              # this lets us abort a script if a command in the middle of it errors out
+              # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
+              config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
+
+              LOGGER.info{"Transformation executing post config command: #{config.post}"}
+              LOGGER.info `#{config.post}`
+              if $?.exitstatus != 0
+                raise "Post command failed execution:  #{config.post}"
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+    # Instances of this object are used accept settings from with
+    # a config file for when it is transformed by Generator
+    class ConfigDescriptor
+      # The output path to write the transformed config file to
+      attr_accessor :path
+      # The command to use for reading the original config file from (e.g. "crontab -l")
+      attr_accessor :read_cmd
+      # The command to use for piping the transformed config file to (e.g. "crontab -")
+      attr_accessor :write_cmd
+      # The command to run after generating the config file if it has changed
+      attr_accessor :post
+      # The owner the output file should have, e.g. "root"
+      attr_accessor :owner
+      # The group the output file should have, e.g. "system"
+      attr_accessor :group
+      # The permissions the output file should have, e.g. 0644 (octal, leading zero is significant)
+      attr_accessor :perms
+      # Sets transformation to be additive, only replaces between given delimiters, e/g/ additive = ["## start", "## end"]
+      attr_accessor :additive
+      # Lets one dynamically determine if a given file gets skipped during transformation
+      attr_accessor :skip
+      # use sudo to write the output file
+      # attr_accessor :sudo
+      # options passed in through code
+      attr_accessor :options
+
+      def get_binding
+        binding
+    end
+    
+    def rubber_env()
+      Rubber::Configuration.rubber_env
+    end
+  
+    def rubber_instances()
+      Rubber::Configuration.rubber_instances
+    end
+
+    end
+
+  end
+end

data/lib/rubber/instance.rb

@@ -0,0 +1,113 @@
+require 'yaml'
+
+module Rubber
+  module Configuration
+
+    # Contains the ec2 instance configuration defined in instance.yml
+    #
+    class Instance
+      attr_reader :file, :artifacts
+      include Enumerable
+
+      def initialize(file)
+        LOGGER.debug{"Reading rubber instances from #{file}"}
+        @file = file
+        @items = {}
+        @artifacts = {'volumes' => {}, 'static_ips' => {}}
+        if ENV['FILTER']
+          @filters = ENV['FILTER'].split(/\s*,\s*/)
+        end
+
+        if File.exist?(@file)
+          item_list = File.open(@file) { |f| YAML.load(f) }
+          if item_list
+            item_list.each do |i|
+              if i.is_a? InstanceItem
+                @items[i.name] = i
+              elsif i.is_a? Hash
+                @artifacts.merge!(i)
+              end
+            end
+          end
+        end
+      end
+
+      def save()
+        data = []
+        data.push(*@items.values)
+        data.push(@artifacts)
+        File.open(@file, "w") { |f| f.write(YAML.dump(data)) }
+      end
+
+      def [](name)
+        @items[name] || @items[name.gsub(/\..*/, '')]
+      end
+
+      # gets the instances for the given role.  If options is nil, all roles
+      # match, otherwise the role has to have options that match exactly
+      def for_role(role_name, options=nil)
+        @items.values.find_all {|ic| ic.roles.any? {|r| r.name == role_name && (! options || r.options == options)}}
+      end
+
+      def filtered()
+        @items.values.find_all {|ic| ! @filters || @filters.include?(ic.name)}
+      end
+
+      def all_roles()
+        @items.collect {|n, i| i.role_names}.flatten.uniq
+      end
+
+      def add(instance_item)
+        @items[instance_item.name] = instance_item
+      end
+
+      def remove(name)
+        @items.delete(name)
+      end
+
+      def each(&block)
+        @items.values.each &block
+      end
+      
+      def size
+        @items.size
+      end
+    end
+
+    # The configuration for a single instance
+    class InstanceItem
+      attr_reader :name, :domain, :roles, :instance_id
+      attr_accessor :external_host, :external_ip
+      attr_accessor :internal_host, :internal_ip
+      attr_accessor :static_ip, :volumes, :partitions
+
+      def initialize(name, domain, roles, instance_id)
+        @name = name
+        @domain = domain
+        @roles = roles
+        @instance_id = instance_id
+      end
+
+      def full_name
+        "#@name.#@domain"
+      end
+
+      def role_names()
+        roles.collect {|r| r.name}
+      end
+    end
+
+    # The configuration for a single role contained in the list
+    # of roles in InstanceItem
+    class RoleItem
+      attr_reader :name, :options
+
+      def initialize(name, options={})
+        @name = name
+        @options = options
+      end
+    end
+
+  end
+end
+

data/lib/rubber/recipes/rubber.rb

@@ -0,0 +1,88 @@
+# add this plugins lib dir to load path for capistrano
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'rubygems'
+require "socket"
+require 'resolv'
+require 'enumerator'
+require 'capistrano/hostcmd'
+require 'pp'
+require 'rubber'
+
+namespace :rubber do
+
+  # advise capistrano's task method so that tasks for non-existant roles don't
+  # fail when roles isn't defined due to using a FILTER for load_roles
+  # If you have a task you need to execute even when there are no
+  # roles, you have to use required_task instead of task - see rubber:create
+  # as an example of this role bootstrapping problem.
+  def allow_optional_tasks(ns)
+    class << ns
+      alias :required_task :task
+      def task(name, options={}, &block)
+        required_task(name, options) do
+          # define empty roles for the case when a task has a role that we don't define anywhere
+          [*options[:roles]].each do |r|
+            roles[r] ||= []
+          end
+          
+          if find_servers_for_task(current_task).empty?
+            logger.info "No servers for task #{name}, skipping"
+            next
+          end
+          block.call
+        end
+      end
+    end
+  end
+
+  allow_optional_tasks(self)
+  on :load, "rubber:init"
+    
+  required_task :init do
+    # Require cap 2.4 since we depend on bugs that have been fixed
+    require 'capistrano/version'
+    if Capistrano::Version::MAJOR < 2 || Capistrano::Version::MINOR < 4
+      fatal "rubber requires capistrano 2.4.0 or greater"
+    end
+    
+    set :rubber_cfg, Rubber::Configuration.get_configuration(RUBBER_ENV)
+    env = rubber_cfg.environment.bind()
+
+    set :cloud, Rubber::Cloud::get_provider(env.cloud_provider || "aws", env, self)
+
+    load_roles() unless rubber_cfg.environment.bind().disable_auto_roles
+    # NOTE: for some reason Capistrano requires you to have both the public and
+    # the private key in the same folder, the public key should have the
+    # extension ".pub".
+    ssh_options[:keys] = env.cloud_providers[env.cloud_provider].key_file
+  end
+
+
+  # Automatically load and define capistrano roles from instance config
+  def load_roles
+    top.roles.clear
+
+    # define empty roles for all known ones so tasks don't fail if a role
+    # doesn't exist due to a filter
+    all_roles = rubber_cfg.instance.all_roles
+    all_roles += rubber_cfg.environment.known_roles
+    all_roles.uniq!
+    all_roles.each {|name| top.roles[name.to_sym] = []}
+
+    # define capistrano host => role mapping for all instances
+    rubber_cfg.instance.filtered.each do |ic|
+      ic.roles.each do |role|
+        opts = Rubber::Util::symbolize_keys(role.options)
+        msg = "Auto role: #{role.name.to_sym} => #{ic.full_name}"
+        msg << ", #{opts.inspect}" if opts.inspect.size > 0
+        logger.info msg
+        top.role role.name.to_sym, ic.full_name, opts
+      end
+    end
+  end
+
+end
+
+Dir[File.join(File.dirname(__FILE__), 'rubber/*.rb')].each do |rubber_part|
+  load(rubber_part)
+end

data/lib/rubber/recipes/rubber/bundles.rb

@@ -0,0 +1,28 @@
+namespace :rubber do
+
+  desc "Back up and register an image of the running instance"
+  task :bundle do
+    if find_servers_for_task(current_task).size > 1
+      fatal "Can only bundle a single instance at a time, use FILTER to limit the scope"
+    end
+    image_name = get_env('IMAGE', "The image name for the bundle", true, Time.now.strftime("%Y%m%d_%H%M"))
+    image_id = cloud.create_image(image_name)
+    logger.info "Newly registered image is: #{image_id}"
+  end
+
+  desc "De-register and Destroy the image for the given name"
+  required_task :destroy_bundle do
+    image_id = get_env('IMAGE_ID', 'The id of the image to be destroyed', true)
+    cloud.destroy_image(image_id)
+  end
+
+  desc "Describes all your own image bundles"
+  required_task :describe_bundles do
+    images = cloud.describe_images()
+    images.each do |image|
+      logger.info "ID: #{image[:id]}"
+      logger.info "Location: #{image[:location]}"
+    end
+  end
+
+end

data/lib/rubber/recipes/rubber/deploy.rb

@@ -0,0 +1,66 @@
+namespace :rubber do
+
+  # Add in some hooks so that we can insert our own hooks at head/tail of
+  # hook chain - this is needed for making monit stop before everyone else.
+  before "deploy:start", "rubber:pre_start"
+  before "deploy:restart", "rubber:pre_restart"
+  before "deploy:stop", "rubber:pre_stop"
+  on :load do
+    after "deploy:start", "rubber:post_start"
+    after "deploy:restart", "rubber:post_restart"
+    after "deploy:stop", "rubber:post_stop"
+  end
+
+  task :pre_start do
+  end
+  task :pre_restart do
+  end
+  task :pre_stop do
+  end
+  task :post_start do
+  end
+  task :post_restart do
+  end
+  task :post_stop do
+  end
+
+  desc <<-DESC
+    Configures the deployed rails application by running the rubber configuration process
+  DESC
+  task :config do
+    opts = {}
+    opts['NO_POST'] = true if ENV['NO_POST']
+    opts['FILE'] = ENV['FILE'] if ENV['FILE']
+    opts['RUBBER_ENV'] = RUBBER_ENV
+
+    # when running deploy:migrations, we need to run config against release_path
+    opts[:deploy_path] = current_release if fetch(:migrate_target, :current).to_sym == :latest
+
+    run_config(opts)
+  end
+
+  def run_config(options={})
+    path = options.delete(:deploy_path) || current_path
+    extra_env = options.keys.inject("") {|all, k|  "#{all} #{k}=\"#{options[k]}\""}
+
+    # Need to do this so we can work with staging instances without having to
+    # checkin instance file between create and bootstrap, as well as during a deploy
+    if fetch(:push_instance_config, false)
+      push_files = [rubber_cfg.instance.file] + rubber_cfg.environment.config_files
+      push_files.each do |file|
+        dest_file = file.sub(/^#{RUBBER_ROOT}\/?/, '')
+        put(File.read(file), File.join(path, dest_file))
+      end
+    end
+
+    # if the user has defined a secret config file, then push it into RUBBER_ROOT/config/rubber
+    secret = rubber_cfg.environment.config_secret
+    if secret && File.exist?(secret)
+      base = rubber_cfg.environment.config_root.sub(/^#{RUBBER_ROOT}\/?/, '')
+      put(File.read(secret), File.join(path, base, File.basename(secret)))
+    end
+
+    sudo "sh -c 'cd #{path} && #{extra_env} rake rubber:config'"
+  end
+
+end