wr0ngway-rubber 1.0.0

data/lib/rubber/dns.rb

@@ -0,0 +1,13 @@
+require "rubber/dns/base.rb"
+
+module Rubber
+  module Dns
+
+    def self.get_provider(provider, env)
+      require "rubber/dns/#{provider}"
+      clazz = Rubber::Dns.const_get(provider.capitalize)
+      return clazz.new(env)
+    end
+    
+  end
+end

data/lib/rubber/environment.rb

@@ -0,0 +1,161 @@
+require 'yaml'
+require 'socket'
+require 'delegate'
+
+module Rubber
+  module Configuration
+    # Contains the configuration defined in rubber.yml
+    # Handles selecting of correct config values based on
+    # the host/role passed into bind
+    class Environment
+      attr_reader :config_root
+      attr_reader :config_files
+      attr_reader :config_secret
+
+      def initialize(config_root)
+        @config_root = config_root
+        @config_files = ["#{@config_root}/rubber.yml"] + Dir["#{@config_root}/rubber-*.yml"].sort
+        @items = {}
+        @config_files.each { |file| read_config(file) }
+        @config_secret = bind().rubber_secret
+        read_config(@config_secret) if @config_secret
+      end
+      
+      def read_config(file)
+        LOGGER.debug{"Reading rubber configuration from #{file}"}
+        if File.exist?(file)
+          @items = Environment.combine(@items, YAML.load_file(file) || {})
+        end
+      end
+
+      def known_roles
+        roles_dir = File.join(@config_root, "role")
+        roles = Dir.entries(roles_dir)
+        roles.delete_if {|d| d =~ /(^\..*)/}
+      end
+
+      def current_host
+        Socket::gethostname.gsub(/\..*/, '')
+      end
+      
+      def current_full_host
+        Socket::gethostname
+      end
+
+      def bind(roles = nil, host = nil)
+        BoundEnv.new(@items, roles, host)
+      end
+
+      # combine old and new into a single value:
+      # non-nil wins if other is nil
+      # arrays just get unioned
+      # hashes also get unioned, but the values of conflicting keys get combined
+      # All else, the new value wins
+      def self.combine(old, new)
+        return old if new.nil?
+        return new if old.nil?
+        value = old
+        if old.is_a?(Hash) && new.is_a?(Hash)
+          value = old.clone
+          new.each do |nk, nv|
+            value[nk] = combine(value[nk], nv)
+          end
+        elsif old.is_a?(Array) && new.is_a?(Array)
+          value = old | new
+        else
+          value = new
+        end
+        return value
+      end
+
+      class HashValueProxy < Hash
+        attr_reader :global
+
+        def initialize(global, receiver)
+          @global = global
+          super()
+          replace(receiver)
+        end
+
+        def [](name)
+          value = super(name)
+          value = global[name] if global && !value
+          return expand(value)
+        end
+
+        def each
+          each_key do |key|
+            yield key, self[key]
+          end
+        end
+
+        def method_missing(method_id)
+          key = method_id.id2name
+          return self[key]
+        end
+
+        def expand_string(val)
+          while val =~ /\#\{[^\}]+\}/
+            val = eval('%Q{' + val + '}', binding)
+          end
+          val = true if val =="true"
+          val = false if val == "false"
+          return val
+        end
+
+        def expand(value)
+          val = case value
+            when Hash
+              HashValueProxy.new(global || self, value)
+            when String
+              expand_string(value)
+            when Enumerable
+              value.collect {|v| expand(v) }
+            else
+              value
+          end
+          return val
+        end
+
+      end
+
+      class BoundEnv < HashValueProxy
+        attr_reader :roles
+        attr_reader :host
+        attr_reader :full_host
+
+        def initialize(global, roles, host)
+          @roles = roles
+          @host = host
+          bound_global = bind_config(global)
+          super(nil, bound_global)
+          @full_host = "#{host}.#{domain}" rescue nil
+        end
+
+        # Forces role/host overrides into config
+        def bind_config(global)
+          global = global.clone()
+          role_overrides = global.delete("roles") || {}
+          host_overrides = global.delete("hosts") || {}
+          roles.to_a.each do |role|
+            role_overrides[role].each do |k, v|
+              global[k] = Environment.combine(global[k], v)
+            end if role_overrides[role]
+          end
+          host_overrides[host].each do |k, v|
+            global[k] = Environment.combine(global[k], v)
+          end if host_overrides[host]
+          return global
+        end
+        
+        def method_missing(method_id)
+          key = method_id.id2name
+          return self[key]
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/rubber/generator.rb

@@ -0,0 +1,197 @@
+require 'erb'
+require 'find'
+require 'fileutils'
+
+module Rubber
+  module Configuration
+
+    # Handles selection and transformation of a set of config files
+    # based on the host/role they belong to
+    class Generator
+      attr_accessor :file_pattern
+      attr_accessor :no_post
+      attr_accessor :force
+      attr_accessor :fake_root
+      attr_accessor :stop_on_error_cmd
+
+      def initialize(config_dir, roles, host, options={})
+        @config_dir = config_dir
+        @roles = roles.to_a.reverse #First roles take precedence
+        @host = host || 'no_host'
+        @options=options
+      end
+
+      def run
+        config_dirs = []
+        config_dirs << "#{@config_dir}/common/**/**"
+        @roles.sort.each {|role| config_dirs <<  "#{@config_dir}/role/#{role}/**/**" }
+        config_dirs << "#{@config_dir}/host/#{@host}/**/**"
+
+        pat = Regexp.new(file_pattern) if file_pattern
+        
+        config_dirs.each do |dir|
+          Dir[dir].sort.each do |f|
+            next if f =~ /\/(CVS|\.svn)\//
+            if File.file?(f) && (! pat || pat.match(f))
+              LOGGER.info{"Transforming #{f}"}
+              begin
+                transform(IO.read(f), @options)
+              rescue Exception => e
+                lines = e.backtrace.grep(/^\(erb\):([0-9]+)/) {|b| Regexp.last_match(1) }
+                LOGGER.error{"Transformation failed for #{f}#{':' + lines.first if lines.first}"}
+                LOGGER.error e.message
+                exit 1
+              end
+            end
+          end
+        end
+      end
+
+      # Transforms the ERB template given in srcfile and writes the result to
+      # dest_file (if not nil) before returning it
+      def transform(src_data, options={})
+        config = ConfigDescriptor.new
+
+        # for development/test, if we have a fake root, echo any
+        # calls to system
+        if fake_root
+          class << config
+            def system(*args)
+              puts ("Not running system command during a fake_root transformation: #{args.inspect}")
+            end
+            def open(*args)
+              if args.first && args.first =~ /^|/
+                puts ("Not running open/pipe command during a fake_root transformation: #{args.inspect}")
+              else
+                super
+              end
+            end
+            alias ` system
+            alias exec system
+            alias fork system
+          end
+        end
+
+        config.options = options
+        template = ERB.new(src_data, nil, "-")
+        result = template.result(config.get_binding())
+        
+        return if config.skip
+
+        config_path = config.path
+
+        # for development/test, if we have a fake root, then send config
+        # output there, and also put write_cmd output there
+        if fake_root
+          config_path = "write_cmd_" + config.write_cmd.gsub(/[^a-z0-9_-]/i, '') if config.write_cmd
+          config_path = "#{fake_root}/#{config_path}" if config_path
+        end
+
+        if ! config_path && ! (config.read_cmd && config.write_cmd)
+          raise "Transformation requires either a output filename or command"
+        end
+
+        reader = config_path || "|#{config.read_cmd}"
+        orig = IO.read(reader) rescue ""
+
+        # When additive is set we need to only replace between our delimiters
+        if config.additive
+          additive = ["# start rubber #{@host}", "# end rubber #{@host}"] unless additive.is_a? Array
+          pat = /#{config.additive[0]}.*#{config.additive[1]}/m
+          new = "#{config.additive[0]}#{result}#{config.additive[1]}"
+          if orig =~ pat
+            result = orig.gsub(pat, new)
+          else
+            result = orig + new + "\n"
+          end
+        end
+
+        # Only do something if the transformed result is different than what
+        # is currently in the destination file
+        if orig != result || force
+          # create dirs as needed
+          FileUtils.mkdir_p(File.dirname(config_path)) if config_path
+
+          # Write a backup of original
+          open("#{config_path}.bak", 'w') { |f| f.write(orig) } if config_path
+
+          # Write out transformed file
+          writer = config_path || "|#{config.write_cmd}"
+          open(writer, 'w') do |pipe|
+            pipe.write(result)
+          end
+          if config.write_cmd && ! fake_root && $?.exitstatus != 0
+            raise "Config command failed execution:  #{config.write_cmd}"
+          end
+
+          # Set file permissions and owner if needed
+          FileUtils.chmod(config.perms, config_path) if config.perms && config_path
+          FileUtils.chown(config.owner, config.group, config_path) if config_path && (config.owner || config.group)
+
+          # Run post transform command if needed
+          if config.post
+            if orig == result
+              LOGGER.info("Nothing to do, not running post command")
+            elsif no_post
+              LOGGER.info("Not running post command as no post specified")
+            elsif fake_root
+              LOGGER.info("Not running post command as a fake root was given: #{config.post}")
+            else
+              # this lets us abort a script if a command in the middle of it errors out
+              # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
+              config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
+
+              LOGGER.info{"Transformation executing post config command: #{config.post}"}
+              LOGGER.info `#{config.post}`
+              if $?.exitstatus != 0
+                raise "Post command failed execution:  #{config.post}"
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+    # Instances of this object are used accept settings from with
+    # a config file for when it is transformed by Generator
+    class ConfigDescriptor
+      # The output path to write the transformed config file to
+      attr_accessor :path
+      # The command to use for reading the original config file from (e.g. "crontab -l")
+      attr_accessor :read_cmd
+      # The command to use for piping the transformed config file to (e.g. "crontab -")
+      attr_accessor :write_cmd
+      # The command to run after generating the config file if it has changed
+      attr_accessor :post
+      # The owner the output file should have, e.g. "root"
+      attr_accessor :owner
+      # The group the output file should have, e.g. "system"
+      attr_accessor :group
+      # The permissions the output file should have, e.g. 0644 (octal, leading zero is significant)
+      attr_accessor :perms
+      # Sets transformation to be additive, only replaces between given delimiters, e/g/ additive = ["## start", "## end"]
+      attr_accessor :additive
+      # Lets one dynamically determine if a given file gets skipped during transformation
+      attr_accessor :skip
+      # use sudo to write the output file
+      # attr_accessor :sudo
+      # options passed in through code
+      attr_accessor :options
+
+      def get_binding
+        binding
+    end
+    
+    def rubber_env()
+      Rubber::Configuration.rubber_env
+    end
+  
+    def rubber_instances()
+      Rubber::Configuration.rubber_instances
+    end
+
+    end
+
+  end
+end

data/lib/rubber/instance.rb

@@ -0,0 +1,165 @@
+require 'yaml'
+
+module Rubber
+  module Configuration
+
+    # Contains the ec2 instance configuration defined in instance.yml
+    #
+    class Instance
+      attr_reader :file, :artifacts
+      include Enumerable
+
+      def initialize(file)
+        LOGGER.debug{"Reading rubber instances from #{file}"}
+        @file = file
+        @items = {}
+        @artifacts = {'volumes' => {}, 'static_ips' => {}}
+        if ENV['FILTER']
+          @filters = ENV['FILTER'].split(/\s*,\s*/)
+        end
+
+        if File.exist?(@file)
+          item_list = File.open(@file) { |f| YAML.load(f) }
+          if item_list
+            item_list.each do |i|
+              if i.is_a? InstanceItem
+                @items[i.name] = i
+              elsif i.is_a? Hash
+                @artifacts.merge!(i)
+              end
+            end
+          end
+        end
+      end
+
+      def save()
+        data = []
+        data.push(*@items.values)
+        data.push(@artifacts)
+        File.open(@file, "w") { |f| f.write(YAML.dump(data)) }
+      end
+
+      def [](name)
+        @items[name] || @items[name.gsub(/\..*/, '')]
+      end
+
+      # gets the instances for the given role.  If options is nil, all roles
+      # match, otherwise the role has to have options that match exactly
+      def for_role(role_name, options=nil)
+        @items.values.find_all {|ic| ic.roles.any? {|r| r.name == role_name && (! options || r.options == options)}}
+      end
+
+      def filtered()
+        @items.values.find_all {|ic| ! @filters || @filters.include?(ic.name)}
+      end
+
+      def all_roles()
+        @items.collect {|n, i| i.role_names}.flatten.uniq
+      end
+
+      def add(instance_item)
+        @items[instance_item.name] = instance_item
+      end
+
+      def remove(name)
+        @items.delete(name)
+      end
+
+      def each(&block)
+        @items.values.each &block
+      end
+      
+      def size
+        @items.size
+      end
+    end
+
+    # The configuration for a single instance
+    class InstanceItem
+      attr_reader :name, :domain, :instance_id, :security_groups
+      attr_accessor :roles, :zone
+      attr_accessor :external_host, :external_ip
+      attr_accessor :internal_host, :internal_ip
+      attr_accessor :static_ip, :volumes, :partitions
+
+      def initialize(name, domain, roles, instance_id, security_group_list=[])
+        @name = name
+        @domain = domain
+        @roles = roles
+        @instance_id = instance_id
+        @security_groups = security_group_list
+      end
+
+      def full_name
+        "#@name.#@domain"
+      end
+
+      def role_names()
+        roles.collect {|r| r.name}
+      end
+    end
+
+    # The configuration for a single role contained in the list
+    # of roles in InstanceItem
+    class RoleItem
+      attr_reader :name, :options
+
+      def self.expand_role_dependencies(roles, dependency_map, expanded=[])
+        roles = Array(roles)
+        roles.each do |role|
+          unless expanded.include?(role)
+            expanded << role
+            needed = dependency_map[role]
+            expand_role_dependencies(needed, dependency_map, expanded)
+          end
+        end
+        return expanded
+      end
+
+      def self.parse(str)
+        data = str.split(':');
+        role = Rubber::Configuration::RoleItem.new(data[0])
+        if data[1]
+          data[1].split(';').each do |pair|
+            p = pair.split('=')
+            val = case p[1]
+                    when 'true' then true
+                    when 'false' then false
+                    else p[1] end
+            role.options[p[0]] = val
+          end
+        end
+        return role
+      end
+
+      def to_s
+        str = @name
+        @options.each_with_index do |kv, i|
+          str += (i == 0 ? ':' : ';')
+          str += "#{kv[0]}=#{kv[1]}"
+        end
+        return str
+      end
+
+      def initialize(name, options={})
+        @name = name
+        @options = options || {}
+      end
+
+      def eql?(rhs)
+        rhs && @name == rhs.name && @options == rhs.options
+      end
+      alias == eql?
+
+      def hash()
+        @name.hash
+      end
+
+      def <=>(rhs)
+        return @name <=> rhs.name
+      end
+    end
+
+  end
+end
+

data/lib/rubber/recipes/rubber/bundles.rb

@@ -0,0 +1,28 @@
+namespace :rubber do
+
+  desc "Back up and register an image of the running instance"
+  task :bundle do
+    if find_servers_for_task(current_task).size > 1
+      fatal "Can only bundle a single instance at a time, use FILTER to limit the scope"
+    end
+    image_name = get_env('IMAGE', "The image name for the bundle", true, Time.now.strftime("%Y%m%d_%H%M"))
+    image_id = cloud.create_image(image_name)
+    logger.info "Newly registered image is: #{image_id}"
+  end
+
+  desc "De-register and Destroy the image for the given name"
+  required_task :destroy_bundle do
+    image_id = get_env('IMAGE_ID', 'The id of the image to be destroyed', true)
+    cloud.destroy_image(image_id)
+  end
+
+  desc "Describes all your own image bundles"
+  required_task :describe_bundles do
+    images = cloud.describe_images()
+    images.each do |image|
+      logger.info "ID: #{image[:id]}"
+      logger.info "Location: #{image[:location]}"
+    end
+  end
+
+end

data/lib/rubber/recipes/rubber/deploy.rb

@@ -0,0 +1,90 @@
+namespace :rubber do
+
+  # Add in some hooks so that we can insert our own hooks at head/tail of
+  # hook chain - this is needed for making monit stop before everyone else.
+  before "deploy:start", "rubber:pre_start"
+  before "deploy:restart", "rubber:pre_restart"
+  before "deploy:stop", "rubber:pre_stop"
+  on :load do
+    after "deploy:start", "rubber:post_start"
+    after "deploy:restart", "rubber:post_restart"
+    after "deploy:stop", "rubber:post_stop"
+  end
+
+  task :pre_start do
+  end
+  task :pre_restart do
+  end
+  task :pre_stop do
+  end
+  task :post_start do
+  end
+  task :post_restart do
+  end
+  task :post_stop do
+  end
+
+  # Don't want to do rubber:config for update_code as that tree isn't official
+  # until it is 'committed' by the symlink task (and doing so causes it to run
+  # for bootstrap_db which should only config the db config file).  However,
+  # deploy:migrations doesn't call update, so we need an additional trigger for
+  # it
+  after "deploy:update", "rubber:config"
+  after "deploy:rollback_code", "rubber:config"
+  before "deploy:migrate", "rubber:config"
+
+  desc <<-DESC
+    Configures the deployed rails application by running the rubber configuration process
+  DESC
+  task :config do
+    opts = {}
+    opts['NO_POST'] = true if ENV['NO_POST']
+    opts['FILE'] = ENV['FILE'] if ENV['FILE']
+    opts['RUBBER_ENV'] = RUBBER_ENV
+    # we need to set rails env as well because when running rake
+    # in a rails project, rails gets loaded before the rubber hook gets run
+    opts['RAILS_ENV'] = RUBBER_ENV
+
+    # when running deploy:migrations, we need to run config against release_path
+    opts[:deploy_path] = current_release if fetch(:migrate_target, :current).to_sym == :latest
+
+    run_config(opts)
+  end
+
+  # because we start server as appserver user, but migrate as root, server needs to be able to write logs, etc.
+  before "rubber:pre_start", "rubber:setup_app_permissions"
+  before "rubber:pre_restart", "rubber:setup_app_permissions"
+
+  desc <<-DESC
+    Sets permissions of files in application directory to be owned by app_user.
+  DESC
+  task :setup_app_permissions do
+    run "find #{shared_path} -name cached-copy -prune -o -print | xargs chown #{rubber_env.app_user}:#{rubber_env.app_user}"
+    run "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{current_path}/tmp"
+  end
+
+  def run_config(options={})
+    path = options.delete(:deploy_path) || current_path
+    extra_env = options.keys.inject("") {|all, k|  "#{all} #{k}=\"#{options[k]}\""}
+
+    # Need to do this so we can work with staging instances without having to
+    # checkin instance file between create and bootstrap, as well as during a deploy
+    if fetch(:push_instance_config, false)
+      push_files = [rubber_instances.file] + rubber_cfg.environment.config_files
+      push_files.each do |file|
+        dest_file = file.sub(/^#{RUBBER_ROOT}\/?/, '')
+        put(File.read(file), File.join(path, dest_file), :mode => "+r")
+      end
+    end
+
+    # if the user has defined a secret config file, then push it into RUBBER_ROOT/config/rubber
+    secret = rubber_cfg.environment.config_secret
+    if secret && File.exist?(secret)
+      base = rubber_cfg.environment.config_root.sub(/^#{RUBBER_ROOT}\/?/, '')
+      put(File.read(secret), File.join(path, base, File.basename(secret)), :mode => "+r")
+    end
+
+    sudo "sh -c 'cd #{path} && #{extra_env} rake rubber:config'"
+  end
+
+end