rubber-jcarnegie 0.0.1

data/lib/rubber/environment.rb

@@ -0,0 +1,177 @@
+require 'yaml'
+require 'socket'
+require 'delegate'
+
+module Rubber
+  module Configuration
+    # Contains the configuration defined in rubber.yml
+    # Handles selecting of correct config values based on
+    # the host/role passed into bind
+    class Environment
+      attr_reader :config_root
+      attr_reader :config_files
+      attr_reader :config_secret
+
+      def initialize(config_root)
+        @config_root = config_root
+        @config_files = ["#{@config_root}/rubber.yml"]
+        @config_files += Dir["#{@config_root}/rubber-*.yml"].sort
+
+        # add a config file for current env only so that you can override
+        #things for specific envs
+        @config_files -= Dir["#{@config_root}/rubber-*-env.yml"]
+        env_yml = "#{@config_root}/rubber-#{Rubber.env}-env.yml"
+        @config_files << env_yml if File.exist?(env_yml)
+        
+        @items = {}
+        @config_files.each { |file| read_config(file) }
+        @config_secret = bind().rubber_secret
+        read_config(@config_secret) if @config_secret
+      end
+      
+      def read_config(file)
+        Rubber.logger.debug{"Reading rubber configuration from #{file}"}
+        if File.exist?(file)
+          @items = Environment.combine(@items, YAML.load_file(file) || {})
+        end
+      end
+
+      def known_roles
+        roles_dir = File.join(@config_root, "role")
+        roles = Dir.entries(roles_dir)
+        roles.delete_if {|d| d =~ /(^\..*)/}
+        roles += @items['roles'].keys
+        return roles.compact.uniq
+      end
+
+      def current_host
+        Socket::gethostname.gsub(/\..*/, '')
+      end
+      
+      def current_full_host
+        Socket::gethostname
+      end
+      
+      def bind(roles = nil, host = nil)
+        BoundEnv.new(@items, roles, host)
+      end
+
+      # combine old and new into a single value:
+      # non-nil wins if other is nil
+      # arrays just get unioned
+      # hashes also get unioned, but the values of conflicting keys get combined
+      # All else, the new value wins
+      def self.combine(old, new)
+        return old if new.nil?
+        return new if old.nil?
+        value = old
+        if old.is_a?(Hash) && new.is_a?(Hash)
+          value = old.clone
+          new.each do |nk, nv|
+            value[nk] = combine(value[nk], nv)
+          end
+        elsif old.is_a?(Array) && new.is_a?(Array)
+          value = old | new
+        else
+          value = new
+        end
+        return value
+      end
+
+      class HashValueProxy < Hash
+        attr_reader :global
+
+        def initialize(global, receiver)
+          @global = global
+          super()
+          replace(receiver)
+        end
+
+        def rubber_instances
+          @rubber_instances ||= Rubber::Configuration::rubber_instances
+        end
+
+        def [](name)
+          value = super(name)
+          value = global[name] if global && !value
+          return expand(value)
+        end
+
+        def each
+          each_key do |key|
+            yield key, self[key]
+          end
+        end
+
+        def method_missing(method_id)
+          key = method_id.id2name
+          return self[key]
+        end
+
+        def expand_string(val)
+          while val =~ /\#\{[^\}]+\}/
+            val = eval('%Q{' + val + '}', binding)
+          end
+          val = true if val =="true"
+          val = false if val == "false"
+          return val
+        end
+
+        def expand(value)
+          val = case value
+            when Hash
+              HashValueProxy.new(global || self, value)
+            when String
+              expand_string(value)
+            when Enumerable
+              value.collect {|v| expand(v) }
+            else
+              value
+          end
+          return val
+        end
+
+      end
+
+      class BoundEnv < HashValueProxy
+        attr_reader :roles
+        attr_reader :host
+
+        def initialize(global, roles, host)
+          @roles = roles
+          @host = host
+          bound_global = bind_config(global)
+          super(nil, bound_global)
+        end
+
+        def full_host
+          @full_host ||= "#{host}.#{domain}" rescue nil
+        end
+
+        # Forces role/host overrides into config
+        def bind_config(global)
+          global = global.clone()
+          role_overrides = global.delete("roles") || {}
+          host_overrides = global.delete("hosts") || {}
+          Array(roles).each do |role|
+            Array(role_overrides[role]).each do |k, v|
+              global[k] = Environment.combine(global[k], v)
+            end
+          end
+          Array(host_overrides[host]).each do |k, v|
+            global[k] = Environment.combine(global[k], v)
+          end
+          return global
+        end
+        
+        def method_missing(method_id)
+          key = method_id.id2name
+          return self[key]
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/rubber/generator.rb

@@ -0,0 +1,209 @@
+require 'erb'
+require 'find'
+require 'fileutils'
+
+module Rubber
+  module Configuration
+
+    # Handles selection and transformation of a set of config files
+    # based on the host/role they belong to
+    class Generator
+      attr_accessor :file_pattern
+      attr_accessor :no_post
+      attr_accessor :force
+      attr_accessor :fake_root
+      attr_accessor :stop_on_error_cmd
+
+      def initialize(config_dir, roles, host, options={})
+        @config_dir = config_dir
+        @roles = roles.to_a.reverse #First roles take precedence
+        @host = host || 'no_host'
+        @options=options
+      end
+
+      def run
+        config_dirs = []
+        config_dirs << "#{@config_dir}/common/**/**"
+        @roles.sort.each {|role| config_dirs <<  "#{@config_dir}/role/#{role}/**/**" }
+        config_dirs << "#{@config_dir}/host/#{@host}/**/**"
+
+        pat = Regexp.new(file_pattern) if file_pattern
+        
+        config_dirs.each do |dir|
+          Dir[dir].sort.each do |f|
+            next if f =~ /\/(CVS|\.svn)\//
+            if File.file?(f) && (! pat || pat.match(f))
+              Rubber.logger.info{"Transforming #{f}"}
+              begin
+                transform(IO.read(f), @options)
+              rescue Exception => e
+                lines = e.backtrace.grep(/^\(erb\):([0-9]+)/) {|b| Regexp.last_match(1) }
+                Rubber.logger.error{"Transformation failed for #{f}#{':' + lines.first if lines.first}"}
+                Rubber.logger.error e.message
+                exit 1
+              end
+            end
+          end
+        end
+      end
+
+      # Transforms the ERB template given in srcfile and writes the result to
+      # dest_file (if not nil) before returning it
+      def transform(src_data, options={})
+        config = ConfigDescriptor.new
+        config.generator = self
+
+        # for development/test, if we have a fake root, echo any
+        # calls to system
+        if fake_root
+          class << config
+            def system(*args)
+              puts ("Not running system command during a fake_root transformation: #{args.inspect}")
+            end
+            def open(*args)
+              if args.first && args.first =~ /^|/
+                puts ("Not running open/pipe command during a fake_root transformation: #{args.inspect}")
+              else
+                super
+              end
+            end
+            alias ` system
+            alias exec system
+            alias fork system
+          end
+        end
+
+        config.options = options
+        template = ERB.new(src_data, nil, "-")
+        result = template.result(config.get_binding())
+        
+        return if config.skip
+
+        config_path = config.path
+
+        # for development/test, if we have a fake root, then send config
+        # output there, and also put write_cmd output there
+        if fake_root
+          config_path = "write_cmd_" + config.write_cmd.gsub(/[^a-z0-9_-]/i, '') if config.write_cmd
+          config_path = "#{fake_root}/#{config_path}" if config_path
+        end
+
+        if ! config_path && ! (config.read_cmd && config.write_cmd)
+          raise "Transformation requires either a output filename or command"
+        end
+
+        reader = config_path || "|#{config.read_cmd}"
+        orig = IO.read(reader) rescue ""
+
+        # When additive is set we need to only replace between our delimiters
+        if config.additive
+          additive = ["# start rubber #{@host}", "# end rubber #{@host}"] unless additive.is_a? Array
+          pat = /#{config.additive[0]}.*#{config.additive[1]}/m
+          new = "#{config.additive[0]}#{result}#{config.additive[1]}"
+          if orig =~ pat
+            result = orig.gsub(pat, new)
+          else
+            result = orig + new + "\n"
+          end
+        end
+
+        # Only do something if the transformed result is different than what
+        # is currently in the destination file
+        if orig != result || force
+          # create dirs as needed
+          FileUtils.mkdir_p(File.dirname(config_path)) if config_path
+
+          # Write a backup of original
+          open("#{config_path}.bak", 'w') { |f| f.write(orig) } if config_path && config.backup
+
+          # Write out transformed file
+          writer = config_path || "|#{config.write_cmd}"
+          open(writer, 'w') do |pipe|
+            pipe.write(result)
+          end
+          if config.write_cmd && ! fake_root && $?.exitstatus != 0
+            raise "Config command failed execution:  #{config.write_cmd}"
+          end
+
+          # Set file permissions and owner if needed
+          FileUtils.chmod(config.perms, config_path) if config.perms && config_path
+          FileUtils.chown(config.owner, config.group, config_path) if config_path && (config.owner || config.group)
+
+          # Run post transform command if needed
+          if config.post
+            if fake_root
+              Rubber.logger.info("Not running post command as a fake root was given: #{config.post}")
+            elsif no_post
+              Rubber.logger.info("Not running post command as no post specified")
+            else
+              if orig != result || force
+                # this lets us abort a script if a command in the middle of it errors out
+                # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
+                config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
+
+                Rubber.logger.info{"Transformation executing post config command: #{config.post}"}
+                Rubber.logger.info `#{config.post}`
+                if $?.exitstatus != 0
+                  raise "Post command failed execution:  #{config.post}"
+                end
+              else
+                Rubber.logger.info("Nothing to do, not running post command")
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+    # Instances of this object are used accept settings from with
+    # a config file for when it is transformed by Generator
+    class ConfigDescriptor
+      # The output path to write the transformed config file to
+      attr_accessor :path
+      # The command to use for reading the original config file from (e.g. "crontab -l")
+      attr_accessor :read_cmd
+      # The command to use for piping the transformed config file to (e.g. "crontab -")
+      attr_accessor :write_cmd
+      # The command to run after generating the config file if it has changed
+      attr_accessor :post
+      # The owner the output file should have, e.g. "root"
+      attr_accessor :owner
+      # The group the output file should have, e.g. "system"
+      attr_accessor :group
+      # The permissions the output file should have, e.g. 0644 (octal, leading zero is significant)
+      attr_accessor :perms
+      # Sets transformation to be additive, only replaces between given delimiters, e/g/ additive = ["## start", "## end"]
+      attr_accessor :additive
+      # Lets one dynamically determine if a given file gets skipped during transformation
+      attr_accessor :skip
+      # Backup file when transforming, defaults to true, set to false to prevent backup
+      attr_accessor :backup
+      # use sudo to write the output file
+      # attr_accessor :sudo
+      # options passed in through code
+      attr_accessor :options
+
+      # allow access to calling generator so can determine stuff like fake_root
+      attr_accessor :generator
+
+      def initialize
+        @backup = true
+      end
+      
+      def get_binding
+        binding
+      end
+    
+      def rubber_env()
+        Rubber::Configuration.rubber_env
+      end
+
+      def rubber_instances()
+        Rubber::Configuration.rubber_instances
+      end
+
+    end
+
+  end
+end

data/lib/rubber/instance.rb

@@ -0,0 +1,176 @@
+require 'yaml'
+
+module Rubber
+  module Configuration
+
+    # Contains the ec2 instance configuration defined in instance.yml
+    #
+    class Instance
+      attr_reader :file, :artifacts
+      include Enumerable
+
+      def initialize(file)
+        Rubber.logger.debug{"Reading rubber instances from #{file}"}
+        @file = file
+        @items = {}
+        @artifacts = {'volumes' => {}, 'static_ips' => {}}
+        if ENV['FILTER']
+          @filters = ENV['FILTER'].split(/\s*,\s*/)
+        end
+
+        if File.exist?(@file)
+          item_list = File.open(@file) { |f| YAML.load(f) }
+          if item_list
+            item_list.each do |i|
+              if i.is_a? InstanceItem
+                @items[i.name] = i
+              elsif i.is_a? Hash
+                @artifacts.merge!(i)
+              end
+            end
+          end
+        end
+      end
+
+      def save()
+        data = []
+        data.push(*@items.values)
+        data.push(@artifacts)
+        File.open(@file, "w") { |f| f.write(YAML.dump(data)) }
+      end
+
+      def [](name)
+        @items[name] || @items[name.gsub(/\..*/, '')]
+      end
+
+      # gets the instances for the given role.  If options is nil, all roles
+      # match, otherwise the role has to have options that match exactly
+      def for_role(role_name, options=nil)
+        @items.values.find_all {|ic| ic.roles.any? {|r| r.name == role_name && (! options || r.options == options)}}
+      end
+
+      def filtered()
+        @items.values.find_all {|ic| ! @filters || @filters.include?(ic.name)}
+      end
+
+      def all_roles()
+        @items.collect {|n, i| i.role_names}.flatten.uniq
+      end
+
+      def add(instance_item)
+        @items[instance_item.name] = instance_item
+      end
+
+      def remove(name)
+        @items.delete(name)
+      end
+
+      def each(&block)
+        @items.values.each &block
+      end
+      
+      def size
+        @items.size
+      end
+    end
+
+    # The configuration for a single instance
+    class InstanceItem
+      attr_reader :name, :domain, :instance_id, :security_groups
+      attr_accessor :roles, :zone
+      attr_accessor :external_host, :external_ip
+      attr_accessor :internal_host, :internal_ip
+      attr_accessor :static_ip, :volumes, :partitions, :root_device_type
+      attr_accessor :spot_instance_request_id
+      attr_accessor :platform
+
+      def initialize(name, domain, roles, instance_id, security_group_list=[])
+        @name = name
+        @domain = domain
+        @roles = roles
+        @instance_id = instance_id
+        @security_groups = security_group_list
+      end
+
+      def full_name
+        "#@name.#@domain"
+      end
+
+      def role_names()
+        roles.collect {|r| r.name}
+      end
+
+      def platform
+        # Deal with old instance configurations that don't have a platform value persisted.
+        @platform || 'linux'
+      end
+
+      def windows?
+        platform == 'windows'
+      end
+    end
+
+    # The configuration for a single role contained in the list
+    # of roles in InstanceItem
+    class RoleItem
+      attr_reader :name, :options
+
+      def self.expand_role_dependencies(roles, dependency_map, expanded=[])
+        roles = Array(roles)
+        roles.each do |role|
+          unless expanded.include?(role)
+            expanded << role
+            needed = dependency_map[role]
+            expand_role_dependencies(needed, dependency_map, expanded)
+          end
+        end
+        return expanded
+      end
+
+      def self.parse(str)
+        data = str.split(':');
+        role = Rubber::Configuration::RoleItem.new(data[0])
+        if data[1]
+          data[1].split(';').each do |pair|
+            p = pair.split('=')
+            val = case p[1]
+                    when 'true' then true
+                    when 'false' then false
+                    else p[1] end
+            role.options[p[0]] = val
+          end
+        end
+        return role
+      end
+
+      def to_s
+        str = @name
+        @options.each_with_index do |kv, i|
+          str += (i == 0 ? ':' : ';')
+          str += "#{kv[0]}=#{kv[1]}"
+        end
+        return str
+      end
+
+      def initialize(name, options={})
+        @name = name
+        @options = options || {}
+      end
+
+      def eql?(rhs)
+        rhs && @name == rhs.name && @options == rhs.options
+      end
+      alias == eql?
+
+      def hash()
+        @name.hash
+      end
+
+      def <=>(rhs)
+        return @name <=> rhs.name
+      end
+    end
+
+  end
+end
+