sml-rubber 0.9.13 → 1.5.5

data/lib/rubber/environment.rb

@@ -14,7 +14,15 @@ module Rubber
 
       def initialize(config_root)
         @config_root = config_root
-        @config_files = ["#{@config_root}/rubber.yml"] + Dir["#{@config_root}/rubber-*.yml"].sort
+        @config_files = ["#{@config_root}/rubber.yml"]
+        @config_files += Dir["#{@config_root}/rubber-*.yml"].sort
+
+        # add a config file for current env only so that you can override
+        #things for specific envs
+        @config_files -= Dir["#{@config_root}/rubber-*-env.yml"]
+        env_yml = "#{@config_root}/rubber-#{Rubber.env}-env.yml"
+        @config_files << env_yml if File.exist?(env_yml)
+        
         @items = {}
         @config_files.each { |file| read_config(file) }
         @config_secret = bind().rubber_secret
@@ -22,7 +30,7 @@ module Rubber
       end
       
       def read_config(file)
-        LOGGER.debug{"Reading rubber configuration from #{file}"}
+        Rubber.logger.debug{"Reading rubber configuration from #{file}"}
         if File.exist?(file)
           @items = Environment.combine(@items, YAML.load_file(file) || {})
         end
@@ -32,6 +40,8 @@ module Rubber
         roles_dir = File.join(@config_root, "role")
         roles = Dir.entries(roles_dir)
         roles.delete_if {|d| d =~ /(^\..*)/}
+        roles += @items['roles'].keys
+        return roles.compact.uniq
       end
 
       def current_host
@@ -41,7 +51,7 @@ module Rubber
       def current_full_host
         Socket::gethostname
       end
-
+      
       def bind(roles = nil, host = nil)
         BoundEnv.new(@items, roles, host)
       end
@@ -77,6 +87,10 @@ module Rubber
           replace(receiver)
         end
 
+        def rubber_instances
+          @rubber_instances ||= Rubber::Configuration::rubber_instances
+        end
+
         def [](name)
           value = super(name)
           value = global[name] if global && !value
@@ -122,14 +136,16 @@ module Rubber
       class BoundEnv < HashValueProxy
         attr_reader :roles
         attr_reader :host
-        attr_reader :full_host
 
         def initialize(global, roles, host)
           @roles = roles
           @host = host
           bound_global = bind_config(global)
           super(nil, bound_global)
-          @full_host = "#{host}.#{domain}" rescue nil
+        end
+
+        def full_host
+          @full_host ||= "#{host}.#{domain}" rescue nil
         end
 
         # Forces role/host overrides into config
@@ -137,14 +153,14 @@ module Rubber
           global = global.clone()
           role_overrides = global.delete("roles") || {}
           host_overrides = global.delete("hosts") || {}
-          roles.to_a.each do |role|
-            role_overrides[role].each do |k, v|
+          Array(roles).each do |role|
+            Array(role_overrides[role]).each do |k, v|
               global[k] = Environment.combine(global[k], v)
-            end if role_overrides[role]
+            end
           end
-          host_overrides[host].each do |k, v|
+          Array(host_overrides[host]).each do |k, v|
             global[k] = Environment.combine(global[k], v)
-          end if host_overrides[host]
+          end
           return global
         end
         

data/lib/rubber/generator.rb

@@ -33,13 +33,13 @@ module Rubber
           Dir[dir].sort.each do |f|
             next if f =~ /\/(CVS|\.svn)\//
             if File.file?(f) && (! pat || pat.match(f))
-              LOGGER.info{"Transforming #{f}"}
+              Rubber.logger.info{"Transforming #{f}"}
               begin
                 transform(IO.read(f), @options)
               rescue Exception => e
                 lines = e.backtrace.grep(/^\(erb\):([0-9]+)/) {|b| Regexp.last_match(1) }
-                LOGGER.error{"Transformation failed for #{f}#{':' + lines.first if lines.first}"}
-                LOGGER.error e.message
+                Rubber.logger.error{"Transformation failed for #{f}#{':' + lines.first if lines.first}"}
+                Rubber.logger.error e.message
                 exit 1
               end
             end
@@ -113,7 +113,7 @@ module Rubber
           FileUtils.mkdir_p(File.dirname(config_path)) if config_path
 
           # Write a backup of original
-          open("#{config_path}.bak", 'w') { |f| f.write(orig) } if config_path
+          open("#{config_path}.bak", 'w') { |f| f.write(orig) } if config_path && config.backup
 
           # Write out transformed file
           writer = config_path || "|#{config.write_cmd}"
@@ -130,21 +130,23 @@ module Rubber
 
           # Run post transform command if needed
           if config.post
-            if orig == result
-              LOGGER.info("Nothing to do, not running post command")
+            if fake_root
+              Rubber.logger.info("Not running post command as a fake root was given: #{config.post}")
             elsif no_post
-              LOGGER.info("Not running post command as no post specified")
-            elsif fake_root
-              LOGGER.info("Not running post command as a fake root was given: #{config.post}")
+              Rubber.logger.info("Not running post command as no post specified")
             else
-              # this lets us abort a script if a command in the middle of it errors out
-              # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
-              config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
-
-              LOGGER.info{"Transformation executing post config command: #{config.post}"}
-              LOGGER.info `#{config.post}`
-              if $?.exitstatus != 0
-                raise "Post command failed execution:  #{config.post}"
+              if orig != result || force
+                # this lets us abort a script if a command in the middle of it errors out
+                # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
+                config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
+
+                Rubber.logger.info{"Transformation executing post config command: #{config.post}"}
+                Rubber.logger.info `#{config.post}`
+                if $?.exitstatus != 0
+                  raise "Post command failed execution:  #{config.post}"
+                end
+              else
+                Rubber.logger.info("Nothing to do, not running post command")
               end
             end
           end
@@ -174,22 +176,28 @@ module Rubber
       attr_accessor :additive
       # Lets one dynamically determine if a given file gets skipped during transformation
       attr_accessor :skip
+      # Backup file when transforming, defaults to true, set to false to prevent backup
+      attr_accessor :backup
       # use sudo to write the output file
       # attr_accessor :sudo
       # options passed in through code
       attr_accessor :options
 
+      def initialize
+        @backup = true
+      end
+      
       def get_binding
         binding
-    end
+      end
     
-    def rubber_env()
-      Rubber::Configuration.rubber_env
-    end
-  
-    def rubber_instances()
-      Rubber::Configuration.rubber_instances
-    end
+      def rubber_env()
+        Rubber::Configuration.rubber_env
+      end
+
+      def rubber_instances()
+        Rubber::Configuration.rubber_instances
+      end
 
     end
 

data/lib/rubber/instance.rb

@@ -10,7 +10,7 @@ module Rubber
       include Enumerable
 
       def initialize(file)
-        LOGGER.debug{"Reading rubber instances from #{file}"}
+        Rubber.logger.debug{"Reading rubber instances from #{file}"}
         @file = file
         @items = {}
         @artifacts = {'volumes' => {}, 'static_ips' => {}}
@@ -81,6 +81,8 @@ module Rubber
       attr_accessor :external_host, :external_ip
       attr_accessor :internal_host, :internal_ip
       attr_accessor :static_ip, :volumes, :partitions
+      attr_accessor :spot_instance_request_id
+      attr_accessor :platform
 
       def initialize(name, domain, roles, instance_id, security_group_list=[])
         @name = name
@@ -97,6 +99,15 @@ module Rubber
       def role_names()
         roles.collect {|r| r.name}
       end
+
+      def platform
+        # Deal with old instance configurations that don't have a platform value persisted.
+        @platform || 'linux'
+      end
+
+      def windows?
+        platform == 'windows'
+      end
     end
 
     # The configuration for a single role contained in the list

data/lib/rubber/railtie.rb

@@ -0,0 +1,14 @@
+require 'rubber'
+require 'rails'
+
+module Rubber
+
+  class Railtie < Rails::Railtie
+
+    initializer "rubber.configure_rails_initialization" do
+      Rubber::initialize(Rails.root, Rails.env)
+    end
+
+  end
+
+end

data/lib/rubber/recipes/rubber.rb

@@ -10,7 +10,13 @@ require 'rubber'
 
 namespace :rubber do
 
-  # advise capistrano's task method so that tasks for non-existant roles don't
+  # Disable connecting to any Windows instance.
+  alias :original_task :task
+  def task(name, options={}, &block)
+    original_task(name, options.merge(:except => { :platform => 'windows' }), &block)
+  end
+
+  # advise capistrano's task method so that tasks for non-existent roles don't
   # fail when roles isn't defined due to using a FILTER for load_roles
   # If you have a task you need to execute even when there are no
   # roles, you have to use required_task instead of task - see rubber:create
@@ -19,7 +25,8 @@ namespace :rubber do
     class << ns
       alias :required_task :task
       def task(name, options={}, &block)
-        required_task(name, options) do
+        # Disable connecting to any Windows instance.
+        required_task(name, options.merge(:except => { :platform => 'windows' })) do
           # define empty roles for the case when a task has a role that we don't define anywhere
           [*options[:roles]].each do |r|
             roles[r] ||= []
@@ -39,16 +46,15 @@ namespace :rubber do
   on :load, "rubber:init"
     
   required_task :init do
-    # Require cap 2.4 since we depend on bugs that have been fixed
-    require 'capistrano/version'
-    if Capistrano::Version::MAJOR < 2 || Capistrano::Version::MINOR < 4
-      fatal "rubber requires capistrano 2.4.0 or greater"
-    end
-    
     set :rubber_cfg, Rubber::Configuration.get_configuration(RUBBER_ENV)
     set :rubber_env, rubber_cfg.environment.bind()
     set :rubber_instances, rubber_cfg.instance
 
+    # Disable connecting to any Windows instance.
+    # pass -l to bash in :shell to that run also gets full env
+    # use a pty so we don't get "stdin: is not a tty" error output
+    set :default_run_options, :pty => true, :shell => "/bin/bash -l", :except => { :platform => 'windows' }
+
     set :cloud, Rubber::Cloud::get_provider(rubber_env.cloud_provider || "aws", rubber_env, self)
 
     load_roles() unless rubber_env.disable_auto_roles
@@ -56,6 +62,7 @@ namespace :rubber do
     # the private key in the same folder, the public key should have the
     # extension ".pub".
     ssh_options[:keys] = rubber_env.cloud_providers[rubber_env.cloud_provider].key_file
+    ssh_options[:timeout] = fetch(:ssh_timeout, 5)
   end
 
 
@@ -73,7 +80,7 @@ namespace :rubber do
     # define capistrano host => role mapping for all instances
     rubber_instances.filtered.each do |ic|
       ic.roles.each do |role|
-        opts = Rubber::Util::symbolize_keys(role.options)
+        opts = Rubber::Util::symbolize_keys(role.options).merge(:platform => ic.platform)
         msg = "Auto role: #{role.name.to_sym} => #{ic.full_name}"
         msg << ", #{opts.inspect}" if opts.inspect.size > 0
         logger.info msg

data/lib/rubber/recipes/rubber/deploy.rb

@@ -59,8 +59,8 @@ namespace :rubber do
     Sets permissions of files in application directory to be owned by app_user.
   DESC
   task :setup_app_permissions do
-    run "find #{shared_path} -name cached-copy -prune -o -print | xargs chown #{rubber_env.app_user}:#{rubber_env.app_user}"
-    run "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{current_path}/tmp"
+    rsudo "find #{shared_path} -name cached-copy -prune -o -print | xargs chown #{rubber_env.app_user}:#{rubber_env.app_user}"
+    rsudo "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{current_path}/tmp"
   end
 
   def run_config(options={})
@@ -84,7 +84,7 @@ namespace :rubber do
       put(File.read(secret), File.join(path, base, File.basename(secret)), :mode => "+r")
     end
 
-    sudo "sh -c 'cd #{path} && #{extra_env} rake rubber:config'"
+    rsudo "cd #{path} && #{extra_env} rake rubber:config"
   end
 
 end

data/lib/rubber/recipes/rubber/instances.rb

@@ -5,7 +5,13 @@ namespace :rubber do
   DESC
   required_task :create do
     instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
-    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+
+    env = rubber_cfg.environment.bind(nil, instance_alias)
+    default_roles = env.instance_roles
+    r = get_env("ROLES", "Instance roles (e.g. web,app,db:primary=true)", true, default_roles)
+
+    create_spot_instance = ENV.delete("SPOT_INSTANCE") || env.cloud_providers[env.cloud_provider].spot_instance
+
     if r == '*'
       instance_roles = rubber_cfg.environment.known_roles
       instance_roles = instance_roles.collect {|role| role == "db" ? "db:primary=true" : role }
@@ -29,7 +35,7 @@ namespace :rubber do
     # Add in roles that the given set of roles depends on
     ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
 
-    create_instance(instance_alias, ir)
+    create_instance(instance_alias, ir, create_spot_instance)
   end
 
   desc <<-DESC
@@ -83,6 +89,10 @@ namespace :rubber do
 
     instance.roles = (instance.roles + ir).uniq
     rubber_instances.save()
+    logger.info "Roles for #{instance_alias} are now:"
+    logger.info instance.role_names.sort.join("\n")
+    logger.info ''
+    logger.info "Run 'cap rubber:bootstrap' if done adding roles"
   end
 
   desc <<-DESC
@@ -105,6 +115,8 @@ namespace :rubber do
 
     instance.roles = (instance.roles - ir).uniq
     rubber_instances.save()
+    logger.info "Roles for #{instance_alias} are now:"
+    logger.info instance.role_names.sort.join("\n")
   end
 
   desc <<-DESC
@@ -144,7 +156,7 @@ namespace :rubber do
 
   # Creates a new ec2 instance with the given alias and roles
   # Configures aliases (/etc/hosts) on local and remote machines
-  def create_instance(instance_alias, instance_roles)
+  def create_instance(instance_alias, instance_roles, create_spot_instance=false)
     fatal "Instance already exists: #{instance_alias}" if rubber_instances[instance_alias]
 
     role_names = instance_roles.collect{|x| x.name}
@@ -157,12 +169,45 @@ namespace :rubber do
     ami = env.cloud_providers[env.cloud_provider].image_id
     ami_type = env.cloud_providers[env.cloud_provider].image_type
     availability_zone = env.availability_zone
-    logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
-    instance_id = cloud.create_instance(ami, ami_type, security_groups, availability_zone)
+
+    if create_spot_instance
+      spot_price = env.cloud_providers[env.cloud_provider].spot_price.to_s
+
+      logger.info "Creating spot instance request for instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
+      request_id = cloud.create_spot_instance_request(spot_price, ami, ami_type, security_groups, availability_zone)
+
+      print "Waiting for spot instance request to be fulfilled"
+      max_wait_time = env.cloud_providers[env.cloud_provider].spot_instance_request_timeout || (1.0 / 0) # Use the specified timeout value or default to infinite.
+      instance_id = nil
+      while instance_id.nil? do
+        print "."
+        sleep 2
+        max_wait_time -= 2
+
+        request = cloud.describe_spot_instance_requests(request_id).first
+        instance_id = request[:instance_id]
+
+        if max_wait_time < 0 && instance_id.nil?
+          cloud.destroy_spot_instance_request(request[:id])
+
+          print "\n"
+          print "Failed to fulfill spot instance in the time specified. Falling back to on-demand instance creation."
+          break
+        end
+      end
+
+      print "\n"
+    end
+
+    if !create_spot_instance || (create_spot_instance && max_wait_time < 0)
+      logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
+      instance_id = cloud.create_instance(ami, ami_type, security_groups, availability_zone)
+    end
 
     logger.info "Instance #{instance_id} created"
 
     instance_item = Rubber::Configuration::InstanceItem.new(instance_alias, env.domain, instance_roles, instance_id, security_groups)
+    instance_item.spot_instance_request_id = request_id if create_spot_instance
     rubber_instances.add(instance_item)
     rubber_instances.save()
 
@@ -180,8 +225,12 @@ namespace :rubber do
         instance_item.external_ip = instance[:external_ip]
         instance_item.internal_host = instance[:internal_host]
         instance_item.zone = instance[:zone]
+        instance_item.platform = instance[:platform]
         rubber_instances.save()
 
+        # turn back on root ssh access if we are using root as the capistrano user for connecting
+        enable_root_ssh(instance_item.external_ip, fetch(:initial_ssh_user, 'ubuntu')) if (user == 'root' && ! instance_item.windows?)
+
         # setup amazon elastic ips if configured to do so
         setup_static_ips
 
@@ -196,7 +245,14 @@ namespace :rubber do
         # so that we can update all aliases
 
         task :_get_ip, :hosts => instance_item.external_ip do
-          instance_item.internal_ip = capture(print_ip_command).strip
+          # There's no good way to get the internal IP for a Windows host, so just set it to the external
+          # and let the router handle mapping to the internal network.
+          if instance_item.windows?
+            instance_item.internal_ip = instance_item.external_ip
+          else
+            instance_item.internal_ip = capture(print_ip_command).strip
+          end
+
           rubber_instances.save()
         end
 
@@ -236,6 +292,10 @@ namespace :rubber do
       instance_item.external_ip = instance[:external_ip]
       instance_item.internal_host = instance[:internal_host]
       instance_item.zone = instance[:zone]
+      instance_item.platform = instance[:platform]
+
+      # turn back on root ssh access if we are using root as the capistrano user for connecting
+      enable_root_ssh(instance_item.external_ip, fetch(:initial_ssh_user, 'ubuntu')) if user == 'root'
 
       # setup amazon elastic ips if configured to do so
       setup_static_ips
@@ -250,8 +310,13 @@ namespace :rubber do
       # Connect to newly created instance and grab its internal ip
       # so that we can update all aliases
       task :_get_ip, :hosts => instance_item.external_ip do
-        instance_item.internal_ip = capture(print_ip_command).strip
-        rubber_instances.save()
+        # There's no good way to get the internal IP for a Windows host, so just set it to the external
+        # and let the router handle mapping to the internal network.
+        if instance_item.windows?
+          instance_item.internal_ip = instance_item.external_ip
+        else
+          instance_item.internal_ip = capture(print_ip_command).strip
+        end
       end
 
       # even though instance is running, sometimes ssh hasn't started yet,