rubber 1.15.0 → 2.0.0.pre1

data/lib/rubber/environment.rb

@@ -42,13 +42,26 @@ module Rubber
       end
 
       def known_roles
-        roles_dir = File.join(@config_root, "role")
-        roles = Dir.entries(roles_dir)
-        roles.delete_if {|d| d =~ /(^\..*)/}
-        roles += @items['roles'].keys
-        return roles.compact.uniq
+        return @known_roles if @known_roles
+        
+        roles = []
+        # all the roles known about in config directory
+        roles.concat Dir["#{@config_root}/role/*"].collect {|f| File.basename(f) }
+        
+        # all the roles known about in script directory
+        roles.concat Dir["#{Rubber.root}/script/*/role/*"].collect {|f| File.basename(f) }
+        
+        # all the roles known about in yml files
+        Dir["#{@config_root}/rubber*.yml"].each do |yml|
+          rubber_yml = YAML.load(File.read(yml)) rescue {}
+          roles.concat(rubber_yml['roles'].keys) rescue nil
+          roles.concat(rubber_yml['role_dependencies'].keys) rescue nil
+          roles.concat(rubber_yml['role_dependencies'].values) rescue nil
+        end
+        
+        @known_roles = roles.flatten.uniq.sort
       end
-
+      
       def current_host
         Socket::gethostname.gsub(/\..*/, '')
       end
@@ -107,6 +120,12 @@ module Rubber
             yield key, self[key]
           end
         end
+        
+        # allows expansion when to_a gets called on hash proxy, e.g. when wrapping
+        # a var in Array() to ensure error free iteration for possible null values
+        def to_a
+          self.collect {|k, v| [k, v]}
+        end
 
         def method_missing(method_id)
           key = method_id.id2name

data/lib/rubber/generator.rb

@@ -1,6 +1,7 @@
 require 'erb'
 require 'find'
 require 'fileutils'
+require 'rubber/util'
 
 module Rubber
   module Configuration
@@ -159,6 +160,7 @@ module Rubber
     # Instances of this object are used accept settings from with
     # a config file for when it is transformed by Generator
     class ConfigDescriptor
+      
       # The output path to write the transformed config file to
       attr_accessor :path
       # The command to use for reading the original config file from (e.g. "crontab -l")

data/lib/rubber/instance.rb

@@ -11,10 +11,12 @@ module Rubber
       include Enumerable
       include MonitorMixin
 
-      def initialize(file)
+      def initialize(instance_storage, opts={})
         super()
-        Rubber.logger.debug{"Reading rubber instances from #{file}"}
-        @file = file
+        
+        @instance_storage = instance_storage
+        @opts = opts
+      
         @items = {}
         @artifacts = {'volumes' => {}, 'static_ips' => {}}
 
@@ -26,29 +28,108 @@ module Rubber
         @filter_roles, @filter_roles_negated = @filter_roles.partition {|f| f !~ /^-/ }
         @filter_roles_negated = @filter_roles_negated.collect {|f| f[1..-1] }
 
-        if File.exist?(@file)
-          item_list = File.open(@file) { |f| YAML.load(f) }
-          if item_list
-            item_list.each do |i|
-              if i.is_a? InstanceItem
-                @items[i.name] = i
-              elsif i.is_a? Hash
-                @artifacts.merge!(i)
-              end
+        load()
+      end
+      
+      def load(instance_storage=@instance_storage)
+        case instance_storage
+          when /file:(.*)/
+            location = $1
+            File.open(location, 'r') {|f| load_from_file(f) } if File.exist?(location)
+          when /storage:(.*)/
+            location = $1
+            bucket = location.split("/")[0]
+            key = location.split("/")[1..-1].join("/")
+            data = Rubber.cloud.storage(bucket).fetch(key)
+            StringIO.open(data, 'r') {|f| load_from_file(f) }
+          when /table:(.*)/
+            location = $1
+            load_from_table(location)
+          else
+            raise "Invalid instance_storage: #{instance_storage}\n" +
+                "Must be one of file:, table:, storage:"
+        end
+      end
+
+      def load_from_file(io)
+        item_list =  YAML.load(io.read)
+        if item_list
+          item_list.each do |i|
+            if i.is_a? InstanceItem
+              @items[i.name] = i
+            elsif i.is_a? Hash
+              @artifacts.merge!(i)
             end
           end
         end
       end
-
-      def save()
+      
+      def load_from_table(table_key)
+        Rubber.logger.debug{"Reading rubber instances from cloud table #{table_key}"}
+        store = Rubber.cloud.table_store(table_key)
+        items = store.find()
+        items.each do |name, data|
+          case name
+            when '_artifacts_'
+              @artifacts = data
+            else
+              ic = InstanceItem.from_hash(data.merge({'name' => name}))
+              @items[ic.name] = ic 
+          end
+        end
+      end
+      
+      def save(instance_storage=@instance_storage, backup=@opts[:backup])
         synchronize do
-          data = []
-          data.push(*@items.values)
-          data.push(@artifacts)
-          File.open(@file, "w") { |f| f.write(YAML.dump(data)) }
+          case instance_storage
+            when /file:(.*)/
+              location = $1
+              File.open(location, 'w') {|f| save_to_file(f) }
+            when /storage:(.*)/
+              location = $1
+              bucket = location.split("/")[0]
+              key = location.split("/")[1..-1].join("/")
+              data = StringIO.open {|f| save_to_file(f); f.string }
+              Rubber.cloud.storage(bucket).store(key, data)
+            when /table:(.*)/
+              location = $1
+              save_to_table(location)
+            else
+              raise "Invalid instance_storage: #{instance_storage}\n" +
+                  "Must be one of file:, table:, storage:"
+          end
         end
+        
+        save(backup, false) if backup
       end
 
+      def save_to_file(io)
+        data = []
+        data.push(*@items.values)
+        data.push(@artifacts)
+        io.write(YAML.dump(data))
+      end
+      
+      def save_to_table(table_key)
+        store = Rubber.cloud.table_store(table_key)
+        
+        # delete all before writing to handle removals
+        store.find().each do |k, v|
+          store.delete(k)
+        end
+        
+        # only write out non-empty artifacts
+        artifacts = @artifacts.select {|k, v| v.size > 0}
+        if artifacts.size > 0
+          store.put('_artifacts_', artifacts)
+        end
+        
+        # write out all the instance data
+        @items.values.each do |item|
+          store.put(item.name, item.to_hash)
+        end
+      end
+      
       def [](name)
         @items[name] || @items[name.gsub(/\..*/, '')]
       end
@@ -132,6 +213,27 @@ module Rubber
         @security_groups = security_group_list
       end
 
+      def self.from_hash(hash)
+        item = allocate
+        hash.each do |k, v|
+          sym = "@#{k}".to_sym
+          v = v.collect {|r| RoleItem.parse(r) } if k == 'roles'
+          item.instance_variable_set(sym, v)
+        end
+        return item
+      end
+      
+      def to_hash
+        hash = {}
+        instance_variables.each do |iv|
+          name = iv.to_s.gsub(/^@/, '')
+          value = instance_variable_get(iv)
+          value = value.collect {|r| r.to_s } if name == 'roles'
+          hash[name] = value
+        end
+        return hash
+      end
+      
       def full_name
         "#@name.#@domain"
       end
@@ -175,7 +277,7 @@ module Rubber
       end
 
       def self.parse(str)
-        data = str.split(':');
+        data = str.split(':')
         role = Rubber::Configuration::RoleItem.new(data[0])
         if data[1]
           data[1].split(';').each do |pair|

data/lib/rubber/recipes/rubber.rb

@@ -46,7 +46,7 @@ namespace :rubber do
   on :load, "rubber:init"
     
   required_task :init do
-    set :rubber_cfg, Rubber::Configuration.get_configuration(RUBBER_ENV)
+    set :rubber_cfg, Rubber::Configuration.get_configuration(Rubber.env)
     set :rubber_env, rubber_cfg.environment.bind()
     set :rubber_instances, rubber_cfg.instance
 
@@ -57,14 +57,13 @@ namespace :rubber do
     default_run_options[:shell] = "/bin/bash -l"
     default_run_options[:except] = { :platform => 'windows' }
 
-    # sharing a Net::HTTP instance across threads doesn't work, so create a new instance per thread
-    set :cloud, Rubber::ThreadSafeProxy.new { Rubber::Cloud::get_provider(rubber_env.cloud_provider || "aws", rubber_env, self) }
+    set :cloud, Rubber.cloud(self)
 
     load_roles() unless rubber_env.disable_auto_roles
     # NOTE: for some reason Capistrano requires you to have both the public and
     # the private key in the same folder, the public key should have the
     # extension ".pub".
-    ssh_options[:keys] = rubber_env.cloud_providers[rubber_env.cloud_provider].key_file
+    ssh_options[:keys] = cloud.env.key_file
     ssh_options[:timeout] = fetch(:ssh_timeout, 5)
   end
 

data/lib/rubber/recipes/rubber/deploy.rb

@@ -38,12 +38,9 @@ namespace :rubber do
   DESC
   task :config do
     opts = {}
-    opts['NO_POST'] = true if ENV['NO_POST']
-    opts['FILE'] = ENV['FILE'] if ENV['FILE']
-    opts['RUBBER_ENV'] = RUBBER_ENV
-    # we need to set rails env as well because when running rake
-    # in a rails project, rails gets loaded before the rubber hook gets run
-    opts['RAILS_ENV'] = RUBBER_ENV
+    opts[:no_post] = true if ENV['NO_POST']
+    opts[:force] = true if ENV['FORCE']
+    opts[:file] = ENV['FILE'] if ENV['FILE']
 
     # when running deploy:migrations, we need to run config against release_path
     opts[:deploy_path] = current_release if fetch(:migrate_target, :current).to_sym == :latest
@@ -65,26 +62,29 @@ namespace :rubber do
 
   def run_config(options={})
     path = options.delete(:deploy_path) || current_path
-    extra_env = options.keys.inject("") {|all, k|  "#{all} #{k}=\"#{options[k]}\""}
+    opts = ""
+    opts += " --no_post" if options[:no_post]
+    opts += " --force" if options[:force]
+    opts += " --file=#{options[:file]}" if options[:file]
 
     # Need to do this so we can work with staging instances without having to
     # checkin instance file between create and bootstrap, as well as during a deploy
     if fetch(:push_instance_config, false)
       push_files = [rubber_instances.file] + rubber_cfg.environment.config_files
       push_files.each do |file|
-        dest_file = file.sub(/^#{RUBBER_ROOT}\/?/, '')
+        dest_file = file.sub(/^#{Rubber.root}\/?/, '')
         put(File.read(file), File.join(path, dest_file), :mode => "+r")
       end
     end
 
-    # if the user has defined a secret config file, then push it into RUBBER_ROOT/config/rubber
+    # if the user has defined a secret config file, then push it into Rubber.root/config/rubber
     secret = rubber_cfg.environment.config_secret
     if secret && File.exist?(secret)
-      base = rubber_cfg.environment.config_root.sub(/^#{RUBBER_ROOT}\/?/, '')
+      base = rubber_cfg.environment.config_root.sub(/^#{Rubber.root}\/?/, '')
       put(File.read(secret), File.join(path, base, File.basename(secret)), :mode => "+r")
     end
-
-    rsudo "cd #{path} && #{extra_env} #{fetch(:rake, 'rake')} rubber:config"
+    
+    rsudo "cd #{path} && RUBBER_ENV=#{Rubber.env} RAILS_ENV=#{Rubber.env} ./script/rubber config #{opts}"
   end
 
 end

data/lib/rubber/recipes/rubber/instances.rb

@@ -21,8 +21,7 @@ namespace :rubber do
     create_spot_instance = ENV.delete("SPOT_INSTANCE")
 
     if r == '*'
-      instance_roles = rubber_cfg.environment.known_roles
-      instance_roles = instance_roles.collect {|role| role == "db" ? "db:primary=true" : role }
+      instance_roles = rubber_cfg.environment.known_roles.reject {|r| r =~ /slave/ || r =~ /^db$/ }
     else
       instance_roles = r.split(",")
     end
@@ -53,7 +52,7 @@ namespace :rubber do
     aliases = Rubber::Util::parse_aliases(instance_aliases)
 
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    destroy_instances(aliases, ENV['FORCE'] == 'true')
+    destroy_instances(aliases, ENV['FORCE'] =~ /^(t|y)/)
   end
 
   desc <<-DESC
@@ -61,7 +60,7 @@ namespace :rubber do
   DESC
   required_task :destroy_all do
     rubber_instances.each do |ic|
-      destroy_instance(ic.name, ENV['FORCE'] == 'true')
+      destroy_instance(ic.name, ENV['FORCE'] =~ /^(t|y)/)
     end
   end
 
@@ -150,15 +149,21 @@ namespace :rubber do
   DESC
   required_task :describe do
     results = []
-    format = "%-10s %-10s %-10s %-15s %-30s"
-    results << format % %w[InstanceID State Zone IP Alias\ (*=unknown)]
+    format = "%-10s %-10s %-10s %-10s %-15s %-30s"
+    results << format % %w[InstanceID Type State Zone IP Alias\ (*=unknown)]
 
     instances = cloud.describe_instances()
+    data = []
     instances.each do |instance|
       local_alias = find_alias(instance[:external_ip], instance[:id], instance[:state] == 'running')
-      results << format % [instance[:id], instance[:state], instance[:zone], instance[:external_ip] || "NoIP", local_alias || "Unknown"]
+      data << [instance[:id], instance[:type], instance[:state], instance[:zone], instance[:external_ip] || "NoIP", local_alias || "Unknown"]
     end
-    results.each {|r| logger.info r}
+
+    # sort by alias
+    data = data.sort {|r1, r2| r1.last <=> r2.last }
+    results.concat(data.collect {|r| format % r})
+
+    results.each {|r| logger.info(r) }
   end
 
   desc <<-DESC
@@ -195,17 +200,13 @@ namespace :rubber do
         roles = env.instance_roles.split(",") rescue []
       end
 
-      roles.each do |r|
-        role = Rubber::Configuration::RoleItem.parse(r)
-
-        # If user doesn't setup a primary db, then be nice and do it
-        if role.name == "db" && role.options["primary"] == nil && rubber_instances.for_role("db").size == 0
-          value = Capistrano::CLI.ui.ask("You do not have a primary db role, should #{instance_alias} be it [y/n]?: ")
-          role.options["primary"] = true if value =~ /^y/
-        end
-
-        ir << role
+      # If user doesn't setup a primary db, then be nice and do it
+      if ! roles.include?("db:primary=true") && rubber_instances.for_role("db").size == 0
+        value = Capistrano::CLI.ui.ask("You do not have a primary db role, should #{instance_alias} be it [y/n]?: ")
+        roles << "db:primary=true" if value =~ /^y/
       end
+      
+      ir.concat roles.collect {|r| Rubber::Configuration::RoleItem.parse(r) }
 
       # Add in roles that the given set of roles depends on
       ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
@@ -253,20 +254,21 @@ namespace :rubber do
     end
     security_groups = get_assigned_security_groups(instance_alias, role_names)
 
-    ami = env.cloud_providers[env.cloud_provider].image_id
-    ami_type = env.cloud_providers[env.cloud_provider].image_type
+    cloud_env = env.cloud_providers[env.cloud_provider]
+    ami = cloud_env.image_id
+    ami_type = cloud_env.image_type
     availability_zone = env.availability_zone
 
-    create_spot_instance ||= env.cloud_providers[env.cloud_provider].spot_instance
+    create_spot_instance ||= cloud_env.spot_instance
 
     if create_spot_instance
-      spot_price = env.cloud_providers[env.cloud_provider].spot_price.to_s
+      spot_price = cloud_env.spot_price.to_s
 
       logger.info "Creating spot instance request for instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
       request_id = cloud.create_spot_instance_request(spot_price, ami, ami_type, security_groups, availability_zone)
 
       print "Waiting for spot instance request to be fulfilled"
-      max_wait_time = env.cloud_providers[env.cloud_provider].spot_instance_request_timeout || (1.0 / 0) # Use the specified timeout value or default to infinite.
+      max_wait_time = cloud_env.spot_instance_request_timeout || (1.0 / 0) # Use the specified timeout value or default to infinite.
       instance_id = nil
       while instance_id.nil? do
         print "."
@@ -302,11 +304,8 @@ namespace :rubber do
 
     # Sometimes tag creation will fail, indicating that the instance doesn't exist yet even though it does.  It seems to
     # be a propagation delay on Amazon's end, so the best we can do is wait and try again.
-    begin
+    Rubber::Util.retry_on_failure(Exception, :retry_sleep => 0.5, :retry_count => 100) do
       Rubber::Tag::update_instance_tags(instance_alias)
-    rescue Exception
-      sleep 0.5
-      retry
     end
   end
 
@@ -406,7 +405,7 @@ namespace :rubber do
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    value = Capistrano::CLI.ui.ask("About to DESTROY #{instance_alias} (#{instance_item.instance_id}) in mode #{RUBBER_ENV}.  Are you SURE [yes/NO]?: ") unless force
+    value = Capistrano::CLI.ui.ask("About to DESTROY #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ") unless force
     fatal("Exiting", 0) if value != "yes" && ! force
 
     if instance_item.static_ip
@@ -451,7 +450,7 @@ namespace :rubber do
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    value = Capistrano::CLI.ui.ask("About to REBOOT #{instance_alias} (#{instance_item.instance_id}) in mode #{RUBBER_ENV}.  Are you SURE [yes/NO]?: ")
+    value = Capistrano::CLI.ui.ask("About to REBOOT #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
 
     logger.info "Rebooting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
@@ -469,7 +468,7 @@ namespace :rubber do
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{RUBBER_ENV}.  Are you SURE [yes/NO]?: ")
+    value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
 
     logger.info "Stopping instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
@@ -487,7 +486,7 @@ namespace :rubber do
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{RUBBER_ENV}.  Are you SURE [yes/NO]?: ")
+    value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
 
     logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"