sml-rubber 0.9.9 → 0.9.10

data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger-apache-vhost.conf

@@ -0,0 +1,46 @@
+<%
+  @path = "/etc/apache2/sites-available/#{rubber_env.app_name}-passenger"
+  @post = "a2enmod rewrite && a2enmod ssl && a2ensite #{rubber_env.app_name}-passenger"
+%>
+
+<% [rubber_env.passenger_listen_port, rubber_env.passenger_listen_ssl_port].each do |port| %>
+
+Listen <%= port %>
+<VirtualHost *:<%= port %>>
+  ServerName      <%= rubber_env.domain %>
+<%- if rubber_env.web_aliases -%>
+  ServerAlias	    <%= rubber_env.web_aliases.join(" ") %>
+<%- end -%>
+  DocumentRoot    <%= RUBBER_ROOT + "/public" %>
+
+  ErrorDocument 404 /404.html
+  ErrorDocument 500 /500.html
+
+  SetEnvIf User-Agent "^(.*MSIE.*)|(.*AppleWebKit.*)$" nokeepalive
+
+  RailsEnv  <%= RUBBER_ENV %>
+  RailsAllowModRewrite on
+
+  RewriteEngine On
+  RewriteCond %{HTTP_HOST} ^<%= rubber_env.domain %>$
+  RewriteRule ^(.*)$ http://www.<%= rubber_env.domain %>$1 [R,L]
+
+  # Include <%= RUBBER_ROOT %>/config/apache/rewrites.conf
+
+  RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f
+  RewriteCond %{SCRIPT_FILENAME} !maintenance.html  
+  RewriteRule ^.*$ /system/maintenance.html [L]
+  
+  <% if port == rubber_env.passenger_listen_ssl_port %>
+  SSLEngine on
+  SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire
+  SSLCertificateFile    /etc/ssl/certs/ssl-cert-snakeoil.pem
+  SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
+  # SSLCertificateFile <%= RUBBER_ROOT %>/config/<%= rubber_env.domain %>.crt
+  # SSLCertificateKeyFile <%= RUBBER_ROOT %>/config/<%= rubber_env.domain %>.key
+  # SSLCertificateChainFile /etc/ssl/certs/gd_intermediate_bundle.crt
+  <% end %>
+
+</VirtualHost>
+
+<% end %>

data/generators/vulcanize/templates/passenger/config/rubber/role/passenger/passenger.conf

@@ -0,0 +1,10 @@
+<%
+  @path = '/etc/apache2/mods-available/passenger.conf'
+  @post = 'cd /etc/apache2/mods-enabled && ln -fs ../mods-available/passenger.conf'
+%>
+LoadModule passenger_module /usr/lib/ruby/gems/1.8/gems/passenger-<%= rubber_env.passenger_version %>/ext/apache2/mod_passenger.so
+PassengerRoot /usr/lib/ruby/gems/1.8/gems/passenger-<%= rubber_env.passenger_version %>
+PassengerRuby /usr/bin/ruby
+PassengerUseGlobalQueue on
+
+PassengerMaxPoolSize <%= rubber_env.max_app_connections %>

data/generators/vulcanize/templates/passenger/config/rubber/rubber-passenger.yml

@@ -1,14 +1,12 @@
-security_groups:
-  web:
-    description: To open up port 80 for http server on web role
-    rules:
-      - protocol: tcp
-        from_port: 80
-        to_port: 80
-        source_ips: [0.0.0.0/0]
+passenger_version: 2.2.5
+passenger_listen_port: 7000
+passenger_listen_ssl_port: 7001
+max_app_connections: 20
+
+role_dependencies:
+  passenger: [apache]
 
 roles:
-  web:
+  app:
     packages: [apache2-mpm-prefork, apache2-prefork-dev]
-    gems: [fastthread, rack, [passenger, 2.2.4]]
-    assigned_security_groups: [web]
+    gems: [fastthread, rack, [passenger, "#{passenger_version}"]]

data/generators/vulcanize/templates/sphinx/config/rubber/deploy-sphinx.rb

@@ -32,7 +32,7 @@ namespace :rubber do
         TMPDIR=`mktemp -d` || exit 1
         cd $TMPDIR
         echo 'Downloading'
-        wget -q http://www.sphinxsearch.com/downloads/sphinx-#{ver}.tar.gz
+        wget -qN http://www.sphinxsearch.com/downloads/sphinx-#{ver}.tar.gz
         echo 'Unpacking'
         tar xf sphinx-#{ver}.tar.gz
         cd sphinx-#{ver}

data/lib/rubber/cloud/aws.rb

@@ -11,6 +11,7 @@ module Rubber
         super(env, capistrano)
         @aws_env = env.cloud_providers.aws
         @ec2 = AWS::EC2::Base.new(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+        @ec2elb = AWS::ELB::Base.new(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
         AWS::S3::Base.establish_connection!(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
       end
 
@@ -269,6 +270,35 @@ module Rubber
         end
       end
 
+      def describe_load_balancers(name=nil)
+        lbs = []
+        opts = {}
+        opts[:load_balancer_names] = name if name
+        response = @ec2elb.describe_load_balancers(opts)
+        response.describeLoadBalancersResult.member.each do |member|
+          lb = {}
+          lb[:name] = member.loadBalancerName
+          lb[:dns_name] = member.dNSName
+
+          member.availabilityZones.member.each do |zone|
+            lb[:zones] ||= []
+            lb[:zones] << zone
+          end
+
+          member.listeners.member.each do |member|
+            listener = {}
+            listener[:protocol] = member.protocol
+            listener[:port] = member.loadBalancerPort
+            listener[:instance_port] = member.instancePort
+            lb[:listeners] ||= []
+            lb[:listeners] << listener
+          end
+
+          lbs << lb
+        end if response.describeLoadBalancersResult
+        return lbs
+      end
+
     end
 
   end

data/lib/rubber/instance.rb

@@ -76,7 +76,8 @@ module Rubber
 
     # The configuration for a single instance
     class InstanceItem
-      attr_reader :name, :domain, :roles, :instance_id
+      attr_reader :name, :domain, :instance_id
+      attr_accessor :roles
       attr_accessor :external_host, :external_ip
       attr_accessor :internal_host, :internal_ip
       attr_accessor :static_ip, :volumes, :partitions
@@ -102,9 +103,59 @@ module Rubber
     class RoleItem
       attr_reader :name, :options
 
+      def self.expand_role_dependencies(roles, dependency_map, expanded=[])
+        roles = Array(roles)
+        roles.each do |role|
+          unless expanded.include?(role)
+            expanded << role
+            needed = dependency_map[role]
+            expand_role_dependencies(needed, dependency_map, expanded)
+          end
+        end
+        return expanded
+      end
+
+      def self.parse(str)
+        data = str.split(':');
+        role = Rubber::Configuration::RoleItem.new(data[0])
+        if data[1]
+          data[1].split(';').each do |pair|
+            p = pair.split('=')
+            val = case p[1]
+                    when 'true' then true
+                    when 'false' then false
+                    else p[1] end
+            role.options[p[0]] = val
+          end
+        end
+        return role
+      end
+
+      def to_s
+        str = @name
+        @options.each_with_index do |kv, i|
+          str += (i == 0 ? ':' : ';')
+          str += "#{kv[0]}=#{kv[1]}"
+        end
+        return str
+      end
+
       def initialize(name, options={})
         @name = name
-        @options = options
+        @options = options || {}
+      end
+
+      def eql?(rhs)
+        rhs && @name == rhs.name && @options == rhs.options
+      end
+      alias == eql?
+
+      def hash()
+        @name.hash
+      end
+
+      def <=>(rhs)
+        return @name <=> rhs.name
       end
     end
 

data/lib/rubber/recipes/rubber.rb

@@ -46,15 +46,16 @@ namespace :rubber do
     end
     
     set :rubber_cfg, Rubber::Configuration.get_configuration(RUBBER_ENV)
-    env = rubber_cfg.environment.bind()
+    set :rubber_env, rubber_cfg.environment.bind()
+    set :rubber_instances, rubber_cfg.instance
 
-    set :cloud, Rubber::Cloud::get_provider(env.cloud_provider || "aws", env, self)
+    set :cloud, Rubber::Cloud::get_provider(rubber_env.cloud_provider || "aws", rubber_env, self)
 
-    load_roles() unless rubber_cfg.environment.bind().disable_auto_roles
+    load_roles() unless rubber_env.disable_auto_roles
     # NOTE: for some reason Capistrano requires you to have both the public and
     # the private key in the same folder, the public key should have the
     # extension ".pub".
-    ssh_options[:keys] = env.cloud_providers[env.cloud_provider].key_file
+    ssh_options[:keys] = rubber_env.cloud_providers[rubber_env.cloud_provider].key_file
   end
 
 
@@ -64,13 +65,13 @@ namespace :rubber do
 
     # define empty roles for all known ones so tasks don't fail if a role
     # doesn't exist due to a filter
-    all_roles = rubber_cfg.instance.all_roles
+    all_roles = rubber_instances.all_roles
     all_roles += rubber_cfg.environment.known_roles
     all_roles.uniq!
     all_roles.each {|name| top.roles[name.to_sym] = []}
 
     # define capistrano host => role mapping for all instances
-    rubber_cfg.instance.filtered.each do |ic|
+    rubber_instances.filtered.each do |ic|
       ic.roles.each do |role|
         opts = Rubber::Util::symbolize_keys(role.options)
         msg = "Auto role: #{role.name.to_sym} => #{ic.full_name}"

data/lib/rubber/recipes/rubber/deploy.rb

@@ -24,6 +24,15 @@ namespace :rubber do
   task :post_stop do
   end
 
+  # Don't want to do rubber:config for update_code as that tree isn't official
+  # until it is 'committed' by the symlink task (and doing so causes it to run
+  # for bootstrap_db which should only config the db config file).  However,
+  # deploy:migrations doesn't call update, so we need an additional trigger for
+  # it
+  after "deploy:update", "rubber:config"
+  after "deploy:rollback_code", "rubber:config"
+  before "deploy:migrate", "rubber:config"
+
   desc <<-DESC
     Configures the deployed rails application by running the rubber configuration process
   DESC
@@ -42,6 +51,18 @@ namespace :rubber do
     run_config(opts)
   end
 
+  # because we start server as appserver user, but migrate as root, server needs to be able to write logs, etc.
+  before "rubber:pre_start", "rubber:setup_app_permissions"
+  before "rubber:pre_restart", "rubber:setup_app_permissions"
+
+  desc <<-DESC
+    Sets permissions of files in application directory to be owned by app_user.
+  DESC
+  task :setup_app_permissions do
+    run "find #{shared_path} -name cached-copy -prune -o -print | xargs chown #{rubber_env.app_user}:#{rubber_env.app_user}"
+    run "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{current_path}/tmp"
+  end
+
   def run_config(options={})
     path = options.delete(:deploy_path) || current_path
     extra_env = options.keys.inject("") {|all, k|  "#{all} #{k}=\"#{options[k]}\""}
@@ -49,7 +70,7 @@ namespace :rubber do
     # Need to do this so we can work with staging instances without having to
     # checkin instance file between create and bootstrap, as well as during a deploy
     if fetch(:push_instance_config, false)
-      push_files = [rubber_cfg.instance.file] + rubber_cfg.environment.config_files
+      push_files = [rubber_instances.file] + rubber_cfg.environment.config_files
       push_files.each do |file|
         dest_file = file.sub(/^#{RUBBER_ROOT}\/?/, '')
         put(File.read(file), File.join(path, dest_file), :mode => "+r")

data/lib/rubber/recipes/rubber/instances.rb

@@ -15,21 +15,10 @@ namespace :rubber do
 
     ir = []
     instance_roles.each do |r|
-      data = r.split(':');
-      role = Rubber::Configuration::RoleItem.new(data[0])
-      if data[1]
-        data[1].split(';').each do |pair|
-          p = pair.split('=')
-          val = case p[1]
-                  when 'true' then true
-                  when 'false' then false
-                  else p[1] end
-          role.options[p[0]] = val
-        end
-      end
+      role = Rubber::Configuration::RoleItem.parse(r)
 
       # If user doesn't setup a primary db, then be nice and do it
-      if role.name == "db" && role.options["primary"] == nil && rubber_cfg.instance.for_role("db").size == 0
+      if role.name == "db" && role.options["primary"] == nil && rubber_instances.for_role("db").size == 0
         value = Capistrano::CLI.ui.ask("You do not have a primary db role, should #{instance_alias} be it [y/n]?: ")
         role.options["primary"] = true if value =~ /^y/
       end
@@ -37,6 +26,9 @@ namespace :rubber do
       ir << role
     end
 
+    # Add in roles that the given set of roles depends on
+    ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
+
     create_instance(instance_alias, ir)
   end
 
@@ -63,11 +55,58 @@ namespace :rubber do
     Destroy ALL the EC2 instances for the current env
   DESC
   task :destroy_all do
-    rubber_cfg.instance.each do |ic|
+    rubber_instances.each do |ic|
       destroy_instance(ic.name)
     end
   end
 
+  desc <<-DESC
+    Adds the given ROLES to the instance named ALIAS
+  DESC
+  required_task :add_role do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+
+    instance_roles = r.split(",")
+
+    ir = []
+    instance_roles.each do |r|
+      role = Rubber::Configuration::RoleItem.parse(r)
+      ir << role
+    end
+
+    # Add in roles that the given set of roles depends on
+    ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
+
+    instance = rubber_instances[instance_alias]
+    fatal "Instance does not exist: #{instance_alias}" unless instance
+
+    instance.roles = (instance.roles + ir).uniq
+    rubber_instances.save()
+  end
+
+  desc <<-DESC
+    Removes the given ROLES from the instance named ALIAS
+  DESC
+  required_task :remove_role do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+
+    instance_roles = r.split(",")
+
+    ir = []
+    instance_roles.each do |r|
+      role = Rubber::Configuration::RoleItem.parse(r)
+      ir << role
+    end
+
+    instance = rubber_instances[instance_alias]
+    fatal "Instance does not exist: #{instance_alias}" unless instance
+
+    instance.roles = (instance.roles - ir).uniq
+    rubber_instances.save()
+  end
+
   desc <<-DESC
     List all your EC2 instances
   DESC
@@ -106,7 +145,7 @@ namespace :rubber do
   # Creates a new ec2 instance with the given alias and roles
   # Configures aliases (/etc/hosts) on local and remote machines
   def create_instance(instance_alias, instance_roles)
-    fatal "Instance already exists: #{instance_alias}" if rubber_cfg.instance[instance_alias]
+    fatal "Instance already exists: #{instance_alias}" if rubber_instances[instance_alias]
 
     role_names = instance_roles.collect{|x| x.name}
     env = rubber_cfg.environment.bind(role_names, instance_alias)
@@ -124,8 +163,8 @@ namespace :rubber do
     logger.info "Instance #{instance_id} created"
 
     instance_item = Rubber::Configuration::InstanceItem.new(instance_alias, env.domain, instance_roles, instance_id)
-    rubber_cfg.instance.add(instance_item)
-    rubber_cfg.instance.save()
+    rubber_instances.add(instance_item)
+    rubber_instances.save()
 
 
     print "Waiting for instance to start"
@@ -140,7 +179,7 @@ namespace :rubber do
         instance_item.external_host = instance[:external_host]
         instance_item.external_ip = instance[:external_ip]
         instance_item.internal_host = instance[:internal_host]
-        rubber_cfg.instance.save()
+        rubber_instances.save()
 
         # setup amazon elastic ips if configured to do so
         setup_static_ips
@@ -157,7 +196,7 @@ namespace :rubber do
 
         task :_get_ip, :hosts => instance_item.external_ip do
           instance_item.internal_ip = capture(print_ip_command).strip
-          rubber_cfg.instance.save()
+          rubber_instances.save()
         end
 
         # even though instance is running, sometimes ssh hasn't started yet,
@@ -182,7 +221,7 @@ namespace :rubber do
   # Refreshes a ec2 instance with the given alias
   # Configures aliases (/etc/hosts) on local and remote machines
   def refresh_instance(instance_alias)
-    instance_item = rubber_cfg.instance[instance_alias]
+    instance_item = rubber_instances[instance_alias]
 
     fatal "Instance does not exist: #{instance_alias}" if ! instance_item
 
@@ -218,13 +257,13 @@ namespace :rubber do
       setup_dns_aliases
     end
 
-    rubber_cfg.instance.save()
+    rubber_instances.save()
   end
 
 
   # Destroys the given ec2 instance
   def destroy_instance(instance_alias)
-    instance_item = rubber_cfg.instance[instance_alias]
+    instance_item = rubber_instances[instance_alias]
     fatal "Instance does not exist: #{instance_alias}" if ! instance_item
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
@@ -250,8 +289,8 @@ namespace :rubber do
 
     cloud.destroy_instance(instance_item.instance_id)
 
-    rubber_cfg.instance.remove(instance_alias)
-    rubber_cfg.instance.save()
+    rubber_instances.remove(instance_alias)
+    rubber_instances.save()
 
     # re-load the roles since we just removed some and setup_remote_aliases
     # shouldn't hit removed ones
@@ -284,4 +323,15 @@ namespace :rubber do
     end
   end
 
+  def get_role_dependencies
+    # convert string format of role_dependencies from rubber.yml into
+    # objects for use by expand_role_dependencies
+    deps = {}
+    rubber_env.role_dependencies.each do |k, v|
+      rhs = Array(v).collect {|r| Rubber::Configuration::RoleItem.parse(r)}
+      deps[Rubber::Configuration::RoleItem.parse(k)] = rhs
+    end
+    return deps
+  end
+  
 end