sml-rubber 0.9.9 → 0.9.10

data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/rubber-complete.yml

@@ -0,0 +1,41 @@
+
+role_dependencies:
+  web: [nginx, haproxy]
+  app: [mongrel]
+  web_tools: [nginx]
+
+haproxy_listen_port: 9000
+web_port: 80
+web_ssl_port: 443
+web_tools_port: 8080
+web_tools_ssl_port: 8443
+
+security_groups:
+  web:
+    description: "To open up port #{web_port}/#{web_ssl_port} for http server on web role"
+    rules:
+      - protocol: tcp
+        from_port: "#{web_port}"
+        to_port: "#{web_port}"
+        source_ips: [0.0.0.0/0]
+      - protocol: tcp
+        from_port: "#{web_ssl_port}"
+        to_port: "#{web_ssl_port}"
+        source_ips: [0.0.0.0/0]
+  web_tools:
+    description: "To open up port #{web_tools_port}/#{web_tools_ssl_port} for internal/tools http server"
+    rules:
+      - protocol: tcp
+        from_port: "#{web_tools_port}"
+        to_port: "#{web_tools_port}"
+        source_ips: [0.0.0.0/0]
+      - protocol: tcp
+        from_port: "#{web_tools_ssl_port}"
+        to_port: "#{web_tools_ssl_port}"
+        source_ips: [0.0.0.0/0]
+
+roles:
+  web:
+    assigned_security_groups: [web]
+  web_tools:
+    assigned_security_groups: [web_tools]

data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/role/haproxy/haproxy-passenger.conf

@@ -0,0 +1,19 @@
+<%
+  @path = "/etc/haproxy/haproxy.cfg"
+  @additive = ["# start haproxy passenger", "# end haproxy passenger"]
+%>
+
+<%
+  backend_hosts = rubber_instances.for_role('passenger').collect { |i| i.name }
+%>
+
+listen passenger_proxy 0.0.0.0:<%= rubber_env.web_port %>
+  <% backend_hosts.each do |server| %>
+    server <%= server %> <%= server %>:<%= rubber_env.passenger_listen_port %> maxconn <%= rubber_env.max_app_connections %> check
+  <% end %>
+
+listen passenger_proxy 0.0.0.0:<%= rubber_env.web_ssl_port %>
+  mode tcp
+  <% backend_hosts.each do |server| %>
+    server <%= server %> <%= server %>:<%= rubber_env.passenger_listen_ssl_port %> maxconn <%= rubber_env.max_app_connections %> check
+  <% end %>

data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/rubber-complete.yml

@@ -0,0 +1,40 @@
+
+role_dependencies:
+  web: [haproxy]
+  app: [passenger]
+  web_tools: [apache]
+
+web_port: 80
+web_ssl_port: 443
+web_tools_port: 8080
+web_tools_ssl_port: 8443
+
+security_groups:
+  web:
+    description: "To open up port #{web_port}/#{web_ssl_port} for http server on web role"
+    rules:
+      - protocol: tcp
+        from_port: "#{web_port}"
+        to_port: "#{web_port}"
+        source_ips: [0.0.0.0/0]
+      - protocol: tcp
+        from_port: "#{web_ssl_port}"
+        to_port: "#{web_ssl_port}"
+        source_ips: [0.0.0.0/0]
+  web_tools:
+    description: "To open up port #{web_tools_port}/#{web_tools_ssl_port} for internal/tools http server"
+    rules:
+      - protocol: tcp
+        from_port: "#{web_tools_port}"
+        to_port: "#{web_tools_port}"
+        source_ips: [0.0.0.0/0]
+      - protocol: tcp
+        from_port: "#{web_tools_ssl_port}"
+        to_port: "#{web_tools_ssl_port}"
+        source_ips: [0.0.0.0/0]
+
+roles:
+  web:
+    assigned_security_groups: [web]
+  web_tools:
+    assigned_security_groups: [web_tools]

data/generators/vulcanize/templates/complete_passenger_mysql/templates.yml

@@ -2,7 +2,9 @@ description: A fairly complete and scalable mysql deployment setup
 dependent_templates:
   - base
   - mysql
+  - haproxy
   - apache
   - passenger
+  - monit
   - munin
-  - memcached
+  - memcached

data/generators/vulcanize/templates/cruise/config/rubber/deploy-cruise.rb

@@ -8,9 +8,8 @@ namespace :rubber do
     after "custom_install_base", "rubber:cruise:custom_install"
 
     task :custom_install, :roles => :cruise do
-      env = rubber_cfg.environment.bind()
       rubber.sudo_script 'install_cruise', <<-ENDSCRIPT
-        export CRUISE_HOME="#{env.cruise_dir}"
+        export CRUISE_HOME="#{rubber_env.cruise_dir}"
         
         if [[ ! -d $CRUISE_HOME ]]; then
             git clone git://github.com/sml/cruisecontrol.rb.git $CRUISE_HOME
@@ -26,26 +25,25 @@ namespace :rubber do
    end
     
     task :setup_project, :roles => :cruise do
-      env = rubber_cfg.environment.bind()      
       rubber.sudo_script 'setup_project', <<-ENDSCRIPT
       
-        export CRUISE_HOME="#{env.cruise_dir}"
-        export CRUISE_DATA_ROOT="#{env.cruise_data_dir}"
-        export CRUISE_PROJECT_ROOT="#{env.cruise_project_dir}"
+        export CRUISE_HOME="#{rubber_env.cruise_dir}"
+        export CRUISE_DATA_ROOT="#{rubber_env.cruise_data_dir}"
+        export CRUISE_PROJECT_ROOT="#{rubber_env.cruise_project_dir}"
         
         if [[ ! -d $CRUISE_HOME ]]; then
-            git clone git://github.com/sml/cruisecontrol.rb.git #{env.cruise_dir}
+            git clone git://github.com/sml/cruisecontrol.rb.git #{rubber_env.cruise_dir}
         fi
         
         if [[ ! -d $CRUISE_PROJECT_ROOT ]]; then
-          if [[ -z "#{env.cruise_repository}" ]]; then
+          if [[ -z "#{rubber_env.cruise_repository}" ]]; then
             echo "cruise_repository must be set in rubber env"
             exit 1
           fi
           
           cd $CRUISE_HOME
-          echo "Adding project repository to cruise: #{env.cruise_repository}"
-          ./cruise add #{env.app_name} #{env.cruise_repository}
+          echo "Adding project repository to cruise: #{rubber_env.cruise_repository}"
+          ./cruise add #{rubber_env.app_name} #{rubber_env.cruise_repository}
           mkdir -p $CRUISE_PROJECT_ROOT/log            
           cd $CRUISE_PROJECT_ROOT && rake db:create RAILS_ENV=test
         fi

data/generators/vulcanize/templates/cruise/config/rubber/role/web_tools/cruise-nginx.conf

@@ -1,6 +1,6 @@
 <%
   cruise_host = rubber_instances.for_role('cruise').first
-  @path = "/etc/nginx/rubber/cruise.conf"
+  @path = "/etc/nginx/rubber/tools/cruise.conf"
 %>
 
 <% if cruise_host %>

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/haproxy-base.conf

@@ -0,0 +1,26 @@
+<%
+  @path = "/etc/haproxy/haproxy.cfg"
+%>
+global
+  log 127.0.0.1 local0 warning
+  maxconn 1024
+ 
+defaults
+  log global
+  mode http
+  retries 3
+  balance roundrobin
+  option abortonclose
+  option redispatch
+  contimeout 4000
+  clitimeout 150000
+  srvtimeout 30000
+
+# Admin interface for proxy stats
+listen admin 0.0.0.0:<%= rubber_env.haproxy_admin_port %>
+  stats uri /haproxy
+
+# So monit can check on haproxy
+# simple response : 'OK'
+listen health_check 0.0.0.0:<%= rubber_env.haproxy_monitor_port %>
+  mode health

data/generators/vulcanize/templates/haproxy/config/rubber/role/web_tools/haproxy-nginx.conf

@@ -1,5 +1,5 @@
 <%
-  @path = "/etc/nginx/rubber/haproxy.conf"
+  @path = "/etc/nginx/rubber/tools/haproxy.conf"
 %>
 
 <% rubber_instances.for_role('haproxy').each do |ic| %>

data/generators/vulcanize/templates/haproxy/config/rubber/rubber-haproxy.yml

@@ -1,10 +1,5 @@
 
-# Uses from appserver (mongrel) rubber config 
-# appserver_base_port: 8000
-# appserver_count: 8
-
 haproxy_monitor_port: 60001
-haproxy_listen_port: 9000
 haproxy_admin_port: 9100
 
 roles:

data/generators/vulcanize/templates/mongrel/config/rubber/deploy-mongrel.rb

@@ -7,9 +7,9 @@ namespace :rubber do
     
     after "rubber:install_gems", "rubber:mongrel:custom_install"
     
-    task :custom_install, :roles => :app do
+    task :custom_install, :roles => :mongrel do
       # Setup system to restart mongrel_cluster on reboot
-      rubber.sudo_script 'install_app', <<-ENDSCRIPT
+      rubber.sudo_script 'setup_mongrel_init', <<-ENDSCRIPT
         mkdir -p /etc/mongrel_cluster
         rm -f /etc/mongrel_cluster/#{application}-#{RUBBER_ENV}.yml && ln -s /mnt/#{application}-#{RUBBER_ENV}/current/config/mongrel_cluster.yml /etc/mongrel_cluster/#{application}-#{RUBBER_ENV}.yml
         find /usr/lib/ruby/gems -path "*/resources/mongrel_cluster" -exec cp {} /etc/init.d/ \\;
@@ -28,7 +28,7 @@ namespace :rubber do
     
     def mongrel_start
         run "cd #{current_path} && mongrel_rails cluster::start --clean"
-        pid_cnt = rubber_cfg.environment.bind().appserver_count
+        pid_cnt = rubber_env.mongrel_count
         logger.info "Waiting for mongrel pid files to show up"
         run "while ((`ls #{current_path}/tmp/pids/mongrel.*.pid 2> /dev/null | wc -l` < #{pid_cnt})); do sleep 1; done"
     end
@@ -36,7 +36,7 @@ namespace :rubber do
     # serial_task can only be called after roles defined - not normally a problem, but
     # rubber auto-roles don't get defined till after all tasks are defined
     on :load do
-      rubber.serial_task self, :serial_restart, :roles => :app do
+      rubber.serial_task self, :serial_restart, :roles => :mongrel do
         teardown_connections_to(sessions.keys)
         mongrel_stop
         mongrel_start
@@ -44,17 +44,17 @@ namespace :rubber do
     end
     
     desc "Restarts the mongrel app server"
-    task :restart, :roles => :app do
+    task :restart, :roles => :mongrel do
       serial_restart
     end
     
     desc "Stops the mongrel app server"
-    task :stop, :roles => :app do
+    task :stop, :roles => :mongrel do
       mongrel_stop
     end
     
     desc "Starts the mongrel app server"
-    task :start, :roles => :app do
+    task :start, :roles => :mongrel do
       mongrel_start
     end
     

data/generators/vulcanize/templates/mongrel/config/rubber/role/{app → mongrel}/mongrel_cluster.yml

@@ -2,11 +2,11 @@
   @path = "#{RUBBER_ROOT}/config/mongrel_cluster.yml"
 %>
 ---
-user: rails
-group: rails
+user: <%= rubber_env.app_user %>
+group: <%= rubber_env.app_user %>
 cwd: <%= RUBBER_ROOT %>
-port: "<%= rubber_env.appserver_base_port %>"
+port: "<%= rubber_env.mongrel_base_port %>"
 environment: <%= RUBBER_ENV %>
 address: <%= rubber_env.host %>
 pid_file: tmp/pids/mongrel.pid
-servers: <%= rubber_env.appserver_count %>
+servers: <%= rubber_env.mongrel_count %>

data/generators/vulcanize/templates/mongrel/config/rubber/role/{app → mongrel}/monit-mongrel.conf

@@ -2,9 +2,9 @@
   @path = '/etc/monit/monit.d/monit-mongrel.conf'
 %>
 
-<% (0..(rubber_env.appserver_count.to_i - 1)).each do |i| %>
+<% (0..(rubber_env.mongrel_count.to_i - 1)).each do |i| %>
 
-  <% PORT = rubber_env.appserver_base_port + i %>
+  <% PORT = rubber_env.mongrel_base_port + i %>
   <% PIDFILE = "#{RUBBER_ROOT}/tmp/pids/mongrel.#{PORT}.pid" %>
   check process mongrel-<%= PORT %> with pidfile <%= PIDFILE %>
     group mongrel-<%= RUBBER_ENV %>

data/generators/vulcanize/templates/mongrel/config/rubber/rubber-mongrel.yml

@@ -1,7 +1,7 @@
 
 # These will probably need to be used by the rubber module used for the web role (nginx)
-appserver_base_port: 8000
-appserver_count: "#{RUBBER_ENV == 'production' ? 8 : 2}"
+mongrel_base_port: 7000
+mongrel_count: "#{RUBBER_ENV == 'production' ? 8 : 2}"
 
 roles:
   app:

data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf

@@ -1,5 +1,5 @@
 <%
-  @path = "/etc/nginx/rubber/monit.conf"
+  @path = "/etc/nginx/rubber/tools/monit.conf"
 %>
 
 <% rubber_instances.each do |ic| %>

data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb

@@ -4,7 +4,23 @@ namespace :rubber do
   namespace :munin do
   
     rubber.allow_optional_tasks(self)
-    
+
+    after "rubber:install_packages", "rubber:munin:custom_install"
+
+    # sometimes apt-get install of munin doesn't end up configuring
+    # plugins (e.g. installing postfix at same time, postfix plugin
+    # configure barfs as postfix not configured yet)
+    desc <<-DESC
+      Reconfigures munin
+    DESC
+    task :custom_install do
+      rubber.run_script 'setup_munin_plugins', <<-ENDSCRIPT
+        munin-node-configure --shell --remove-also > /tmp/setup-munin-plugins
+        sh /tmp/setup-munin-plugins
+      ENDSCRIPT
+      restart
+    end
+
     # after "deploy:stop", "rubber:munin:stop"
     # before "deploy:start", "rubber:munin:start"
     # after "deploy:restart", "rubber:munin:restart"

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf

@@ -1,5 +1,5 @@
 <%
-  @path = "/etc/nginx/rubber/munin.conf"
+  @path = "/etc/nginx/rubber/tools/munin.conf"
 %>
 
 location /munin

data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml

@@ -1,6 +1,6 @@
 # Uses admin_email from base
 
-packages: [munin-node]
+packages: [munin-node, munin-plugins-extra]
 
 roles:
   web_tools:

data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb

@@ -5,35 +5,17 @@ namespace :rubber do
     
     rubber.allow_optional_tasks(self)
     
-    after "rubber:create", "rubber:mysql:set_db_role"
-      
-    # Capistrano needs db:primary role for migrate to work
-    task :set_db_role do
-      db_instances = rubber_cfg.instance.for_role("mysql_master")
-      db_instances.each do |instance|
-        if ! instance.role_names.find {|n| n == 'db'}
-          role = Rubber::Configuration::RoleItem.new('db')
-          primary_exists = rubber_cfg.instance.for_role("db", "primary" => true).size > 0
-          role.options["primary"] = true  unless primary_exists
-          instance.roles << role
-        end
-      end
-      db_instances = rubber_cfg.instance.for_role("mysql_slave")
+    after "rubber:create", "rubber:mysql:validate_db_roles"
+
+    task :validate_db_roles do
+      db_instances = rubber_instances.for_role("mysql_slave")
       db_instances.each do |instance|
         if instance.role_names.find {|n| n == 'mysql_master'}
-          logger.info "Cannot have a mysql slave and master on the same instance, removing slave role"
-          instance.roles.delete_if {|r| r.name == 'mysql_slave'}
-          next
-        end
-        if ! instance.role_names.find {|n| n == 'db'}
-          role = Rubber::Configuration::RoleItem.new('db')
-          instance.roles << role
+          fatal "Cannot have a mysql slave and master on the same instance, please removing slave role for #{instance.name}"
         end
       end
-      rubber_cfg.instance.save()
-      load_roles() unless rubber_cfg.environment.bind().disable_auto_roles
     end
-  
+
     after "rubber:bootstrap", "rubber:mysql:bootstrap"
   
     
@@ -46,7 +28,7 @@ namespace :rubber do
       # Conditionaly bootstrap for each node/role only if that node has not
       # been boostrapped for that role before
       
-      master_instances = rubber_cfg.instance.for_role("mysql_master") & rubber_cfg.instance.filtered  
+      master_instances = rubber_instances.for_role("mysql_master") & rubber_instances.filtered  
       master_instances.each do |ic|
         task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym()
         task task_name, :hosts => ic.full_name do
@@ -67,7 +49,7 @@ namespace :rubber do
         send task_name
       end
     
-      slave_instances = rubber_cfg.instance.for_role("mysql_slave") & rubber_cfg.instance.filtered  
+      slave_instances = rubber_instances.for_role("mysql_slave") & rubber_instances.filtered  
       slave_instances.each do |ic|
         task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym()
         task task_name, :hosts => ic.full_name do
@@ -78,13 +60,13 @@ namespace :rubber do
             sudo "dpkg-reconfigure --frontend=noninteractive mysql-server-5.0"
             sleep 5
 
-            master = rubber_cfg.instance.for_role("mysql_master").first
+            master = rubber_instances.for_role("mysql_master").first
 
             # Doing a mysqldump locks the db, so ideally we'd do it against a slave replica thats
             # not serving traffic (mysql_util role), but if thats not available try a regular
             # slave (mysql_slave role), and finally default dumping from master (mysql_master role)
             # TODO: handle simultaneous creating of multi slaves/utils
-            slaves = rubber_cfg.instance.for_role("mysql_slave")
+            slaves = rubber_instances.for_role("mysql_slave")
             slaves.delete(ic) # don't want to try and dump from self
             source = slaves.find {|sc| sc.role_names.include?("mysql_util")}
             source = slaves.first unless source
@@ -134,20 +116,16 @@ namespace :rubber do
       rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/#{role}|role/db/my.cnf", :deploy_path => release_path)
     end
     
-    after "rubber:install_packages", "rubber:mysql:custom_install"
+    before "rubber:munin:custom_install", "rubber:mysql:custom_install_munin"
 
     desc <<-DESC
       Installs some extra munin graphs
     DESC
-    task :custom_install, :roles => [:mysql_master, :mysql_slave] do
+    task :custom_install_munin, :roles => [:mysql_master, :mysql_slave] do
       rubber.run_script 'install_munin_mysql', <<-ENDSCRIPT
-        if [ ! -f /etc/munin/plugins/mysql_ ]; then
-          wget -q -O /etc/munin/plugins/mysql_ http://github.com/kjellm/munin-mysql/raw/master/mysql_
+        if [ ! -f /usr/share/munin/plugins/mysql_ ]; then
+          wget -q -O /usr/share/munin/plugins/mysql_ http://github.com/kjellm/munin-mysql/raw/master/mysql_
           wget -q -O /etc/munin/plugin-conf.d/mysql_.conf http://github.com/kjellm/munin-mysql/raw/master/mysql_.conf
-
-          cd /etc/munin/plugins
-          chmod +x mysql_
-          ./mysql_ suggest | while read x; do ln -sf mysql_ $x; done
         fi
       ENDSCRIPT
     end