Agiley-ec2onrails 0.9.9 → 0.9.10

data/examples/Capfile

@@ -0,0 +1,3 @@
+load 'deploy' if respond_to?(:namespace) # cap2 differentiator
+load 'config/deploy'
+require 'ec2onrails/recipes'

data/examples/deploy.rb

@@ -0,0 +1,88 @@
+# This is a sample Capistrano config file for EC2 on Rails.
+# It should be edited and customized.
+
+set :application, "yourapp"
+
+set :repository, "http://svn.foo.com/svn/#{application}/trunk"
+
+# NOTE: for some reason Capistrano requires you to have both the public and
+# the private key in the same folder, the public key should have the 
+# extension ".pub".
+ssh_options[:keys] = ["#{ENV['HOME']}/.ssh/your-ec2-key"]
+
+# Your EC2 instances. Use the ec2-xxx....amazonaws.com hostname, not
+# any other name (in case you have your own DNS alias) or it won't
+# be able to resolve to the internal IP address.
+role :web,      "ec2-12-xx-xx-xx.z-1.compute-1.amazonaws.com"
+role :app,      "ec2-34-xx-xx-xx.z-1.compute-1.amazonaws.com"
+role :db,       "ec2-56-xx-xx-xx.z-1.compute-1.amazonaws.com", :primary => true
+role :memcache, "ec2-12-xx-xx-xx.z-1.compute-1.amazonaws.com"
+
+# Whatever you set here will be taken set as the default RAILS_ENV value
+# on the server. Your app and your hourly/daily/weekly/monthly scripts
+# will run with RAILS_ENV set to this value.
+set :rails_env, "production"
+
+# EC2 on Rails config. 
+# NOTE: Some of these should be omitted if not needed.
+set :ec2onrails_config, {
+  # S3 bucket and "subdir" used by the ec2onrails:db:restore task
+  :restore_from_bucket => "your-bucket",
+  :restore_from_bucket_subdir => "database",
+  
+  # S3 bucket and "subdir" used by the ec2onrails:db:archive task
+  # This does not affect the automatic backup of your MySQL db to S3, it's
+  # just for manually archiving a db snapshot to a different bucket if 
+  # desired.
+  :archive_to_bucket => "your-other-bucket",
+  :archive_to_bucket_subdir => "db-archive/#{Time.new.strftime('%Y-%m-%d--%H-%M-%S')}",
+  
+  # Set a root password for MySQL. Run "cap ec2onrails:db:set_root_password"
+  # to enable this. This is optional, and after doing this the
+  # ec2onrails:db:drop task won't work, but be aware that MySQL accepts 
+  # connections on the public network interface (you should block the MySQL
+  # port with the firewall anyway). 
+  # If you don't care about setting the mysql root password then remove this.
+  :mysql_root_password => "your-mysql-root-password",
+  
+  # Any extra Ubuntu packages to install if desired
+  # If you don't want to install extra packages then remove this.
+  :packages => ["logwatch", "imagemagick"],
+  
+  # Any extra RubyGems to install if desired: can be "gemname" or if a 
+  # particular version is desired "gemname -v 1.0.1"
+  # If you don't want to install extra rubygems then remove this
+  :rubygems => ["rmagick", "rfacebook -v 0.9.7"],
+  
+  # Defines the web proxy that will be used.  Choices are :apache or :nginx
+  :web_proxy_server => :apache,
+  
+  # Set the server timezone. run "cap -e ec2onrails:server:set_timezone" for 
+  # details
+  :timezone => "UTC",
+  
+  # Files to deploy to the server (they'll be owned by root). It's intended
+  # mainly for customized config files for new packages installed via the 
+  # ec2onrails:server:install_packages task. Subdirectories and files inside
+  # here will be placed in the same structure relative to the root of the
+  # server's filesystem. 
+  # If you don't need to deploy customized config files to the server then
+  # remove this.
+  :server_config_files_root => "../server_config",
+  
+  # If config files are deployed, some services might need to be restarted.
+  # If you don't need to deploy customized config files to the server then
+  # remove this.
+  :services_to_restart => %w(postfix sysklogd),
+  
+  # Set an email address to forward admin mail messages to. If you don't
+  # want to receive mail from the server (e.g. monit alert messages) then
+  # remove this.
+  :mail_forward_address => "you@yourdomain.com",
+  
+  # Set this if you want SSL to be enabled on the web server. The SSL cert 
+  # and key files need to exist on the server, The cert file should be in
+  # /etc/ssl/certs/default.pem and the key file should be in
+  # /etc/ssl/private/default.key (see :server_config_files_root).
+  :enable_ssl => true
+}

data/examples/s3.yml

@@ -0,0 +1,9 @@
+staging:
+  aws_access_key: ABC123
+  aws_secret_access_key: abc123abc123abc123abc123
+  bucket_base_name: yourbucket
+
+production:
+  aws_access_key: DEF456
+  aws_secret_access_key: def456def456def456def456
+  bucket_base_name: yourbucket

data/lib/ec2onrails/capistrano_utils.rb

@@ -10,17 +10,6 @@ module Ec2onrails
       sudo "sh -c 'if [ -x /etc/init.d/#{script} ] ; then /etc/init.d/#{script} #{arg}; fi'"
     end
     
-    def make_admin_role_for(role)
-      newrole = "#{role.to_s}_admin".to_sym
-      roles[role].each do |srv_def|
-        options = srv_def.options.dup
-        options[:user] = "admin"
-        options[:port] = srv_def.port
-        options[:no_release] = true
-        role newrole, srv_def.host, options
-      end
-    end
-    
     # return hostnames for the role named role_sym that has the specified options
     def hostnames_for_role(role_sym, options = {})
       role = roles[role_sym]

data/lib/ec2onrails/recipes.rb

@@ -35,6 +35,9 @@ Capistrano::Configuration.instance.load do
   end
   
   cfg = ec2onrails_config
+  
+  #:apache or :nginx
+  cfg[:web_proxy_server] ||= :apache
 
   set :ec2onrails_version, Ec2onrails::VERSION::STRING
   set :image_id_32_bit, Ec2onrails::VERSION::AMI_ID_32_BIT
@@ -43,47 +46,45 @@ Capistrano::Configuration.instance.load do
   set :use_sudo, false
   set :user, "app"
 
-  # make an "admin" role for each role, and create arrays containing
-  # the names of admin roles and non-admin roles for convenience
-  set :all_admin_role_names, []
-  set :all_non_admin_role_names, []
-  roles.keys.clone.each do |name|
-    make_admin_role_for(name)
-    all_non_admin_role_names << name
-    all_admin_role_names << "#{name.to_s}_admin".to_sym
-  end
-  
-  after "deploy:symlink", "ec2onrails:server:set_roles"
-  after "deploy:cold", "ec2onrails:db:init_backup"
+  #in case any changes were made to the configs, like changing the number of mongrels
+  after "deploy:symlink", "ec2onrails:server:set_roles", "ec2onrails:server:init_services"
+  after "deploy:cold", "ec2onrails:db:init_backup", "ec2onrails:db:optimize", "ec2onrails:server:restrict_sudo_access"
+  before "ec2onrails:server:install_gems", "ec2onrails:server:add_gem_sources"
   
   # override default start/stop/restart tasks
   namespace :deploy do
     desc <<-DESC
-      Overrides the default Capistrano deploy:restart, uses \
+      Overrides the default Capistrano deploy:start, uses \
       /etc/init.d/mongrel
     DESC
-    task :start, :roles => :app_admin do
-      run_init_script("mongrel", "start")
-      run "sleep 30" # give the service 30 seconds to start before attempting to monitor it
-      sudo "monit -g app monitor all"
+    task :start, :roles => :app do
+      ec2onrails.server.allow_sudo do
+        run_init_script("mongrel", "start")
+        run "sleep 30" # give the service 30 seconds to start before attempting to monitor it
+        sudo "monit -g app monitor all"
+      end
     end
     
     desc <<-DESC
-      Overrides the default Capistrano deploy:restart, uses \
+      Overrides the default Capistrano deploy:stop, uses \
       /etc/init.d/mongrel
     DESC
-    task :stop, :roles => :app_admin do
-      sudo "monit -g app unmonitor all"
-      run_init_script("mongrel", "stop")
+    task :stop, :roles => :app do
+      ec2onrails.server.allow_sudo do
+        sudo "monit -g app unmonitor all"
+        run_init_script("mongrel", "stop")
+      end
     end
     
     desc <<-DESC
       Overrides the default Capistrano deploy:restart, uses \
       /etc/init.d/mongrel
     DESC
-    task :restart, :roles => :app_admin do
-      deploy.stop
-      deploy.start
+    task :restart, :roles => :app do
+      ec2onrails.server.allow_sudo do
+        deploy.stop
+        deploy.start
+      end
     end
   end
   
@@ -128,12 +129,14 @@ Capistrano::Configuration.instance.load do
     desc <<-DESC
       Prepare a newly-started instance for a cold deploy.
     DESC
-    task :setup, :roles => all_admin_role_names do
-      server.set_admin_mail_forward_address
+    task :setup do
+      server.set_mail_forward_address
       server.set_timezone
       server.install_packages
       server.install_gems
       server.deploy_files
+      server.setup_web_proxy
+      server.set_roles
       server.enable_ssl if cfg[:enable_ssl]
       server.set_rails_env
       server.restart_services
@@ -168,12 +171,13 @@ Capistrano::Configuration.instance.load do
       DESC
       task :load_config do
         unless hostnames_for_role(:db, :primary => true).empty?
-          db_config = YAML::load(ERB.new(File.read("config/database.yml")).result)[rails_env.to_s]
-          cfg[:db_name] = db_config['database']
-          cfg[:db_user] = db_config['username'] || db_config['user'] 
-          cfg[:db_password] = db_config['password']
-          cfg[:db_host] = db_config['host']
-          cfg[:db_socket] = db_config['socket']
+          db_config = YAML::load(ERB.new(File.read("config/database.yml")).result)[rails_env.to_s] || {}
+          cfg[:db_name]     ||= db_config['database']
+          cfg[:db_user]     ||= db_config['username'] || db_config['user'] 
+          cfg[:db_password] ||= db_config['password']
+          cfg[:db_host]     ||= db_config['host']
+          cfg[:db_port]     ||= db_config['port']
+          cfg[:db_socket]   ||= db_config['socket']
         
           if (cfg[:db_host].nil? || cfg[:db_host].empty?) && (cfg[:db_socket].nil? || cfg[:db_socket].empty?)
               raise "ERROR: missing database config. Make sure database.yml contains a '#{rails_env}' section with either 'host: hostname' or 'socket: /var/run/mysqld/mysqld.sock'."
@@ -199,8 +203,11 @@ Capistrano::Configuration.instance.load do
         load_config
         start
         
-        # For some reason the default db on Hardy contains users with '' as the name.
-        # This causes authentication problems when connecting from localhost
+        # remove the default test database
+        #run %{mysql -u root -e "drop database test; flush privileges;"}
+        
+        # removing anonymous mysql accounts
+        run %{mysql -u root -D mysql -e "delete from db where User = ''; flush privileges;"}
         run %{mysql -u root -D mysql -e "delete from user where User = ''; flush privileges;"}
         
         run %{mysql -u root -e "create database if not exists #{cfg[:db_name]};"}
@@ -214,7 +221,7 @@ Capistrano::Configuration.instance.load do
         hasn't been set, e.g. when called from ec2onrails:setup.
         (But don't enable monitoring on it.)
       DESC
-      task :start, :roles => :db_admin do
+      task :start, :roles => :db do
         sudo "chmod a+x /etc/init.d/mysql"
         # The mysql init script can fail on the first startup if mysql takes too long 
         # to create the logfiles, so try again
@@ -275,6 +282,25 @@ Capistrano::Configuration.instance.load do
       task :init_backup, :roles => :db do
         run "/usr/local/ec2onrails/bin/backup_app_db.rb --reset"
       end
+      
+      # do NOT run if the flag does not exist.  This is placed by a startup script
+      # and it is only run on the first-startup.  This means after the db has been
+      # optimized, this task will not work again.  
+      #
+      # Of course you can overload it or call the file directly
+      task :optimize, :roles => :db do
+        found = capture("test -e /tmp/optimize_db_flag && echo 'file exists'") rescue false
+        if found
+          begin
+            sudo "/usr/local/ec2onrails/bin/optimize_mysql.rb"
+          ensure
+            sudo "rm -rf /tmp/optimize_db_flag" #remove so we cannot run again
+          end
+        else
+          puts "skipping as it looks like this task has already been run"
+        end
+      end
+      
     end
     
     namespace :server do
@@ -283,18 +309,34 @@ Capistrano::Configuration.instance.load do
         the appropriate settings for each role, and starts and/or stops the \
         relevant services.
       DESC
-      task :set_roles, :roles => all_admin_role_names do
+      task :set_roles do
         # TODO generate this based on the roles that actually exist so arbitrary new ones can be added
         roles = {
-          :web =>        hostnames_for_role(:web),
-          :app =>        hostnames_for_role(:app),
+          :web        => hostnames_for_role(:web),
+          :app        => hostnames_for_role(:app),
           :db_primary => hostnames_for_role(:db, :primary => true),
-          :memcache =>   hostnames_for_role(:memcache)
+          # doing th ebelow can cause errors elsewhere unless :db is populated.
+          # :db         => hostnames_for_role(:db),
+          :memcache   => hostnames_for_role(:memcache)
         }
         roles_yml = YAML::dump(roles)
         put roles_yml, "/tmp/roles.yml"
-        sudo "cp /tmp/roles.yml /etc/ec2onrails"
-        sudo "/usr/local/ec2onrails/bin/set_roles.rb"
+        server.allow_sudo do
+          sudo "cp /tmp/roles.yml /etc/ec2onrails"
+          #we want everyone to be able to read to it
+          sudo "chmod a+r /etc/ec2onrails/roles.yml"
+          sudo "/usr/local/ec2onrails/bin/set_roles.rb"
+        end
+      end
+      
+      task :init_services do
+        server.allow_sudo do
+          sudo "/usr/local/ec2onrails/bin/init_services.rb"
+        end
+      end
+      
+      task :setup_web_proxy, :roles => :web do
+        sudo "/usr/local/ec2onrails/bin/setup_web_proxy.rb --mode #{cfg[:web_proxy_server].to_s}"
       end
 
       desc <<-DESC
@@ -303,7 +345,7 @@ Capistrano::Configuration.instance.load do
         for "environment". The value is specified in :rails_env.
         Be sure to do deploy:restart after this.
       DESC
-      task :set_rails_env, :roles => all_admin_role_names do
+      task :set_rails_env do
         rails_env = fetch(:rails_env, "production")
         sudo "/usr/local/ec2onrails/bin/set_rails_env #{rails_env}"
       end
@@ -311,15 +353,15 @@ Capistrano::Configuration.instance.load do
       desc <<-DESC
         Upgrade to the newest versions of all Ubuntu packages.
       DESC
-      task :upgrade_packages, :roles => all_admin_role_names do
+      task :upgrade_packages do
         sudo "aptitude -q update"
-        run "export DEBIAN_FRONTEND=noninteractive; sudo aptitude -q -y safe-upgrade"
+        sudo "sh -c 'export DEBIAN_FRONTEND=noninteractive; aptitude -q -y safe-upgrade'"
       end
       
       desc <<-DESC
         Upgrade to the newest versions of all rubygems.
       DESC
-      task :upgrade_gems, :roles => all_admin_role_names do
+      task :upgrade_gems do
         sudo "gem update --system --no-rdoc --no-ri"
         sudo "gem update --no-rdoc --no-ri" do |ch, str, data|
           ch[:data] ||= ""
@@ -339,13 +381,14 @@ Capistrano::Configuration.instance.load do
         Install extra Ubuntu packages. Set ec2onrails_config[:packages], it \
         should be an array of strings.
         NOTE: the package installation will be non-interactive, if the packages \
-        require configuration either log in as 'admin' and run \
+        require configuration either log in as 'root' and run \
         'dpkg-reconfigure packagename' or replace the package's config files \
         using the 'ec2onrails:server:deploy_files' task.
       DESC
-      task :install_packages, :roles => all_admin_role_names do
+      task :install_packages do
+        sudo "aptitude -q update"
         if cfg[:packages] && cfg[:packages].any?
-          run "export DEBIAN_FRONTEND=noninteractive; sudo aptitude -q -y install #{cfg[:packages].join(' ')}"
+          sudo "sh -c 'export DEBIAN_FRONTEND=noninteractive; aptitude -q -y install #{cfg[:packages].join(' ')}'"
         end
       end
       
@@ -353,7 +396,7 @@ Capistrano::Configuration.instance.load do
         Install extra rubygems. Set ec2onrails_config[:rubygems], it should \
         be with an array of strings.
       DESC
-      task :install_gems, :roles => all_admin_role_names do
+      task :install_gems do
         if cfg[:rubygems]
           cfg[:rubygems].each do |gem|
             sudo "gem install #{gem} --no-rdoc --no-ri" do |ch, str, data|
@@ -371,11 +414,23 @@ Capistrano::Configuration.instance.load do
         end
       end
       
+      desc <<-DESC
+        Add extra gem sources to rubygems (to be able to fetch gems from for example gems.github.com).
+        Set ec2onrails_config[:rubygems_sources], it should be with an array of strings.
+      DESC
+      task :add_gem_sources do
+        if cfg[:rubygems_sources]
+          cfg[:rubygems_sources].each do |gem_source|
+            sudo "gem sources -a #{gem_source}"
+          end
+        end
+      end
+      
       desc <<-DESC
         A convenience task to upgrade existing packages and gems and install \
         specified new ones.
       DESC
-      task :upgrade_and_install_all, :roles => all_admin_role_names do
+      task :upgrade_and_install_all do
         upgrade_packages
         upgrade_gems
         install_packages
@@ -391,7 +446,7 @@ Capistrano::Configuration.instance.load do
         directory and file as the value. For example 'Africa/Abidjan' or \
         'posix/GMT' or 'Canada/Eastern'.
       DESC
-      task :set_timezone, :roles => all_admin_role_names do
+      task :set_timezone do
         if cfg[:timezone]
           sudo "bash -c 'echo #{cfg[:timezone]} > /etc/timezone'"
           sudo "cp /usr/share/zoneinfo/#{cfg[:timezone]} /etc/localtime"
@@ -406,7 +461,7 @@ Capistrano::Configuration.instance.load do
         inside here will be placed within the same directory structure \
         relative to the root of the server's filesystem.
       DESC
-      task :deploy_files, :roles => all_admin_role_names do
+      task :deploy_files do
         if cfg[:server_config_files_root]
           begin
             filename = "config_files.tar"
@@ -419,7 +474,7 @@ Capistrano::Configuration.instance.load do
             sudo "tar xvf #{remote_file} -o -C /"
           ensure
             rm_rf local_file
-            run "rm -f #{remote_file}"
+            sudo "rm -f #{remote_file}"
           end
         end
       end
@@ -429,7 +484,7 @@ Capistrano::Configuration.instance.load do
         to an array of strings. It's assumed that each service has a script \
         in /etc/init.d
       DESC
-      task :restart_services, :roles => all_admin_role_names do
+      task :restart_services do
         if cfg[:services_to_restart] && cfg[:services_to_restart].any?
           cfg[:services_to_restart].each do |service|
             run_init_script(service, "restart")
@@ -438,10 +493,11 @@ Capistrano::Configuration.instance.load do
       end
       
       desc <<-DESC
-        Set the email address that mail to the admin user forwards to.
+        Set the email address that mail to the app user forwards to.
       DESC
-      task :set_admin_mail_forward_address, :roles => all_admin_role_names do
-        put cfg[:admin_mail_forward_address], "/home/admin/.forward" if cfg[:admin_mail_forward_address]
+      task :set_mail_forward_address do
+        run "echo '#{cfg[:mail_forward_address]}' >> /home/app/.forward" if cfg[:mail_forward_address]
+        # put cfg[:admin_mail_forward_address], "/home/admin/.forward" if cfg[:admin_mail_forward_address]
       end
 
       desc <<-DESC
@@ -449,12 +505,62 @@ Capistrano::Configuration.instance.load do
         /etc/ssl/certs/default.pem and the SSL key file should be in
         /etc/ssl/private/default.key (use the deploy_files task).
       DESC
-      task :enable_ssl, :roles => :web_admin do
+      task :enable_ssl, :roles => :web do
+        #TODO: enable for nginx
         sudo "a2enmod ssl"
         sudo "a2ensite default-ssl"
-        run_init_script("apache2", "restart")
+        run_init_script("web_proxy", "restart")
+      end
+      
+      desc <<-DESC
+        Restrict the main user's sudo access.
+        Defaults the user to only be able to \
+        sudo to monit
+      DESC
+      task :restrict_sudo_access do
+        sudo "cp -f /etc/sudoers.restricted_access /etc/sudoers"
+        # run "ln -sf /etc/sudoers.restricted_access /etc/sudoers"
+      end
+
+      desc <<-DESC
+        Grant *FULL* sudo access to the main user.
+      DESC
+      task :grant_sudo_access do
+        allow_sudo
       end
+
+      @within_sudo = 0
+      def allow_sudo
+        @within_sudo += 1
+        old_user = fetch(:user)
+        if @within_sudo > 1
+          yield if block_given?
+          true
+        elsif capture("ls -l /etc/sudoers /etc/sudoers.full_access | awk '{print $5}'").split.uniq.size == 1
+          yield if block_given?
+          false
+        else
+          begin
+            # need to cheet and temporarily set the user to ROOT so we
+            # can (re)grant full sudo access.  
+            # we can do this because the root and app user have the same
+            # ssh login preferences....
+            set :user, 'root'
+            sessions.clear #clear out sessions cache..... this way the ssh connections are reinitialized
+            run "cp -f /etc/sudoers.full_access /etc/sudoers"
+            yield if block_given?
+          ensure
+            @within_sudo -= 1
+            server.restrict_sudo_access if block_given?
+            set :user, old_user
+            sessions.clear
+            true
+          end
+        end
+      end
+
     end
     
   end
 end
+