skippy-ec2onrails 0.9.10 → 0.9.11

data/lib/ec2onrails/version.rb

@@ -20,7 +20,7 @@ module Ec2onrails #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 9
-    TINY  = 10
+    TINY  = 11
     # GEM_UPDATE  = 0
     # STRING = [MAJOR, MINOR, TINY, GEM_UPDATE].join('.')
     STRING = [MAJOR, MINOR, TINY].join('.')

data/server/files/etc/apache2/sites-available/app.common

@@ -43,7 +43,7 @@ RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f
 RewriteRule ^/(.*)$ balancer://mongrel_cluster%{REQUEST_URI} [P,QSA,L]
 
 # Deflate
-AddOutputFilterByType DEFLATE text/html text/plain text/xml application/xml application/xhtml+xml text/javascript text/css
+AddOutputFilterByType DEFLATE text/html text/plain text/xml application/xml application/xhtml+xml text/javascript application/x-javascript text/css
 BrowserMatch ^Mozilla/4 gzip-only-text/html
 BrowserMatch ^Mozilla/4.0[678] no-gzip
 BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
@@ -54,3 +54,8 @@ BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
 #DeflateFilterNote Ratio ratio_info
 #LogFormat '"%r" %{output_info}n/%{input_info}n (%{ratio_info}n%%)' deflate
 #CustomLog logs/myapp_deflate_log deflate
+
+# Fix rails "IP spoofing attack?!" error
+# see http://iprog.com/posting/2008/08/rails_500_error_ip_spoofing_attack
+# see http://rails.lighthouseapp.com/projects/8994/tickets/322-don-t-return-500-if-client-ip-and-x-forwarded-for-agree
+RequestHeader unset Client-IP

data/server/files/etc/cron.d/{backup_app_db_to_s3 → ec2onrails}

@@ -14,3 +14,11 @@
 # with EBS:
 #   Full snapshot every 2 hours
 11 */2 * * *  root  test -f /etc/mysql/conf.d/mysql-ec2-ebs.cnf && /usr/local/ec2onrails/bin/backup_app_db.rb
+
+
+# Check if the hostname has been updated every 15 minutes.
+# This can happen if the user associated or disassociated an Elastic IP
+12,27,42,57 * * * * root /usr/local/ec2onrails/bin/update_hostname
+
+
+

data/server/files/etc/cron.daily/app

@@ -4,6 +4,21 @@
 #      /usr/local/ec2onrails/bin/exec_runner
 #      to run a script under a specific role
 #      see the file for details
-test -e /mnt/app/current/script/daily || exit 0
+
 cd /mnt/app/current
-sudo -u app /usr/local/ec2onrails/bin/rails_env script/daily
+
+if test -e /mnt/app/current/script/cron/daily; then 
+   if test -f /mnt/app/current/script/cron/daily; then 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/daily;
+   else 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/daily/*;
+   fi
+   exit 0;
+fi
+
+#DEPRECATED: just for old usage....
+if test -e /mnt/app/current/script/daily
+then 
+   sudo -u app /usr/local/ec2onrails/bin/rails_env script/daily
+   exit 0;
+fi

data/server/files/etc/cron.hourly/app

@@ -5,6 +5,20 @@
 #      to run a script under a specific role
 #      see the file for details
 
-test -e /mnt/app/current/script/hourly || exit 0
 cd /mnt/app/current
-sudo -u app /usr/local/ec2onrails/bin/rails_env script/hourly
+
+if test -e /mnt/app/current/script/cron/hourly; then 
+   if test -f /mnt/app/current/script/cron/hourly; then 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/hourly;
+   else 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/hourly/*;
+   fi
+   exit 0;
+fi
+
+#DEPRECATED: just for old usage....
+if test -e /mnt/app/current/script/hourly
+then 
+   sudo -u app /usr/local/ec2onrails/bin/rails_env script/hourly
+   exit 0;
+fi

data/server/files/etc/cron.monthly/app

@@ -5,6 +5,20 @@
 #      to run a script under a specific role
 #      see the file for details
 
-test -e /mnt/app/current/script/monthly || exit 0
 cd /mnt/app/current
-sudo -u app /usr/local/ec2onrails/bin/rails_env script/monthly
+
+if test -e /mnt/app/current/script/cron/monthly; then 
+   if test -f /mnt/app/current/script/cron/monthly; then 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/monthly;
+   else 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/monthly/*;
+   fi
+   exit 0;
+fi
+
+#DEPRECATED: just for old usage....
+if test -e /mnt/app/current/script/monthly
+then 
+   sudo -u app /usr/local/ec2onrails/bin/rails_env script/monthly
+   exit 0;
+fi

data/server/files/etc/cron.weekly/app

@@ -5,6 +5,20 @@
 #      to run a script under a specific role
 #      see the file for details
 
-test -e /mnt/app/current/script/weekly || exit 0
 cd /mnt/app/current
-sudo -u app /usr/local/ec2onrails/bin/rails_env script/weekly
+
+if test -e /mnt/app/current/script/cron/weekly; then 
+   if test -f /mnt/app/current/script/cron/weekly; then 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/weekly;
+   else 
+      sudo -u app /usr/local/ec2onrails/bin/rails_env script/cron/weekly/*;
+   fi
+   exit 0;
+fi
+
+#DEPRECATED: just for old usage....
+if test -e /mnt/app/current/script/weekly
+then 
+   sudo -u app /usr/local/ec2onrails/bin/rails_env script/weekly
+   exit 0;
+fi

data/server/files/etc/ec2onrails/README

@@ -1,5 +1,5 @@
 You can place a *.erb file in this directory and the set_roles script will 
-format them.  This is helpful if you want to customize your balancer_memebers
+format them.  This is helpful if you want to customize your balancer_members
 or nginx_upstream_members files.  For example, if you want to define multiple
 instances for nginx to hook into, you can put something like this in
 nginx_upstream_members.erb:

data/server/files/etc/god/app.god

@@ -1,6 +1,8 @@
 # rolling restart idea plagiarized directly from:
 # http://blog.pragmatic-it.de/articles/2008/07/09/poor-mans-rolling-restart-for-thin-god
-restart_time  = 60.seconds #how long to restart the entire cluster
+#NOTE: this doesn't do what you think it does...
+#      requests are queued up at nginx and requests start to time out
+restart_time  = 2.seconds #how long to restart the entire cluster
 rolling_delay = (restart_time / @configs.web_num_instances.to_f).ceil
 @configs.web_port_range.each_with_index do |port, i|
   God.watch do |w|
@@ -20,13 +22,16 @@ rolling_delay = (restart_time / @configs.web_num_instances.to_f).ceil
     default_configurations(w)
     create_pid_dir(w)
     restart_if_resource_hog(w, :memory_usage => 170.megabytes) do |restart|
+      #NOTE: this will hit every instance, meaning every minute you have a hit for every port you have a mongrel on.  
+      #      adding the port number to the call just to help with making this obvious in the logs
       restart.condition(:http_response_code) do |c|
         c.code_is_not = %w(200 304)
         c.host = '127.0.0.1'
-        c.path = '/'
+        c.path = "/?port=#{port}" 
         c.port = port
         c.timeout = 10.seconds
         c.times = 2
+        c.interval = 1.minute
       end
     end
     

data/server/files/etc/god/dkim_filter.god

@@ -0,0 +1,20 @@
+if File.exists?('/etc/init.d/dkim-filter')
+  #we have it installed, so lets register it with God
+  God.watch do |w|
+    w.name = 'dkim_filter'
+    w.group = 'app'
+    w.autostart = false
+
+    w.start    = "/etc/init.d/dkim-filter start" 
+    w.stop     = "/etc/init.d/dkim-filter stop" 
+    w.restart  = "/etc/init.d/dkim-filter restart" 
+
+    w.pid_file = "/var/run/dkim-filter/dkim-filter.pid"
+
+    default_configurations(w)
+    create_pid_dir(w)
+    restart_if_resource_hog(w, :memory_usage => 20.megabytes, :cpu_usage => 10.percent)
+    monitor_lifecycle(w)
+  end
+end
+

data/server/files/etc/god/system.god

@@ -15,7 +15,7 @@ God.watch do |w|
     end 
 
     on.condition(:disk_usage) do |c| 
-      c.mount_point = "/mng"
+      c.mount_point = "/mnt"
       c.above = 75
     end 
     

data/server/files/etc/god/web.god

@@ -2,21 +2,23 @@ nginx_enabled = system("which nginx 2>&1 > /dev/null")
 
 God.watch do |w|
   applog(w, :info, "web: using #{nginx_enabled ? 'nginx' : 'apache2'}")
-
+  server_status_path = '/'
   if nginx_enabled
     w.name = "nginx"
     w.start = "/etc/init.d/nginx start"
     w.stop = "/etc/init.d/nginx stop"
     w.restart = "/etc/init.d/nginx restart"
     w.pid_file = "/var/run/nginx.pid"  
+    server_status_path = '/nginx_status'
   else
     w.name = "apache"
     w.start = "/etc/init.d/apache2 start"
     w.stop = "/etc/init.d/apache2 stop"
     w.restart = "/etc/init.d/apache2 restart"
     w.pid_file = "/var/run/apache2.pid"
+    server_status_path = '/server-status'
   end
-  w.grace = 5.seconds
+  w.grace = 30.seconds
   w.group = 'web'
   w.autostart = false
 
@@ -25,9 +27,9 @@ God.watch do |w|
     restart.condition(:http_response_code) do |c|
       c.host = '127.0.0.1'
       c.port = 80
-      c.path = '/'
+      c.path = server_status_path
       c.code_is_not = 200
-      c.timeout = 15.seconds
+      c.timeout = 5.seconds
       c.times = [3, 5] # 3 out of 5 intervals
     end
   end

data/server/files/etc/mysql/my.cnf

@@ -42,6 +42,8 @@ tmpdir          = /mnt/mysql_data/tmp
 language        = /usr/share/mysql/english
 skip-external-locking
 default-storage-engine = InnoDB
+character-set-server   = utf8
+collation-server       = utf8_general_ci
 
 #
 # Instead of skip-networking the default is now to listen only on
@@ -126,6 +128,7 @@ quote-names
 max_allowed_packet      = 16M
 
 [mysql]
+default-character-set   = utf8
 #no-auto-rehash # faster start of mysql but no tab completition
 
 [isamchk]

data/server/files/etc/nginx/nginx.conf

@@ -27,7 +27,7 @@ http {
   log_format main '$remote_addr [$time_local] '
                   '"$scheme $host $request" $status $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" '
-                  '($request_time');
+                  '($request_time)';
   
   # main access log
   access_log  /mnt/log/nginx/access.log  main;
@@ -51,6 +51,7 @@ http {
                     text/html \
                     text/css \
                     application/x-javascript \
+                    application/json \
                     text/xml \
                     application/xml \
                     application/xml+rss \
@@ -103,7 +104,15 @@ http {
     location = /500.html {
       root /mnt/app/current/public;
     }
-
+    
+    #hide hidden files and folders
+    location ~ /\..+ {
+      deny  all;
+    }
+    
+    #do not show the nginx version number in the server header
+    server_tokens off;
+    
     # this allows people to use images and css in their maintenance.html file
     if ($request_filename ~* \.(css|jpg|gif|png)$) {
 	    break;

data/server/files/etc/rcS.d/S91ec2-first-startup

@@ -1 +1,36 @@
-../init.d/ec2-first-startup
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# This script runs the ec2 startup scripts
+
+FILE=/ec2onrails-first-boot
+
+if [ -e $FILE ] ; then
+
+	# this is an ugly hack to make sure DHCP initialization is finished:
+	sleep 15
+
+	for SCRIPT in `ls -I README /usr/local/ec2onrails/startup-scripts/first-startup` ; do
+		"/usr/local/ec2onrails/startup-scripts/first-startup/$SCRIPT"
+	done
+	
+	rm $FILE
+fi

data/server/files/etc/rcS.d/S92ec2-every-startup

@@ -1 +1,29 @@
-../init.d/ec2-every-startup
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# This script runs the ec2 startup scripts
+
+# this is an ugly hack to make sure DHCP initialization is finished:
+sleep 15
+
+for SCRIPT in `ls -I README /usr/local/ec2onrails/startup-scripts/every-startup` ; do
+	"/usr/local/ec2onrails/startup-scripts/every-startup/$SCRIPT"
+done

data/server/files/etc/rcS.d/S99set_roles

@@ -1 +1,3 @@
-../init.d/set_roles
+#!/bin/sh
+
+/usr/local/ec2onrails/bin/set_roles.rb

data/server/files/etc/sudoers

@@ -1 +1,26 @@
-sudoers.full_access
+# /etc/sudoers
+#
+# This file MUST be edited with the 'visudo' command as root.
+#
+# See the man page for details on how to write a sudoers file.
+# Host alias specification
+
+# User alias specification
+
+# Cmnd alias specification
+
+# Defaults
+
+Defaults        !lecture,tty_tickets,!fqdn
+
+# User privilege specification
+root    ALL=(ALL) ALL
+
+# The 'app' user can run sudo without a password
+# This is a security hole.  Use sudoers.restricted when running in regular mode
+app     ALL=(ALL) NOPASSWD: ALL
+
+
+# If you add named administrator accounts, add them to the group 'sudoers'
+# to give them sudo access
+%sudoers  ALL=(ALL) ALL

data/server/files/usr/local/ec2onrails/bin/backup_app_db.rb

@@ -44,12 +44,13 @@ module CommandLineArgs extend OptiFlagSet
   optional_flag "dir"
   optional_switch_flag "incremental"
   optional_switch_flag "reset"
+  optional_switch_flag "no_ebs"
   and_process!
 end
 @mysql = Ec2onrails::MysqlHelper.new
 
 
-if File.exists?("/etc/mysql/conf.d/mysql-ec2-ebs.cnf")
+if File.exists?("/etc/mysql/conf.d/mysql-ec2-ebs.cnf")  && !ARGV.flags.no_ebs
   # we have ebs enabled....
         
   @aws   = Ec2onrails::AwsHelper.new
@@ -57,7 +58,7 @@ if File.exists?("/etc/mysql/conf.d/mysql-ec2-ebs.cnf")
   ec2 = EC2::Base.new( :access_key_id => @aws.aws_access_key, :secret_access_key => @aws.aws_secret_access_key )
   
   #lets make sure we have space: AMAZON puts a 500 limit on the number of snapshots
-  snaps = ec2.describe_snapshots['DescribeSnapshotsResponse']['snapshotSet']['item'] rescue nil
+  snaps = ec2.describe_snapshots['snapshotSet']['item'] rescue nil
   if snaps && snaps.size > 450
     # TODO:
     # can we make this a bit smarter?  With a limit of 500, that is difficult.  

data/server/files/usr/local/ec2onrails/bin/backup_dir.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/ruby
+      
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+  
+require "rubygems"
+require "optiflag"
+require "fileutils"
+require 'EC2'
+require "#{File.dirname(__FILE__)}/../lib/mysql_helper"
+require "#{File.dirname(__FILE__)}/../lib/s3_helper"
+require "#{File.dirname(__FILE__)}/../lib/aws_helper"
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+
+require "#{File.dirname(__FILE__)}/../lib/utils"
+
+# Only run if this instance is the db_pimrary
+# The original code would run on any instance that had /etc/init.d/mysql
+# Which was pretty much all instances no matter what role
+include Ec2onrails::RolesHelper
+
+
+module CommandLineArgs extend OptiFlagSet
+  curr_env = Ec2onrails::Utils.rails_env
+  default_bucket = "#{curr_env}_backup"
+
+  flag "dir" do
+    description "the directory that will be tarred and compressed and put on S3 with the name DIR_#{Ec2onrails::Utils.hostname}_TIMESTAMP.tgz"
+  end
+  
+  optional_flag "role" do
+    description "The role of this server, as defined by capistrano. ex. 'db', or 'app'  If not used, will be applied to all roles"
+  end
+
+  optional_flag "only_env" do
+    description "Only apply the script if it is running within this environment"
+  end
+  
+  optional_flag "bucket" do
+    description "The s3 bucket you would like to save this backup to.  Will default to #{default_bucket}"
+  end
+
+  optional_switch_flag "v" do
+    description "let you know if the script stopped because it was running in either a different role or environment than the one specified"
+  end
+  
+  and_process!
+end
+curr_env = Ec2onrails::Utils.rails_env
+default_bucket = "#{curr_env}_backup"
+
+verbose = ARGV.flags.v
+dir = ARGV.flags.dir
+bucket = ARGV.flags.bucket || default_bucket
+curr_env = Ec2onrails::Utils.rails_env
+default_bucket = "#{curr_env}_backup"
+
+if ARGV.flags.role && !in_role?(ARGV.flags.role.sub(/^:/, '').to_sym)
+  puts "This script is not being run because the server is not running under the #{role} role" if verbose
+  exit
+end
+
+if ARGV.flags.only_env && ARGV.flags.only_env.strip.downcase != curr_env.strip.downcase
+  puts "This script is not being run because the server is not running under the #{curr_env} environment" if verbose
+  exit
+end
+
+if !dir || File.exists?(dir)
+  puts "The directory '#{dir}' does not exist.  Please enter a valid, full path to a directory you would like backed up" if verbose
+end
+
+
+@s3 = Ec2onrails::S3Helper.new(bucket, dir)
+@s3.store_dir(dir, :compress => true)