skippy-ec2onrails 0.9.10

data/server/files/usr/local/ec2onrails/bin/set_roles.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/ruby
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#    To customze the apache or nginx config files, you can setup a template like:
+#    /etc/ec2onrails/balancer_members.erb
+#    /etc/ec2onrails/nginx_upstream_members.erb
+#
+
+require 'erb'
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+include Ec2onrails::RolesHelper
+
+puts "Roles: "
+pp roles
+
+
+# web role:
+if in_role?(:web)
+  puts "setting up reverse proxy for web role.  starting port: #{web_starting_port} up to #{web_starting_port + web_num_instances - 1}"
+  
+  ## lets update/modify web balancer file templates, if need be
+  files_written = []
+  Dir["/etc/ec2onrails/*.erb"].each do |filename|
+    #what other variables would be helpful?
+    @web_port_range = web_port_range
+    @web_starting_port = web_starting_port
+    @roles = roles
+    file = ERB.new(IO.read(filename)).result(binding)
+    file_name = filename.sub(/\.erb$/, '')
+    files_written << file_name
+    File.open(file_name, 'w'){|f| f << file}
+  end
+  
+
+  if system("which apache2")
+    apache_config_file = "/etc/ec2onrails/balancer_members"
+    unless files_written.index(apache_config_file)
+      File.open(apache_config_file, "w") do |f|
+        roles[:app].each do |address|
+          web_port_range.each do |port|
+            f << "BalancerMember http://#{address}:#{port}\n"
+          end
+          f << "\n"
+        end
+      end
+    end
+  end
+
+  if system("which nginx")
+    nginx_config_file = "/etc/ec2onrails/nginx_upstream_members"
+    unless files_written.index(nginx_config_file)
+      File.open(nginx_config_file, "w") do |f|
+        f << "upstream mongrel{\n"
+        roles[:app].each do |address|
+          web_port_range.each do |port|
+            f << "\tserver #{address}:#{port};\n"
+          end
+        end
+        f << "fair;\n}\n"
+      end
+    end
+  end
+end
+
+if in_role?(:db_primary) || in_role?(:app)
+  db_primary_addr = roles[:db_primary][0]
+  add_etc_hosts_entry('db_primary', db_primary_addr)
+end
+
+

data/server/files/usr/local/ec2onrails/bin/setup_web_proxy.rb

@@ -0,0 +1,109 @@
+#!/usr/bin/ruby
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#    enable either apache or nginx as the proxy server.  
+#
+#    to do this, we symlink a few folders to use a common name web_proxy,
+#    which makes the handling of log files and what not easier to keep 
+#    track of
+
+require "rubygems"
+require "optiflag"
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+include Ec2onrails::RolesHelper
+
+PROXY_CHOICES = ["apache","nginx"]
+module CommandLineArgs extend OptiFlagSet
+  flag "mode" do
+    # this is our second new clause-level modifier
+    value_in_set PROXY_CHOICES
+    description "The web proxy server that will be installed and used."
+  end
+  and_process!
+end
+
+# make the log directories even if they won't be used...
+# keeps logrotate configs easy
+sudo "mkdir -p -m 755 /mnt/log/apache2"
+sudo "mkdir -p -m 755 /mnt/log/nginx"
+
+case ARGV.flags.mode
+when 'apache'
+  # it is a *LOT* easier to have apache2 pre-installed on the image 
+  # (and have the default server configs placed after apache2 is installed)
+  # 
+  # so assume it is installed by this time
+  # sudo "sh -c 'export DEBIAN_FRONTEND=noninteractive; aptitude -q -y install apache2'"
+  
+  sudo "a2enmod deflate"
+  sudo "a2enmod proxy_balancer"
+  sudo "a2enmod proxy_http"
+  sudo "a2enmod rewrite"
+  
+  
+  sudo "rm -rf /var/log/apache2"
+  sudo "ln -sf /mnt/log/apache2 /var/log/apache2"
+  sudo "ln -sf /etc/init.d/apache2 /etc/init.d/web_proxy"
+  sudo "ln -sf /mnt/log/apache2 /mnt/log/web_proxy"
+  
+when 'nginx'
+  #nginx does not have a precompiled package, so....
+  
+  src_dir = "#{Dir.pwd}/src"
+
+  nginx_img = "http://sysoev.ru/nginx/nginx-0.6.32.tar.gz"
+  fair_bal_img = "http://github.com/gnosek/nginx-upstream-fair/tarball/master"
+  nginx_dir = "#{src_dir}/nginx"
+  puts "installing nginx 6.32 (src dir: #{nginx_dir})"
+  run "mkdir -p -m 755 #{nginx_dir} &&  rm -rf #{nginx_dir}/*"
+  run "mkdir -p -m 755 #{nginx_dir}/modules/nginx-upstream-fair"
+  run "cd #{nginx_dir} && wget -q #{nginx_img} && tar -xzf nginx-0.6.32.tar.gz"
+  
+  run "cd #{nginx_dir}/modules && \
+       wget -q #{fair_bal_img} && \
+       tar -xzf *nginx-upstream-fair*.tar.gz -o -C ./nginx-upstream-fair && \
+       mv nginx-upstream-fair/*/* nginx-upstream-fair/."
+       
+  sudo "sh -c 'export DEBIAN_FRONTEND=noninteractive; aptitude -q -y install libpcre3-dev'"
+    
+  run "cd #{nginx_dir}/nginx-0.6.32 && \
+       ./configure \
+         --sbin-path=/usr/sbin \
+         --conf-path=/etc/nginx/nginx.conf \
+         --pid-path=/var/run/nginx.pid \
+         --with-http_ssl_module \
+         --with-http_stub_status_module \
+         --add-module=#{nginx_dir}/modules/nginx-upstream-fair && \
+       make && \
+       sudo make install"
+
+  run "sudo rm -rf /usr/local/nginx/logs; sudo ln -sf /mnt/log/nginx /usr/local/nginx/logs"
+  #an init.d script is in the default server config... lets link it up
+  sudo "ln -sf /etc/init.d/nginx /etc/init.d/web_proxy"
+  sudo "ln -sf /mnt/log/nginx /mnt/log/web_proxy"
+  # sudo "ln -sf /usr/local/nginx/sbin/nginx /usr/sbin/nginx"  
+  # sudo "ln -sf /usr/local/nginx/conf /etc/nginx"
+else
+  puts "The mode: #{ARGV.flags.mode} was not recognized.  Must be one of these #{["apache","nginx"].join(', ')}"
+  exit 1
+end
+
+#restart god... the config file will automatically pick up the changes but we need to restart god
+sudo("/etc/init.d/god restart")

data/server/files/usr/local/ec2onrails/config

@@ -0,0 +1,30 @@
+############################################################
+# This script is sourced by .bashrc and by /usr/local/ec2onrails/rebundle.sh
+# You shouldn't need to change anything in here
+
+if [ -r /mnt/aws-config/config ] ; then
+  . /mnt/aws-config/config
+  export EC2_PRIVATE_KEY="/mnt/aws-config/$KEY_FILE_NAME"
+  export EC2_CERT="/mnt/aws-config/$CERT_FILE_NAME"
+
+fi
+
+export EC2_HOME=/usr/local/ec2-api-tools
+export PATH="$PATH:$EC2_HOME/bin"
+
+export JAVA_HOME="/usr/lib/jvm/java-6-sun"
+
+# Set the bash prompt to include the rails environment name
+case "$TERM" in
+xterm-color)
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]-\[\033[01;35m\]`/usr/local/ec2onrails/bin/rails_env`\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+    ;;
+*)
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h-`/usr/local/ec2onrails/bin/rails_env`:\w\$ '
+    ;;
+esac
+
+# Set some useful aliases
+alias app="pushd /mnt/app/current && sudo su app && popd"
+alias log="tail -f /mnt/app/current/log/`/usr/local/ec2onrails/bin/rails_env`.log"
+alias logs="tail -f /mnt/app/current/log/*.log /mnt/log/syslog /mnt/log/*.log /mnt/log/*/*.log /mnt/log/*/*/.log"

data/server/files/usr/local/ec2onrails/lib/aws_helper.rb

@@ -0,0 +1,76 @@
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems'
+require 'aws/s3'
+require 'yaml'
+require 'erb'
+require 'fileutils'
+require "#{File.dirname(__FILE__)}/utils"
+
+module Ec2onrails
+  class AwsHelper
+
+    DEFAULT_CONFIG_FILE = "/mnt/app/current/config/aws.yml"
+    DEFAULT_CONFIG_FILE_OLD = "/mnt/app/current/config/s3.yml"
+
+    # make attributes available for specs
+    attr_accessor :config_file
+    attr_accessor :rails_env
+    attr_accessor :aws_access_key
+    attr_accessor :aws_secret_access_key
+    attr_accessor :bucket_base_name
+
+    def initialize(config_file = AwsHelper.default_config_file, rails_env = Utils.rails_env)
+      @rails_env = rails_env
+      @config_file = config_file
+
+      if File.exists?(@config_file)
+        aws_config = YAML::load(ERB.new(File.read(@config_file)).result)
+
+        # try to load the section for the current RAILS_ENV
+        section = aws_config[@rails_env]
+        if section.nil?
+          # fall back to keys at the root of the tree
+          section = aws_config
+        end
+
+        @aws_access_key        = section['aws_access_key']
+        @aws_secret_access_key = section['aws_secret_access_key']
+        @bucket_base_name      = section['bucket_base_name']
+      else
+        if !File.exists?('/mnt/aws-config/config')
+          raise "Can't find either #{@config_file} or /mnt/aws-config/config"
+        end
+        @aws_access_key        = get_bash_config('AWS_ACCESS_KEY_ID')
+        @aws_secret_access_key = get_bash_config('AWS_SECRET_ACCESS_KEY')
+        @bucket_base_name      = get_bash_config('BUCKET_BASE_NAME')  
+      end
+    end      
+
+    # load an env value from the shared config file
+    def get_bash_config(name)
+      `bash -c 'source /mnt/aws-config/config; echo $#{name}'`.strip
+    end   
+    
+    def self.default_config_file
+      File.exists?(DEFAULT_CONFIG_FILE) ? DEFAULT_CONFIG_FILE : DEFAULT_CONFIG_FILE_OLD
+    end
+    
+  end
+end

data/server/files/usr/local/ec2onrails/lib/god_helper.rb

@@ -0,0 +1,129 @@
+module  GodHelper
+  require '/usr/local/ec2onrails/lib/roles_helper'
+  require 'fileutils'
+
+  def default_configurations(w)
+    w.interval = 30.seconds
+    w.grace    = 30.seconds
+
+
+    w.behavior(:clean_pid_file)
+
+    w.start_if do |start|
+      start.condition(:process_running) do |c|
+        c.notify = {:contacts => ['default'], :category => 'process not started...starting'}
+        c.interval = 5.seconds
+        c.running = false
+      end
+    end
+
+    # w.start_if do |start|
+    #   start.condition(:process_running) do |c|
+    #     c.interval = 5.seconds
+    #     c.running = false
+    #     c.notify = {:contacts => ['default'], :category => 'process exited...restarting'}
+    #   end
+    # end
+
+    # determine when process has finished starting
+    # w.transition([:start, :restart], :up) do |on|
+    #   on.condition(:process_running) do |c|
+    #     c.running = true
+    #   end
+    # 
+    #   # failsafe
+    #   on.condition(:tries) do |c|
+    #     c.times = 8
+    #     c.within = 2.minutes
+    #     c.transition = :start
+    #   end
+    # end
+    # 
+    # # start if process is not running
+    # w.transition(:up, :start) do |on|
+    #   on.condition(:process_exits) do |c|
+    #     c.notify = {:contacts => ['default'], :category => 'process exited...restarting'}
+    #   end
+    # end
+  end
+
+  def restart_if_resource_hog(w, options={})
+    options = {:memory_usage => 175.megabytes, :cpu_usage => 50.percent}.merge(options)
+    w.restart_if do |restart|
+      if options[:memory_usage]
+        restart.condition(:memory_usage) do |c|
+          c.notify = {:contacts => ['default'], :category => "process over #{options[:memory_usage]/1.megabyte}MB.  restarting"}
+          c.above = options[:memory_usage]
+          c.times = [3,5]
+        end
+      end
+
+      if options[:cpu_usage]
+        restart.condition(:cpu_usage) do |c|
+          c.notify = {:contacts => ['default'], :category => "process over #{options[:cpu_usage]*100}%.  restarting"}
+          c.above = options[:cpu_usage]
+          c.times = 5
+        end
+      end
+
+      yield restart if block_given?
+
+    end
+  end
+  
+  def create_pid_dir(w)
+    pid_dir = File.dirname(w.pid_file)
+    return if File.exist?(pid_dir)
+  
+    #we need to make sure it is writable...but include all the directories we created...
+    starting_dir = pid_dir
+    starting_dir = File.dirname(starting_dir) until File.exist?(File.dirname(starting_dir))
+
+    FileUtils.mkdir_p(pid_dir)
+    FileUtils.chown_R(w.uid, w.gid, starting_dir)
+  end
+
+  def monitor_lifecycle(w)
+    # w.transition(:up, :unmonitored) do |on|
+    w.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.notify = {:contacts => ['default'], :category => 'process flapping...restarting'}
+        c.to_state = [:start, :restart]
+        c.times = 5
+        c.within = 5.minutes
+        c.transition = :unmonitored
+        c.retry_in = 10.minutes
+        c.retry_times = 5
+        c.retry_within = 2.hours
+      end
+    end
+    # w.lifecycle do |on|
+    #   on.condition(:flapping) do |c| 
+    #     c.notify = {:contacts => ['default'], :category => 'process flapping...restarting'}
+    #     c.to_state = [:start, :restart] 
+    #     c.times = 5 
+    #     c.within = 60.seconds 
+    #     c.retry_in = 10.minutes
+    #     c.retry_times = 5
+    #     c.retry_within = 2.hours
+    #   end 
+    # end
+    
+    # w.lifecycle do |on|
+    #   on.condition(:flapping) do |c|
+    #     c.to_state = [:start, :restart]
+    #     c.times = 5
+    #     c.within = 5.minutes
+    #     c.transition = :unmonitored
+    #     c.retry_in = 10.minutes
+    #     c.retry_times = 5
+    #     c.retry_within = 2.hours
+    #   end
+    # end
+  end
+  
+  class Configs
+     include Ec2onrails::RolesHelper
+  end
+  
+end

data/server/files/usr/local/ec2onrails/lib/god_patch.rb

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'god'
+
+# this is a hack to put the bin directory back at the top of the load path.... 
+# the /usr/bin/god script will try to load 'god', which is in the bin path of the gem
+# but is also the name of the module.... it should really be called something like god_bin 
+# or god_cli, so then it would have no problem being found.
+$:.unshift File.join($:[0], *%w[.. .. bin])
+
+module God
+  def self.control(name, command)
+    # get the list of items
+    items = Array(self.watches[name] || self.groups[name]).dup
+    
+    jobs = []
+    # do the command
+    case command
+      when "start", "monitor"
+        items.each { |w| jobs << Thread.new { w.monitor if w.state != :up } }
+      when "restart"
+        items.each { |w| jobs << Thread.new { w.move(:restart) } }
+      when "stop"
+        # items.each { |w| jobs << Thread.new { w.unmonitor.action(:stop) if w.state != :unmonitored } }
+        items.each do |w| 
+          jobs << Thread.new do
+            w.unmonitor if w.state != :unmonitored
+            w.action(:stop) if w.alive?
+          end
+        end
+      when "unmonitor"
+        items.each { |w| jobs << Thread.new { w.unmonitor if w.state != :unmonitored } }
+      when "remove"
+        items.each { |w| self.unwatch(w) }
+      else
+        raise InvalidCommandError.new
+    end
+    
+    jobs.each { |j| j.join }
+    
+    items.map { |x| x.name }
+  end
+  
+end

data/server/files/usr/local/ec2onrails/lib/mysql_helper.rb

@@ -0,0 +1,101 @@
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems'
+require 'aws/s3'
+require 'yaml'
+require 'erb'
+require 'fileutils'
+require "#{File.dirname(__FILE__)}/utils"
+
+module Ec2onrails
+  class MysqlHelper
+
+    DEFAULT_CONFIG_FILE = "/mnt/app/current/config/database.yml"
+
+    attr_accessor :database
+    attr_accessor :user
+    attr_accessor :password
+
+    def initialize(config_file = DEFAULT_CONFIG_FILE, rails_env = Utils.rails_env)
+      @rails_env = rails_env
+      load_db_config(config_file)
+    end
+
+    def load_db_config(config_file)
+      db_config = YAML::load(ERB.new(File.read(config_file)).result)
+      if db_config && db_config[@rails_env].nil?
+        puts "the rails environment '#{@rails_env}' was not found in this db onfig file: #{config_file}"
+      end
+      db_config = db_config[@rails_env]
+      @database = db_config['database']
+      @user = db_config['username']
+      @password = db_config['password']
+    end
+
+    def execute_sql(sql)
+      raise "@user not set" unless @user
+      raise "sql not given" unless sql
+      cmd = %{mysql -u #{@user} -e "#{sql}"}
+      cmd += " -p'#{@password}' " unless @password.nil?
+      Utils.run cmd
+    end
+    
+    def execute
+      require "mysql"
+      
+      begin
+        # connect to the MySQL server
+        dbh = Mysql.real_connect("localhost", "#{@user}", "#{@password}", "#{@database}")
+        yield dbh
+      rescue Mysql::Error => e
+        puts "Error code: #{e.errno}"
+        puts "Error message: #{e.error}"
+        puts "Error SQLSTATE: #{e.sqlstate}" if e.respond_to?("sqlstate")
+      ensure
+        # disconnect from server
+        dbh.close if dbh
+      end
+      
+      
+    end
+    
+    def dump(out_file, reset_logs)
+      cmd = "mysqldump --quick --single-transaction --create-options -u#{@user} "
+      if reset_logs
+        cmd += " --flush-logs --master-data=2 --delete-master-logs "
+      end
+      cmd += " -p'#{@password}' " unless @password.nil?
+      cmd += " #{@database} | gzip > #{out_file}"
+      Utils.run cmd    
+    end
+    
+    def load_from_dump(in_file)
+      cmd = "gunzip -c #{in_file} | mysql -u#{@user} "
+      cmd += " -p'#{@password}' " unless @password.nil?
+      cmd += " #{@database}"
+      Utils.run cmd
+    end
+    
+    def execute_binary_log(log_file)
+      cmd = "mysqlbinlog --database=#{@database} #{log_file} | mysql -u#{@user} "
+      cmd += " -p'#{@password}' " unless @password.nil?
+      Utils.run cmd
+    end
+  end
+end