hennk-hennk-ec2onrails 0.9.9.2

data/server/files/usr/local/ec2onrails/lib/s3_helper.rb

@@ -0,0 +1,126 @@
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems'
+require 'aws/s3'
+require 'yaml'
+require 'erb'
+require 'fileutils'
+require "#{File.dirname(__FILE__)}/utils"
+
+module Ec2onrails
+  class S3Helper
+
+    DEFAULT_CONFIG_FILE = "/mnt/app/current/config/s3.yml"
+  
+    # make attributes available for specs
+    attr_accessor :bucket
+    attr_accessor :dir
+    attr_accessor :config_file
+    attr_accessor :rails_env
+    attr_accessor :aws_access_key
+    attr_accessor :aws_secret_access_key
+    attr_accessor :bucket
+
+    def initialize(bucket, dir, config_file = DEFAULT_CONFIG_FILE, rails_env = Utils.rails_env)
+      @dir = dir
+      @config_file = config_file
+      @rails_env = rails_env
+      load_s3_config
+      @bucket = bucket || "#{@bucket_base_name}-#{Ec2onrails::Utils.hostname}"
+      AWS::S3::Base.establish_connection!(:access_key_id => @aws_access_key, :secret_access_key => @aws_secret_access_key, :use_ssl => true)
+    end
+
+    def load_s3_config
+      if File.exists?(@config_file)
+        s3_config = YAML::load(ERB.new(File.read(@config_file)).result)
+    
+        # try to load the section for the current RAILS_ENV
+        section = s3_config[@rails_env]
+        if section.nil?
+          # fall back to keys at the root of the tree
+          section = s3_config
+        end
+    
+        @aws_access_key        = section['aws_access_key']
+        @aws_secret_access_key = section['aws_secret_access_key']
+        @bucket_base_name      = section['bucket_base_name']
+      else
+        if !File.exists?('/mnt/aws-config/config')
+          raise "Can't find either #{@config_file} or /mnt/aws-config/config"
+        end
+        @aws_access_key        = get_bash_config('AWS_ACCESS_KEY_ID')
+        @aws_secret_access_key = get_bash_config('AWS_SECRET_ACCESS_KEY')
+        @bucket_base_name      = get_bash_config('BUCKET_BASE_NAME')  
+      end
+    end
+
+    def create_bucket
+      retries = 0
+      begin
+        AWS::S3::Bucket.find(@bucket)
+      rescue AWS::S3::NoSuchBucket
+        AWS::S3::Bucket.create(@bucket)
+        sleep 1 # If we try to use the bucket too quickly sometimes it's not found
+        retry if (retries += 1) < 15
+      end
+    end
+
+    def store_file(file)
+      create_bucket
+      AWS::S3::S3Object.store(s3_key(file), open(file), @bucket)
+    end
+
+    def retrieve_file(file)
+      key = s3_key(file)
+      AWS::S3::S3Object.find(key, @bucket)
+      open(file, 'w') do |f|
+        AWS::S3::S3Object.stream(key, @bucket) do |chunk|
+          f.write chunk
+        end
+      end
+    end
+    
+    def list_keys(filename_prefix)
+      prefix = @dir ? "#{@dir}/#{filename_prefix}" : filename_prefix
+      AWS::S3::Bucket.objects(@bucket, :prefix => prefix).collect{|obj| obj.key}
+    end
+    
+    def retrieve_files(filename_prefix, local_dir)
+      list_keys(filename_prefix).each do |k|
+        file = "#{local_dir}/#{File.basename(k)}"
+        retrieve_file(file)
+      end      
+    end
+
+    def delete_files(filename_prefix)
+      list_keys(filename_prefix).each do |k|
+        AWS::S3::S3Object.delete(k, @bucket)
+      end
+    end
+  
+    def s3_key(file)
+      @dir ? "#{@dir}/#{File.basename(file)}" : File.basename(file)
+    end
+
+    # load an env value from the shared config file
+    def get_bash_config(name)
+      `bash -c 'source /mnt/aws-config/config; echo $#{name}'`.strip
+    end
+  end
+end

data/server/files/usr/local/ec2onrails/lib/utils.rb

@@ -0,0 +1,16 @@
+module Ec2onrails
+  module Utils
+    def self.run(command)
+      result = system command
+      raise("error: #{$?}") unless result
+    end
+  
+    def self.rails_env
+      `/usr/local/ec2onrails/bin/rails_env`.strip
+    end
+    
+    def self.hostname
+      `hostname -s`.strip
+    end
+  end
+end

data/server/files/usr/local/ec2onrails/startup-scripts/every-startup/get-hostname.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Set the hostname to this instance's public hostname
+
+hostname `curl http://169.254.169.254/latest/meta-data/public-hostname`
+sudo monit reload
+

data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/create-dirs.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+make_dir() {
+  mkdir -p $1
+  if [ $2 ] ; then
+    chown -R $2 $1
+  fi
+}
+
+make_dir /mnt/app app:app
+
+make_dir /mnt/log
+make_dir /mnt/log/apache2    www-data:www-data
+make_dir /mnt/log/fsck
+make_dir /mnt/log/mysql      mysql:mysql
+
+make_dir /mnt/mysql_data     mysql:mysql
+make_dir /mnt/mysql_data/tmp mysql:mysql
+
+make_dir /mnt/tmp
+chmod 777 /mnt/tmp

data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/generate-default-web-cert-and-key.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Generate a new self-signed cert and key for https
+
+echo "Generating default self-signed SSL cert and key..."
+
+export RANDFILE=/tmp/randfile
+
+cd /tmp
+openssl genrsa -out server.key 1024
+openssl req -new -key server.key -out server.csr <<END
+CA
+.
+.
+.
+.
+.
+.
+.
+.
+
+END
+openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+
+mkdir -p /etc/ec2onrails/ssl/cert
+mkdir -p -m 700 /etc/ec2onrails/ssl/private
+mv server.key /etc/ec2onrails/ssl/private/ec2onrails-default.key
+mv server.crt /etc/ec2onrails/ssl/cert/ec2onrails-default.crt
+rm $RANDFILE
+rm server.csr

data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/misc.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+chmod 777 /tmp
+
+cp /etc/hosts /etc/hosts.original
+
+#make sure the monitrc files have the correct permissions set
+chmod 700 /etc/monit/*
+
+# Rebuild postfix db
+newaliases

data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/prepare-mysql-data-dir.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+mkdir -p /mnt/mysql_data/tmp
+chown -R mysql:mysql /mnt/mysql_data/
+cp -rp /var/lib/mysql/mysql /mnt/mysql_data/

data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/setup_credentials.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#    This file helps spread the root key so we can log in/deploy using the 
+#    app user instead of root, but with the same ec2 key
+
+#make sure we have the credentials
+/etc/init.d/ec2-get-credentials
+
+mkdir -p -m 700 /home/app/.ssh
+cp /root/.ssh/authorized_keys /home/app/.ssh
+chown -R app:app /home/app/.ssh
+
+mkdir -p -m 700 /home/admin/.ssh
+cp /root/.ssh/authorized_keys /home/admin/.ssh
+chown -R admin:admin /home/admin/.ssh

data/server/rakefile.rb

@@ -0,0 +1,245 @@
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# This script is meant to be run by build-ec2onrails.sh, which is run by
+# Eric Hammond's Ubuntu build script: http://alestic.com/
+# e.g.:
+# bash /mnt/ec2ubuntu-build-ami --script /mnt/ec2onrails/server/build-ec2onrails.sh ...
+
+
+
+require "rake/clean"
+require 'yaml'
+require 'erb'
+require "#{File.dirname(__FILE__)}/../gem/lib/ec2onrails/version"
+
+if `whoami`.strip != 'root'
+  raise "Sorry, this buildfile must be run as root."
+end
+
+@packages = %w(
+  adduser
+  apache2
+  aptitude
+  bison
+  ca-certificates
+  cron
+  curl
+  flex
+  gcc
+  git-core
+  irb
+  less
+  libdbm-ruby
+  libgdbm-ruby
+  libmysql-ruby
+  libopenssl-ruby
+  libreadline-ruby
+  libruby
+  libssl-dev
+  libyaml-ruby
+  libzlib-ruby
+  logrotate
+  make
+  mailx
+  memcached
+  mysql-client
+  mysql-server
+  nano
+  openssh-server
+  postfix
+  rdoc
+  ri
+  rsync
+  ruby
+  ruby1.8-dev
+  subversion
+  unzip
+  vim
+  wget
+  xfsprogs
+)
+
+# HACK: some packages just fail with apt-get but work fine
+#       with aptitude.  These generally are virtual packages
+@aptitude_packages = %w(
+  libmysqlclient-dev
+)
+
+# NOTE: the amazon-ec2 gem is now at github, maintained by
+#       grempe-amazon-ec2.  Will move back to regular amazon-ec2
+#       gem if/when he cuts a new release with volume and snapshot
+#       support included
+@rubygems = [
+  "grempe-amazon-ec2",
+  "aws-s3",
+  "memcache-client",
+  "mongrel",
+  "mongrel_cluster",
+  "optiflag",
+  "rails -v 2.2.2",
+  "rails -v 2.0.2",
+  "rake",
+  "archive-tar-minitar"
+]
+
+@build_root = "/mnt/build"
+@fs_dir = "#{@build_root}/ubuntu"
+
+@version = [Ec2onrails::VERSION::MAJOR, Ec2onrails::VERSION::MINOR, Ec2onrails::VERSION::TINY].join('.')
+
+task :default => :configure
+
+desc "Removes all build files"
+task :clean_all do |t|
+  rm_rf @build_root
+end
+
+desc "Use apt-get to install required packages inside the image's filesystem"
+task :install_packages do |t|
+  unless_completed(t) do
+    ENV['DEBIAN_FRONTEND'] = 'noninteractive'
+    ENV['LANG'] = ''
+    run_chroot "apt-get install -y #{@packages.join(' ')}"
+    run_chroot "apt-get clean"
+    
+    #lets run the aptitude-only packages
+    run_chroot "aptitude install -y #{@aptitude_packages.join(' ')}"
+    run_chroot "aptitude clean"
+  end
+end
+
+desc "Install required ruby gems inside the image's filesystem"
+task :install_gems => [:install_packages] do |t|
+  unless_completed(t) do
+    run_chroot "sh -c 'cd /tmp && wget -q http://rubyforge.org/frs/download.php/55066/rubygems-1.3.2.tgz && tar zxf rubygems-1.3.2.tgz'"
+    run_chroot "sh -c 'cd /tmp/rubygems-1.3.2 && ruby setup.rb'"
+    run_chroot "ln -sf /usr/bin/gem1.8 /usr/bin/gem"
+    run_chroot "gem update --system --no-rdoc --no-ri"
+    run_chroot "gem update --no-rdoc --no-ri"
+    run_chroot "gem sources -a http://gems.github.com"
+    @rubygems.each do |gem|
+      run_chroot "gem install #{gem} --no-rdoc --no-ri"
+    end
+  end
+end
+
+desc "Compile and install monit"
+task :install_monit => [:install_packages] do |t|
+  unless_completed(t) do
+    run_chroot "sh -c 'cd /tmp && wget -q http://www.tildeslash.com/monit/dist/monit-4.10.1.tar.gz'"
+    run_chroot "sh -c 'cd /tmp && tar xzvf monit-4.10.1.tar.gz'"
+    run_chroot "sh -c 'cd /tmp/monit-4.10.1 && ./configure  --sysconfdir=/etc/monit/ --localstatedir=/var/run && make && make install'"
+  end
+end
+
+desc "Configure the image"
+task :configure => [:install_gems, :install_monit] do |t|
+  unless_completed(t) do
+    sh("cp -r files/* #{@fs_dir}")
+    sh("find #{@fs_dir} -type d -name .svn | xargs rm -rf")
+
+    replace("#{@fs_dir}/etc/motd.tail", /!!VERSION!!/, "Version #{@version}")
+    
+    run_chroot "a2enmod deflate"
+    run_chroot "a2enmod proxy_balancer"
+    run_chroot "a2enmod proxy_http"
+    run_chroot "a2enmod rewrite"
+    
+    run_chroot "/usr/sbin/adduser --gecos ',,,' --disabled-password app"
+    run_chroot "/usr/sbin/adduser --gecos ',,,' --disabled-password admin"
+    run_chroot "/usr/sbin/adduser admin adm"
+    run_chroot "/usr/sbin/addgroup sudoers"
+    
+    run "echo '. /usr/local/ec2onrails/config' >> #{@fs_dir}/root/.bashrc"
+    run "echo '. /usr/local/ec2onrails/config' >> #{@fs_dir}/home/app/.bashrc"
+    run "echo '. /usr/local/ec2onrails/config' >> #{@fs_dir}/home/admin/.bashrc"
+    
+    %w(apache2 mysql auth.log daemon.log kern.log mail.err mail.info mail.log mail.warn syslog user.log).each do |f|
+      rm_rf "#{@fs_dir}/var/log/#{f}"
+      run_chroot "ln -sf /mnt/log/#{f} /var/log/#{f}"
+    end
+    
+    run "touch #{@fs_dir}/ec2onrails-first-boot"
+    
+    # TODO find out the most correct solution here, there seems to be a bug in
+    # both feisty and gutsy where the dhcp daemon runs as dhcp but the dir
+    # that it tries to write to is owned by root and not writable by others.
+    run_chroot "chown -R dhcp /var/lib/dhcp3"
+  end
+end
+
+desc "This task is for deploying the contents of /files to a running server image to test config file changes without rebuilding."
+task :deploy_files do |t|
+  raise "need 'key' and 'host' env vars defined" unless ENV['key'] && ENV['host']
+  run "rsync -rlvzcC --rsh='ssh -l root -i #{ENV['key']}' files/ #{ENV['host']}:/"
+end
+
+##################
+
+# Execute a given block and touch a stampfile. The block won't be run if the stampfile exists.
+def unless_completed(task, &proc)
+  stampfile = "#{@build_root}/#{task.name}.completed"
+  unless File.exists?(stampfile)
+    yield  
+    touch stampfile
+  end
+end
+
+def run_chroot(command, ignore_error = false)
+  run "chroot '#{@fs_dir}' #{command}", ignore_error
+end
+
+def run(command, ignore_error = false)
+  puts "*** #{command}" 
+  result = system command
+  raise("error: #{$?}") unless result || ignore_error
+end
+
+# def mount(type, mount_point)
+#   unless mounted?(mount_point)
+#     puts
+#     puts "********** Mounting #{type} on #{mount_point}..."
+#     puts
+#     run "mount -t #{type} none #{mount_point}"
+#   end
+# end
+# 
+# def mounted?(mount_point)
+#   mount_point_regex = mount_point.gsub(/\//, "\\/")
+#   `mount`.select {|line| line.match(/#{mount_point_regex}/) }.any?
+# end
+
+def replace_line(file, newline, linenum)
+  contents = File.open(file, 'r').readlines
+  contents[linenum - 1] = newline
+  File.open(file, 'w') do |f|
+    contents.each {|line| f << line}
+  end
+end
+
+def replace(file, pattern, text)
+  contents = File.open(file, 'r').readlines
+  contents.each do |line|
+    line.gsub!(pattern, text)
+  end
+  File.open(file, 'w') do |f|
+    contents.each {|line| f << line}
+  end
+end